From fa0c9e3195e8d87925e2e0b05248a4bcdcb70f31 Mon Sep 17 00:00:00 2001 From: Happy2C0de <46957159+Happy2C0de@users.noreply.github.com> Date: Wed, 13 Oct 2021 17:09:32 +0200 Subject: [PATCH 001/489] Add support for Openstack Compute Microversion (#1065) * Init commit to support compute microversion Signed-off-by: Happy2C0de <46957159+Happy2C0de@users.noreply.github.com> * Wrap NewComputeV2 to support microversions Signed-off-by: Happy2C0de <46957159+Happy2C0de@users.noreply.github.com> * Change compute-based function signatures to gophercloud.ServiceClient Signed-off-by: Happy2C0de <46957159+Happy2C0de@users.noreply.github.com> * Move computeAPIVersion validation to getNewComputeV2 func Signed-off-by: Happy2C0de <46957159+Happy2C0de@users.noreply.github.com> * Undo removal of osErrorToTerminalError Signed-off-by: Happy2C0de <46957159+Happy2C0de@users.noreply.github.com> * Add error statement when calling getNewComputeV2 Signed-off-by: Happy2C0de <46957159+Happy2C0de@users.noreply.github.com> --- docs/cloud-provider.md | 2 + examples/openstack-machinedeployment.yaml | 1 + .../provider/openstack/helper.go | 47 ++++++++++--------- .../provider/openstack/provider.go | 44 +++++++++++------ .../provider/openstack/provider_test.go | 10 ++++ .../provider/openstack/types/types.go | 1 + 6 files changed, 70 insertions(+), 35 deletions(-) diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index 0514d7384..d79f1c88d 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -121,6 +121,8 @@ availabilityZone: "" region: "" # the name of the network to use network: "" +# compute microversion +computeAPIVersion: "" # set trust-device-path flag for kubelet trustDevicePath: false # set root disk size diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index 17256893a..8320c16da 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -100,6 +100,7 @@ spec: namespace: kube-system name: machine-controller-openstack key: region + computeAPIVersion: "2.67" image: "Ubuntu 18.04 amd64" flavor: "m1.small" rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" diff --git a/pkg/cloudprovider/provider/openstack/helper.go b/pkg/cloudprovider/provider/openstack/helper.go index 0b5548d1f..957e0f57c 100644 --- a/pkg/cloudprovider/provider/openstack/helper.go +++ b/pkg/cloudprovider/provider/openstack/helper.go @@ -19,6 +19,7 @@ package openstack import ( "errors" "fmt" + "strconv" "sync" "time" @@ -77,12 +78,26 @@ func getRegions(client *gophercloud.ProviderClient) ([]osregions.Region, error) return regions, nil } -func getAvailabilityZones(client *gophercloud.ProviderClient, region string) ([]osavailabilityzones.AvailabilityZone, error) { - computeClient, err := goopenstack.NewComputeV2(client, gophercloud.EndpointOpts{Region: region}) +func getNewComputeV2(client *gophercloud.ProviderClient, c *Config) (*gophercloud.ServiceClient, error) { + computeClient, err := goopenstack.NewComputeV2(client, gophercloud.EndpointOpts{Region: c.Region}) if err != nil { return nil, err } + if c.ComputeAPIVersion != "" { + // Validation - empty value default to microversion 2.0=2.1 + version, err := strconv.ParseFloat(c.ComputeAPIVersion, 32) + if err != nil || version < 2.0 { + return nil, fmt.Errorf("invalid computeAPIVersion: %v", err) + } + + // See https://github.com/gophercloud/gophercloud/blob/master/docs/MICROVERSIONS.md + computeClient.Microversion = c.ComputeAPIVersion + } + return computeClient, nil +} + +func getAvailabilityZones(computeClient *gophercloud.ServiceClient, c *Config) ([]osavailabilityzones.AvailabilityZone, error) { allPages, err := osavailabilityzones.List(computeClient).AllPages() if err != nil { return nil, err @@ -90,14 +105,14 @@ func getAvailabilityZones(client *gophercloud.ProviderClient, region string) ([] return osavailabilityzones.ExtractAvailabilityZones(allPages) } -func getAvailabilityZone(client *gophercloud.ProviderClient, region, name string) (*osavailabilityzones.AvailabilityZone, error) { - zones, err := getAvailabilityZones(client, region) +func getAvailabilityZone(computeClient *gophercloud.ServiceClient, c *Config) (*osavailabilityzones.AvailabilityZone, error) { + zones, err := getAvailabilityZones(computeClient, c) if err != nil { return nil, err } for _, z := range zones { - if z.ZoneName == name { + if z.ZoneName == c.AvailabilityZone { return &z, nil } } @@ -105,15 +120,10 @@ func getAvailabilityZone(client *gophercloud.ProviderClient, region, name string return nil, errNotFound } -func getImageByName(client *gophercloud.ProviderClient, region, name string) (*osimages.Image, error) { - computeClient, err := goopenstack.NewComputeV2(client, gophercloud.EndpointOpts{Region: region}) - if err != nil { - return nil, err - } - +func getImageByName(computeClient *gophercloud.ServiceClient, c *Config) (*osimages.Image, error) { var allImages []osimages.Image - pager := osimages.ListDetail(computeClient, osimages.ListOpts{Name: name}) - err = pager.EachPage(func(page pagination.Page) (bool, error) { + pager := osimages.ListDetail(computeClient, osimages.ListOpts{Name: c.Image}) + err := pager.EachPage(func(page pagination.Page) (bool, error) { images, err := osimages.ExtractImages(page) if err != nil { return false, err @@ -131,16 +141,11 @@ func getImageByName(client *gophercloud.ProviderClient, region, name string) (*o return &allImages[0], nil } -func getFlavor(client *gophercloud.ProviderClient, region, name string) (*osflavors.Flavor, error) { - computeClient, err := goopenstack.NewComputeV2(client, gophercloud.EndpointOpts{Region: region}) - if err != nil { - return nil, err - } - +func getFlavor(computeClient *gophercloud.ServiceClient, c *Config) (*osflavors.Flavor, error) { var allFlavors []osflavors.Flavor pager := osflavors.ListDetail(computeClient, osflavors.ListOpts{}) - err = pager.EachPage(func(page pagination.Page) (bool, error) { + err := pager.EachPage(func(page pagination.Page) (bool, error) { flavors, err := osflavors.ExtractFlavors(page) if err != nil { return false, err @@ -153,7 +158,7 @@ func getFlavor(client *gophercloud.ProviderClient, region, name string) (*osflav } for _, f := range allFlavors { - if f.Name == name { + if f.Name == c.Flavor { return &f, nil } } diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index d7c158b8d..47929c6d4 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -89,6 +89,7 @@ type Config struct { TenantID string TokenID string Region string + ComputeAPIVersion string // Machine details Image string @@ -236,6 +237,10 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt if err != nil { return nil, nil, nil, err } + c.ComputeAPIVersion, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ComputeAPIVersion) + if err != nil { + return nil, nil, nil, err + } c.RootDiskSizeGB = rawConfig.RootDiskSizeGB c.RootDiskVolumeType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.RootDiskVolumeType) if err != nil { @@ -326,9 +331,14 @@ func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, } } + computeClient, err := getNewComputeV2(client, c) + if err != nil { + return spec, osErrorToTerminalError(err, "failed to get computeClient") + } + if c.AvailabilityZone == "" { klog.V(3).Infof("Trying to default availability zone for machine '%s'...", spec.Name) - availabilityZones, err := getAvailabilityZones(client, c.Region) + availabilityZones, err := getAvailabilityZones(computeClient, c) if err != nil { return spec, osErrorToTerminalError(err, "failed to get availability zones") } @@ -429,7 +439,13 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { return fmt.Errorf("failed to get region %q: %v", c.Region, err) } - image, err := getImageByName(client, c.Region, c.Image) + // Get OS Compute Client + computeClient, err := getNewComputeV2(client, c) + if err != nil { + return fmt.Errorf("failed to get compute client: %v", err) + } + + image, err := getImageByName(computeClient, c) if err != nil { return fmt.Errorf("failed to get image %q: %v", c.Image, err) } @@ -440,7 +456,7 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { } } - if _, err := getFlavor(client, c.Region, c.Flavor); err != nil { + if _, err := getFlavor(computeClient, c); err != nil { return fmt.Errorf("failed to get flavor %q: %v", c.Flavor, err) } @@ -463,7 +479,7 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { } } - if _, err := getAvailabilityZone(client, c.Region, c.AvailabilityZone); err != nil { + if _, err := getAvailabilityZone(computeClient, c); err != nil { return fmt.Errorf("failed to get availability zone %q: %v", c.AvailabilityZone, err) } if pc.OperatingSystem == providerconfigtypes.OperatingSystemSLES { @@ -500,12 +516,17 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr return nil, osErrorToTerminalError(err, "failed to get a openstack client") } - flavor, err := getFlavor(client, c.Region, c.Flavor) + computeClient, err := getNewComputeV2(client, c) + if err != nil { + return nil, osErrorToTerminalError(err, "failed to get a openstack client") + } + + flavor, err := getFlavor(computeClient, c) if err != nil { return nil, osErrorToTerminalError(err, fmt.Sprintf("failed to get flavor %s", c.Flavor)) } - image, err := getImageByName(client, c.Region, c.Image) + image, err := getImageByName(computeClient, c) if err != nil { return nil, osErrorToTerminalError(err, fmt.Sprintf("failed to get image %s", c.Image)) } @@ -530,11 +551,6 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr securityGroups = append(securityGroups, securityGroupName) } - computeClient, err := goopenstack.NewComputeV2(client, gophercloud.EndpointOpts{Availability: gophercloud.AvailabilityPublic, Region: c.Region}) - if err != nil { - return nil, osErrorToTerminalError(err, "failed to get compute client") - } - // we check against reserved tags in Validation method allTags := c.Tags allTags[machineUIDMetaKey] = string(machine.UID) @@ -667,7 +683,7 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.P return false, osErrorToTerminalError(err, "failed to get a openstack client") } - computeClient, err := goopenstack.NewComputeV2(client, gophercloud.EndpointOpts{Availability: gophercloud.AvailabilityPublic, Region: c.Region}) + computeClient, err := getNewComputeV2(client, c) if err != nil { return false, osErrorToTerminalError(err, "failed to get compute client") } @@ -697,7 +713,7 @@ func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provider return nil, osErrorToTerminalError(err, "failed to get a openstack client") } - computeClient, err := goopenstack.NewComputeV2(client, gophercloud.EndpointOpts{Availability: gophercloud.AvailabilityPublic, Region: c.Region}) + computeClient, err := getNewComputeV2(client, c) if err != nil { return nil, osErrorToTerminalError(err, "failed to get compute client") } @@ -740,7 +756,7 @@ func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { return osErrorToTerminalError(err, "failed to get a openstack client") } - computeClient, err := goopenstack.NewComputeV2(client, gophercloud.EndpointOpts{Availability: gophercloud.AvailabilityPublic, Region: c.Region}) + computeClient, err := getNewComputeV2(client, c) if err != nil { return osErrorToTerminalError(err, "failed to get compute client") } diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index 2bd807f3b..5c60d1729 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -145,6 +145,7 @@ type openstackProviderSpecConf struct { RootDiskVolumeType string ApplicationCredentialID string ApplicationCredentialSecret string + ComputeAPIVersion string } func (o openstackProviderSpecConf) rawProviderSpec(t *testing.T) []byte { @@ -162,6 +163,9 @@ func (o openstackProviderSpecConf) rawProviderSpec(t *testing.T) []byte { "region": "eu-de", "instanceReadyCheckPeriod": "2m", "instanceReadyCheckTimeout": "2m", + {{- if .ComputeAPIVersion }} + "computeAPIVersion": {{ .ComputeAPIVersion }}, + {{- end }} {{- if .RootDiskSizeGB }} "rootDiskSizeGB": {{ .RootDiskSizeGB }}, {{- end }} @@ -240,6 +244,12 @@ func TestCreateServer(t *testing.T) { userdata: "fake-userdata", wantServerReq: expectedServerRequest, }, + { + name: "Compute API Version", + specConf: openstackProviderSpecConf{ComputeAPIVersion: "2.67"}, + userdata: "fake-userdata", + wantServerReq: expectedServerRequest, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/cloudprovider/provider/openstack/types/types.go b/pkg/cloudprovider/provider/openstack/types/types.go index bba08e5b9..5be18b599 100644 --- a/pkg/cloudprovider/provider/openstack/types/types.go +++ b/pkg/cloudprovider/provider/openstack/types/types.go @@ -34,6 +34,7 @@ type RawConfig struct { Region providerconfigtypes.ConfigVarString `json:"region,omitempty"` InstanceReadyCheckPeriod providerconfigtypes.ConfigVarString `json:"instanceReadyCheckPeriod,omitempty"` InstanceReadyCheckTimeout providerconfigtypes.ConfigVarString `json:"instanceReadyCheckTimeout,omitempty"` + ComputeAPIVersion providerconfigtypes.ConfigVarString `json:"computeAPIVersion,omitempty"` // Machine details Image providerconfigtypes.ConfigVarString `json:"image"` From ca58e4f596e45d0c72ea3dde4d9fa98058eb42a3 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Fri, 15 Oct 2021 11:45:32 +0200 Subject: [PATCH 002/489] Consider CPU arch when selecting default AMIs in AWS (#1070) * Consider instance type CPU architecture in default AMI IDs * Add ARM64 filters for OSes that have an equivalent image Signed-off-by: Marvin Beckers * always run e2e test for ARM machines on AWS * return and consume single CPUArchitecture in AMI functions for AWS Signed-off-by: Marvin Beckers --- .prow.yaml | 2 +- pkg/cloudprovider/provider/aws/provider.go | 140 ++++++++++++++---- pkg/cloudprovider/provider/aws/types/types.go | 9 ++ .../machinedeployment-aws-arm-machines.yaml | 2 +- 4 files changed, 121 insertions(+), 32 deletions(-) diff --git a/.prow.yaml b/.prow.yaml index 05cd4228a..f749ee1ca 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -317,7 +317,7 @@ presubmits: cpu: 500m - name: pull-machine-controller-e2e-aws-arm - always_run: false + always_run: true decorate: true error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 985994e2a..9b7e50fe7 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -95,41 +95,78 @@ var ( ec2.VolumeTypeSt1, ) - amiFilters = map[providerconfigtypes.OperatingSystem]amiFilter{ + amiFilters = map[providerconfigtypes.OperatingSystem]map[awstypes.CPUArchitecture]amiFilter{ providerconfigtypes.OperatingSystemCentOS: { - description: "CentOS Linux 7 x86_64 HVM EBS*", - // The AWS marketplace ID from AWS - owner: "679593333241", - productCode: "aw0evgkw8e5c1q413zgy5pjce", + awstypes.CPUArchitectureX86_64: { + description: "CentOS Linux 7 x86_64 HVM EBS*", + // The AWS marketplace ID from AWS + owner: "679593333241", + productCode: "aw0evgkw8e5c1q413zgy5pjce", + }, + // 2021-10-14 - No CentOS 7 ARM64 image available under legacy product code }, providerconfigtypes.OperatingSystemAmazonLinux2: { - description: "Amazon Linux 2 AMI * x86_64 HVM gp2", - // The AWS marketplace ID from Amazon - owner: "137112412989", + awstypes.CPUArchitectureX86_64: { + description: "Amazon Linux 2 AMI * x86_64 HVM gp2", + // The AWS marketplace ID from Amazon + owner: "137112412989", + }, + awstypes.CPUArchitectureARM64: { + description: "Amazon Linux 2 LTS Arm64 AMI * arm64 HVM gp2", + // The AWS marketplace ID from Amazon + owner: "137112412989", + }, }, providerconfigtypes.OperatingSystemUbuntu: { - // Be as precise as possible - otherwise we might get a nightly dev build - description: "Canonical, Ubuntu, 20.04 LTS, amd64 focal image build on ????-??-??", - // The AWS marketplace ID from Canonical - owner: "099720109477", + awstypes.CPUArchitectureX86_64: { + // Be as precise as possible - otherwise we might get a nightly dev build + description: "Canonical, Ubuntu, 20.04 LTS, amd64 focal image build on ????-??-??", + // The AWS marketplace ID from Canonical + owner: "099720109477", + }, + awstypes.CPUArchitectureARM64: { + // Be as precise as possible - otherwise we might get a nightly dev build + description: "Canonical, Ubuntu, 20.04 LTS, arm64 focal image build on ????-??-??", + // The AWS marketplace ID from Canonical + owner: "099720109477", + }, }, providerconfigtypes.OperatingSystemSLES: { - // Be as precise as possible - otherwise we might get a nightly dev build - description: "SUSE Linux Enterprise Server 15 SP1 (HVM, 64-bit, SSD-Backed)", - // The AWS marketplace ID from SLES - owner: "013907871322", + awstypes.CPUArchitectureX86_64: { + // Be as precise as possible - otherwise we might get a nightly dev build + description: "SUSE Linux Enterprise Server 15 SP1 (HVM, 64-bit, SSD-Backed)", + // The AWS marketplace ID from SLES + owner: "013907871322", + }, + awstypes.CPUArchitectureARM64: { + // Be as precise as possible - otherwise we might get a nightly dev build + description: "SUSE Linux Enterprise Server 15 SP1 (HVM, 64-bit, SSD-Backed)", + // The AWS marketplace ID from SLES + owner: "013907871322", + }, }, providerconfigtypes.OperatingSystemRHEL: { - // Be as precise as possible - otherwise we might get a nightly dev build - description: "Provided by Red Hat, Inc.", - // The AWS marketplace ID from RedHat - owner: "309956199498", + awstypes.CPUArchitectureX86_64: { + // Be as precise as possible - otherwise we might get a nightly dev build + description: "Provided by Red Hat, Inc.", + // The AWS marketplace ID from RedHat + owner: "309956199498", + }, + awstypes.CPUArchitectureARM64: { + // Be as precise as possible - otherwise we might get a nightly dev build + description: "Provided by Red Hat, Inc.", + // The AWS marketplace ID from RedHat + owner: "309956199498", + }, }, providerconfigtypes.OperatingSystemFlatcar: { - // Be as precise as possible - otherwise we might get a nightly dev build - description: "Flatcar Container Linux stable *", - // The AWS marketplace ID from AWS - owner: "075585003325", + awstypes.CPUArchitectureX86_64: { + // Be as precise as possible - otherwise we might get a nightly dev build + description: "Flatcar Container Linux stable *", + // The AWS marketplace ID from AWS + owner: "075585003325", + }, + // 2021-10-14 - Flatcar stable does not support ARM yet (only alpha channels supports it) }, } @@ -170,16 +207,21 @@ type amiFilter struct { productCode string } -func getDefaultAMIID(client *ec2.EC2, os providerconfigtypes.OperatingSystem, region string) (string, error) { +func getDefaultAMIID(client *ec2.EC2, os providerconfigtypes.OperatingSystem, region string, cpuArchitecture awstypes.CPUArchitecture) (string, error) { cacheLock.Lock() defer cacheLock.Unlock() - filter, osSupported := amiFilters[os] + osFilter, osSupported := amiFilters[os] if !osSupported { return "", fmt.Errorf("operating system %q not supported", os) } - cacheKey := fmt.Sprintf("ami-id-%s-%s", region, os) + filter, archSupported := osFilter[cpuArchitecture] + if !archSupported { + return "", fmt.Errorf("CPU architecture '%s' not supported for operating system '%s'", cpuArchitecture, os) + } + + cacheKey := fmt.Sprintf("ami-id-%s-%s-%s", region, os, cpuArchitecture) amiID, found := cache.Get(cacheKey) if found { klog.V(3).Info("found AMI-ID in cache!") @@ -203,7 +245,7 @@ func getDefaultAMIID(client *ec2.EC2, os providerconfigtypes.OperatingSystem, re }, { Name: aws.String("architecture"), - Values: aws.StringSlice([]string{"x86_64"}), + Values: aws.StringSlice([]string{string(cpuArchitecture)}), }, }, } @@ -221,7 +263,7 @@ func getDefaultAMIID(client *ec2.EC2, os providerconfigtypes.OperatingSystem, re } if len(imagesOut.Images) == 0 { - return "", fmt.Errorf("could not find Image for '%s'", os) + return "", fmt.Errorf("could not find Image for '%s' with arch '%s'", os, cpuArchitecture) } if os == providerconfigtypes.OperatingSystemRHEL { @@ -244,6 +286,34 @@ func getDefaultAMIID(client *ec2.EC2, os providerconfigtypes.OperatingSystem, re return *image.ImageId, nil } +func getCPUArchitecture(client *ec2.EC2, instanceType string) (awstypes.CPUArchitecture, error) { + // read the instance type to know which cpu architecture is needed in the AMI + instanceTypes, err := client.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{ + InstanceTypes: []*string{aws.String(instanceType)}, + }) + + if err != nil { + return "", err + } + + if len(instanceTypes.InstanceTypes) != 1 { + return "", fmt.Errorf("unexpected length of instance type list: %d", len(instanceTypes.InstanceTypes)) + } + + if instanceTypes.InstanceTypes[0].ProcessorInfo != nil && + len(instanceTypes.InstanceTypes[0].ProcessorInfo.SupportedArchitectures) > 0 { + for _, v := range instanceTypes.InstanceTypes[0].ProcessorInfo.SupportedArchitectures { + // machine-controller currently supports x86_64 and ARM64, so only CPU architectures + // that are supported will be returned if found in the AWS API response + if arch := awstypes.CPUArchitecture(*v); arch == awstypes.CPUArchitectureX86_64 || arch == awstypes.CPUArchitectureARM64 { + return arch, nil + } + } + } + + return "", errors.New("returned instance type data did not include supported architectures") +} + func getDefaultRootDevicePath(os providerconfigtypes.OperatingSystem) (string, error) { const ( rootDevicePathSDA = "/dev/sda1" @@ -537,7 +607,17 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr amiID := config.AMI if amiID == "" { - if amiID, err = getDefaultAMIID(ec2Client, pc.OperatingSystem, config.Region); err != nil { + // read the instance type to know which cpu architecture is needed in the AMI + cpuArchitecture, err := getCPUArchitecture(ec2Client, config.InstanceType) + + if err != nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Failed to find instance type %s in region %s: %v", config.InstanceType, config.Region, err), + } + } + + if amiID, err = getDefaultAMIID(ec2Client, pc.OperatingSystem, config.Region, cpuArchitecture); err != nil { return nil, cloudprovidererrors.TerminalError{ Reason: common.InvalidConfigurationMachineError, Message: fmt.Sprintf("Failed to get AMI-ID for operating system %s in region %s: %v", pc.OperatingSystem, config.Region, err), diff --git a/pkg/cloudprovider/provider/aws/types/types.go b/pkg/cloudprovider/provider/aws/types/types.go index c19b17932..75565c078 100644 --- a/pkg/cloudprovider/provider/aws/types/types.go +++ b/pkg/cloudprovider/provider/aws/types/types.go @@ -48,3 +48,12 @@ type SpotInstanceConfig struct { PersistentRequest providerconfigtypes.ConfigVarBool `json:"persistentRequest,omitempty"` InterruptionBehavior providerconfigtypes.ConfigVarString `json:"interruptionBehavior,omitempty"` } + +// CPUArchitecture defines processor architectures returned by the AWS API +type CPUArchitecture string + +const ( + CPUArchitectureARM64 CPUArchitecture = "arm64" + CPUArchitectureX86_64 CPUArchitecture = "x86_64" + CPUArchitectureI386 CPUArchitecture = "i386" +) diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml index c0f35bfa9..e4c0d6375 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml @@ -34,7 +34,7 @@ spec: diskSize: 50 diskType: "gp2" ebsVolumeEncrypted: false - ami: "ami-0489277f0e9a94f8d" + ami: "<< AMI >>" securityGroupIDs: - "sg-a2c195ca" tags: From 6764efaa00adadb1b3c22a879f4f5f0887133973 Mon Sep 17 00:00:00 2001 From: Yakul Garg <2000yeshu@gmail.com> Date: Fri, 15 Oct 2021 22:40:31 +0530 Subject: [PATCH 003/489] Allow-downgrades for dockerce and containerd (#1073) If dockerce or/and containerd is/are already present on the machine, then the setup script will not install them if the version installed is already higher than what we require, so I have added a allow-downgrades flag to the installation script Signed-off-by: Yakul Garg <2000yeshu@gmail.com> Co-authored-by: Yakul Garg --- pkg/containerruntime/containerd.go | 2 +- pkg/containerruntime/docker.go | 2 +- pkg/userdata/ubuntu/testdata/containerd.yaml | 2 +- pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml | 2 +- .../ubuntu/testdata/kubelet-version-without-v-prefix.yaml | 2 +- pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml | 2 +- pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml | 2 +- .../ubuntu/testdata/openstack-overwrite-cloud-config.yaml | 2 +- pkg/userdata/ubuntu/testdata/openstack.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.17.16.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.18.14.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.19.4.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.20.1.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere.yaml | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index 63b18ec0f..e0e49106c 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -144,7 +144,7 @@ Restart=always EnvironmentFile=-/etc/environment EOF -apt-get install -y containerd.io={{ .ContainerdVersion }}* +apt-get install -y --allow-downgrades containerd.io={{ .ContainerdVersion }}* apt-mark hold containerd.io systemctl daemon-reload diff --git a/pkg/containerruntime/docker.go b/pkg/containerruntime/docker.go index 40f8110b4..580af1a68 100644 --- a/pkg/containerruntime/docker.go +++ b/pkg/containerruntime/docker.go @@ -151,7 +151,7 @@ Restart=always EnvironmentFile=-/etc/environment EOF -apt-get install -y \ +apt-get install --allow-downgrades -y \ {{- if .ContainerdVersion }} containerd.io={{ .ContainerdVersion }}* \ docker-ce-cli=5:{{ .DockerVersion }}* \ diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index fa38dc643..16c3a4e07 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -105,7 +105,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y containerd.io=1.4* + apt-get install -y --allow-downgrades containerd.io=1.4* apt-mark hold containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 5d8d62487..fbccfa0d6 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -102,7 +102,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y \ + apt-get install --allow-downgrades -y \ containerd.io=1.4* \ docker-ce-cli=5:19.03* \ docker-ce=5:19.03* diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index d04e63a50..64b02b0bd 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -100,7 +100,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y \ + apt-get install --allow-downgrades -y \ containerd.io=1.4* \ docker-ce-cli=5:19.03* \ docker-ce=5:19.03* diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 9f3b468a1..ba3b58964 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -100,7 +100,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y \ + apt-get install --allow-downgrades -y \ containerd.io=1.4* \ docker-ce-cli=5:19.03* \ docker-ce=5:19.03* diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 74ae95b4f..31a45b767 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -102,7 +102,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y \ + apt-get install --allow-downgrades -y \ containerd.io=1.4* \ docker-ce-cli=5:19.03* \ docker-ce=5:19.03* diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index 70a188de2..a31a8a522 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -100,7 +100,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y \ + apt-get install --allow-downgrades -y \ containerd.io=1.4* \ docker-ce-cli=5:19.03* \ docker-ce=5:19.03* diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 2031720cd..f355ecbcf 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -100,7 +100,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y \ + apt-get install --allow-downgrades -y \ containerd.io=1.4* \ docker-ce-cli=5:19.03* \ docker-ce=5:19.03* diff --git a/pkg/userdata/ubuntu/testdata/version-1.17.16.yaml b/pkg/userdata/ubuntu/testdata/version-1.17.16.yaml index 8439a6c1c..7f8fb5142 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.17.16.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.17.16.yaml @@ -100,7 +100,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y \ + apt-get install --allow-downgrades -y \ containerd.io=1.4* \ docker-ce-cli=5:19.03* \ docker-ce=5:19.03* diff --git a/pkg/userdata/ubuntu/testdata/version-1.18.14.yaml b/pkg/userdata/ubuntu/testdata/version-1.18.14.yaml index e6b960198..82fc6ba3a 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.18.14.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.18.14.yaml @@ -100,7 +100,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y \ + apt-get install --allow-downgrades -y \ containerd.io=1.4* \ docker-ce-cli=5:19.03* \ docker-ce=5:19.03* diff --git a/pkg/userdata/ubuntu/testdata/version-1.19.4.yaml b/pkg/userdata/ubuntu/testdata/version-1.19.4.yaml index f306774c4..0eaef6879 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.19.4.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.19.4.yaml @@ -100,7 +100,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y \ + apt-get install --allow-downgrades -y \ containerd.io=1.4* \ docker-ce-cli=5:19.03* \ docker-ce=5:19.03* diff --git a/pkg/userdata/ubuntu/testdata/version-1.20.1.yaml b/pkg/userdata/ubuntu/testdata/version-1.20.1.yaml index 77131c0c6..ff0d98592 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.20.1.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.20.1.yaml @@ -100,7 +100,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y \ + apt-get install --allow-downgrades -y \ containerd.io=1.4* \ docker-ce-cli=5:19.03* \ docker-ce=5:19.03* diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index e5bed40c4..b66b6707c 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -110,7 +110,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y \ + apt-get install --allow-downgrades -y \ containerd.io=1.4* \ docker-ce-cli=5:19.03* \ docker-ce=5:19.03* diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 8e7327e71..3eef17ae4 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -110,7 +110,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y \ + apt-get install --allow-downgrades -y \ containerd.io=1.4* \ docker-ce-cli=5:19.03* \ docker-ce=5:19.03* diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index bfa48a242..eb8423114 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -101,7 +101,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y \ + apt-get install --allow-downgrades -y \ containerd.io=1.4* \ docker-ce-cli=5:19.03* \ docker-ce=5:19.03* From 2260d02d203f8f79f099abe9a7a14f023971f633 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Fri, 22 Oct 2021 14:51:43 +0200 Subject: [PATCH 004/489] Disable nm-cloud-setup on rhel (#1076) * disable nm-cloud-setup on rhel Signed-off-by: Moath Qasim * update fixtures Signed-off-by: Moath Qasim --- pkg/userdata/rhel/provider.go | 31 +++++++++++++++++++ .../kubelet-containerd-v1.20-aws.yaml | 31 +++++++++++++++++++ .../rhel/testdata/kubelet-v1.17-aws.yaml | 31 +++++++++++++++++++ .../rhel/testdata/kubelet-v1.18-aws.yaml | 31 +++++++++++++++++++ .../rhel/testdata/kubelet-v1.19-aws.yaml | 31 +++++++++++++++++++ .../testdata/kubelet-v1.20-aws-external.yaml | 31 +++++++++++++++++++ .../rhel/testdata/kubelet-v1.20-aws.yaml | 31 +++++++++++++++++++ .../kubelet-v1.20-vsphere-mirrors.yaml | 31 +++++++++++++++++++ .../testdata/kubelet-v1.20-vsphere-proxy.yaml | 31 +++++++++++++++++++ .../rhel/testdata/kubelet-v1.20-vsphere.yaml | 31 +++++++++++++++++++ 10 files changed, 310 insertions(+) diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index c596bb46d..65df90901 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -130,6 +130,8 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { // UserData template. const userDataTemplate = `#cloud-config +bootcmd: +- modprobe ip_tables {{ if ne .CloudProviderName "aws" }} hostname: {{ .MachineSpec.Name }} fqdn: {{ .MachineSpec.Name }} @@ -315,6 +317,34 @@ write_files: append: true {{- end }} +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + rh_subscription: {{- if .OSConfig.RHELUseSatelliteServer }} org: "{{.OSConfig.RHELOrganizationName}}" @@ -329,4 +359,5 @@ rh_subscription: runcmd: - systemctl start setup.service +- systemctl start disable-nm-cloud-setup.service ` diff --git a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml index 9aa932e11..cf548d790 100644 --- a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml @@ -1,4 +1,6 @@ #cloud-config +bootcmd: +- modprobe ip_tables ssh_pwauth: no @@ -413,6 +415,34 @@ write_files: WantedBy=multi-user.target +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + rh_subscription: username: "" password: "" @@ -420,3 +450,4 @@ rh_subscription: runcmd: - systemctl start setup.service +- systemctl start disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml index 017ee840c..a2e08eca3 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml @@ -1,4 +1,6 @@ #cloud-config +bootcmd: +- modprobe ip_tables ssh_pwauth: no @@ -397,6 +399,34 @@ write_files: WantedBy=multi-user.target +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + rh_subscription: username: "" password: "" @@ -404,3 +434,4 @@ rh_subscription: runcmd: - systemctl start setup.service +- systemctl start disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml index c05513d4a..948e83652 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml @@ -1,4 +1,6 @@ #cloud-config +bootcmd: +- modprobe ip_tables ssh_pwauth: no @@ -397,6 +399,34 @@ write_files: WantedBy=multi-user.target +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + rh_subscription: username: "" password: "" @@ -404,3 +434,4 @@ rh_subscription: runcmd: - systemctl start setup.service +- systemctl start disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml index a7bb33afe..5dee91a5f 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml @@ -1,4 +1,6 @@ #cloud-config +bootcmd: +- modprobe ip_tables ssh_pwauth: no @@ -397,6 +399,34 @@ write_files: WantedBy=multi-user.target +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + rh_subscription: username: "" password: "" @@ -404,3 +434,4 @@ rh_subscription: runcmd: - systemctl start setup.service +- systemctl start disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml index 8cf198435..af133ca9a 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml @@ -1,4 +1,6 @@ #cloud-config +bootcmd: +- modprobe ip_tables ssh_pwauth: no @@ -396,6 +398,34 @@ write_files: WantedBy=multi-user.target +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + rh_subscription: username: "" password: "" @@ -403,3 +433,4 @@ rh_subscription: runcmd: - systemctl start setup.service +- systemctl start disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml index 2ed5fa953..b7ae6dc89 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml @@ -1,4 +1,6 @@ #cloud-config +bootcmd: +- modprobe ip_tables ssh_pwauth: no @@ -397,6 +399,34 @@ write_files: WantedBy=multi-user.target +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + rh_subscription: username: "" password: "" @@ -404,3 +434,4 @@ rh_subscription: runcmd: - systemctl start setup.service +- systemctl start disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml index 17a7c5564..81715766a 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml @@ -1,4 +1,6 @@ #cloud-config +bootcmd: +- modprobe ip_tables hostname: node1 fqdn: node1 @@ -415,6 +417,34 @@ write_files: WantedBy=multi-user.target +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + rh_subscription: username: "" password: "" @@ -422,3 +452,4 @@ rh_subscription: runcmd: - systemctl start setup.service +- systemctl start disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml index ff4cdfca4..921995455 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml @@ -1,4 +1,6 @@ #cloud-config +bootcmd: +- modprobe ip_tables hostname: node1 fqdn: node1 @@ -415,6 +417,34 @@ write_files: WantedBy=multi-user.target +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + rh_subscription: username: "" password: "" @@ -422,3 +452,4 @@ rh_subscription: runcmd: - systemctl start setup.service +- systemctl start disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml index 632b8d979..eb4187f9b 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml @@ -1,4 +1,6 @@ #cloud-config +bootcmd: +- modprobe ip_tables hostname: node1 fqdn: node1 @@ -406,6 +408,34 @@ write_files: WantedBy=multi-user.target +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + rh_subscription: username: "" password: "" @@ -413,3 +443,4 @@ rh_subscription: runcmd: - systemctl start setup.service +- systemctl start disable-nm-cloud-setup.service From 117dc9c3bf7801cc07ff29d0b2ff2638cc852fe8 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Fri, 22 Oct 2021 17:03:58 +0200 Subject: [PATCH 005/489] Fix RHEL 8 host-pod connectivity (#1078) * fix pod networking for rhel on azure Signed-off-by: Moath Qasim * update fixtures Signed-off-by: Moath Qasim --- cmd/machine-controller/main.go | 16 +++++++++++++++- pkg/apis/plugin/plugin.go | 2 ++ pkg/controller/machine/machine_controller.go | 8 ++++++++ pkg/userdata/rhel/provider.go | 11 +++++++++-- .../testdata/kubelet-containerd-v1.20-aws.yaml | 2 -- .../rhel/testdata/kubelet-v1.17-aws.yaml | 2 -- .../rhel/testdata/kubelet-v1.18-aws.yaml | 2 -- .../rhel/testdata/kubelet-v1.19-aws.yaml | 2 -- .../testdata/kubelet-v1.20-aws-external.yaml | 2 -- .../rhel/testdata/kubelet-v1.20-aws.yaml | 2 -- .../testdata/kubelet-v1.20-vsphere-mirrors.yaml | 2 -- .../testdata/kubelet-v1.20-vsphere-proxy.yaml | 2 -- .../rhel/testdata/kubelet-v1.20-vsphere.yaml | 2 -- 13 files changed, 34 insertions(+), 21 deletions(-) diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 4137cfa60..f3730a010 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -79,6 +79,8 @@ var ( nodeHyperkubeImage string nodeKubeletRepository string nodeContainerRuntime string + podCidr string + nodePortRange string ) const ( @@ -120,6 +122,12 @@ type controllerRunOptions struct { nodeCSRApprover bool node machinecontroller.NodeSettings + + // Assigns the POD networks that will be allocated. + podCidr string + + // A port range to reserve for services with NodePort visibility + nodePortRange string } func main() { @@ -153,7 +161,9 @@ func main() { flag.StringVar(&nodeKubeletRepository, "node-kubelet-repository", "quay.io/kubermatic/kubelet", "Repository for the kubelet container. Only has effect on Flatcar Linux, and for kubernetes >= 1.18.") flag.StringVar(&nodeContainerRuntime, "node-container-runtime", "docker", "container-runtime to deploy") flag.StringVar(&caBundleFile, "ca-bundle", "", "path to a file containing all PEM-encoded CA certificates (will be used instead of the host's certificates if set)") - flag.BoolVar(&nodeCSRApprover, "node-csr-approver", true, "Enable NodeCSRApprover controller to automatically approve node serving certificate requests.") + flag.BoolVar(&nodeCSRApprover, "node-csr-approver", true, "Enable NodeCSRApprover controller to automatically approve node serving certificate requests") + flag.StringVar(&podCidr, "pod-cidr", "172.25.0.0/16", "The network ranges from which POD networks are allocated") + flag.StringVar(&nodePortRange, "node-port-range", "30000-32767", "A port range to reserve for services with NodePort visibility") flag.Parse() kubeconfig = flag.Lookup("kubeconfig").Value.(flag.Getter).Get().(string) @@ -268,6 +278,8 @@ func main() { containerruntime.WithRegistryMirrors(registryMirrors), ), }, + podCidr: podCidr, + nodePortRange: nodePortRange, } if err := nodeFlags.UpdateNodeSettings(&runOptions.node); err != nil { @@ -401,6 +413,8 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { bs.opt.bootstrapTokenServiceAccountName, bs.opt.skipEvictionAfter, bs.opt.node, + bs.opt.podCidr, + bs.opt.nodePortRange, ); err != nil { return fmt.Errorf("failed to add Machine controller to manager: %v", err) } diff --git a/pkg/apis/plugin/plugin.go b/pkg/apis/plugin/plugin.go index 5559e0c54..d4a25b71d 100644 --- a/pkg/apis/plugin/plugin.go +++ b/pkg/apis/plugin/plugin.go @@ -54,6 +54,8 @@ type UserDataRequest struct { KubeletRepository string KubeletFeatureGates map[string]bool ContainerRuntime containerruntime.Config + PodCIDR string + NodePortRange string } // UserDataResponse contains the responded user data. diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 2186fb5df..f99e513f3 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -111,6 +111,8 @@ type Reconciler struct { nodeSettings NodeSettings redhatSubscriptionManager rhsm.RedHatSubscriptionManager satelliteSubscriptionManager rhsm.SatelliteSubscriptionManager + podCIDR string + nodePortRange string } type NodeSettings struct { @@ -167,6 +169,8 @@ func Add( bootstrapTokenServiceAccountName *types.NamespacedName, skipEvictionAfter time.Duration, nodeSettings NodeSettings, + podCIDR string, + nodePortRange string, ) error { reconciler := &Reconciler{ kubeClient: kubeClient, @@ -182,6 +186,8 @@ func Add( nodeSettings: nodeSettings, redhatSubscriptionManager: rhsm.NewRedHatSubscriptionManager(), satelliteSubscriptionManager: rhsm.NewSatelliteSubscriptionManager(), + podCIDR: podCIDR, + nodePortRange: nodePortRange, } m, err := userdatamanager.New() if err != nil { @@ -723,6 +729,8 @@ func (r *Reconciler) ensureInstanceExistsForMachine( NoProxy: r.nodeSettings.NoProxy, HTTPProxy: r.nodeSettings.HTTPProxy, ContainerRuntime: r.nodeSettings.ContainerRuntime, + PodCIDR: r.podCIDR, + NodePortRange: r.nodePortRange, } userdata, err := userdataPlugin.UserData(req) diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 65df90901..52f1ee66f 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -224,17 +224,24 @@ write_files: open-vm-tools \ {{- end }} ipvsadm - {{ .ContainerRuntimeScript | indent 4 }} - {{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + {{ if eq .CloudProviderName "azure" }} + firewall-cmd --permanent --zone=trusted --add-source={{ .PodCIDR }} + firewall-cmd --permanent --add-port=8472/udp + firewall-cmd --permanent --add-port={{ .NodePortRange }}/tcp + firewall-cmd --permanent --add-port={{ .NodePortRange }}/udp + firewall-cmd --reload + systemctl restart firewalld + {{ end -}} {{ if eq .CloudProviderName "vsphere" }} systemctl enable --now vmtoolsd.service {{ end -}} + systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml index cf548d790..ab0678015 100644 --- a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml @@ -82,7 +82,6 @@ write_files: curl \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true @@ -104,7 +103,6 @@ write_files: systemctl daemon-reload systemctl enable --now containerd - opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml index a2e08eca3..611f21b0b 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml @@ -82,7 +82,6 @@ write_files: curl \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true @@ -105,7 +104,6 @@ write_files: systemctl daemon-reload systemctl enable --now docker - opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml index 948e83652..610b2f705 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml @@ -82,7 +82,6 @@ write_files: curl \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true @@ -105,7 +104,6 @@ write_files: systemctl daemon-reload systemctl enable --now docker - opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml index 5dee91a5f..da2a8f20b 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml @@ -82,7 +82,6 @@ write_files: curl \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true @@ -105,7 +104,6 @@ write_files: systemctl daemon-reload systemctl enable --now docker - opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml index af133ca9a..ffc30f63f 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml @@ -82,7 +82,6 @@ write_files: curl \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true @@ -105,7 +104,6 @@ write_files: systemctl daemon-reload systemctl enable --now docker - opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml index b7ae6dc89..a32bc4b33 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml @@ -82,7 +82,6 @@ write_files: curl \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true @@ -105,7 +104,6 @@ write_files: systemctl daemon-reload systemctl enable --now docker - opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml index 81715766a..1d9e483cc 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml @@ -96,7 +96,6 @@ write_files: open-vm-tools \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true @@ -119,7 +118,6 @@ write_files: systemctl daemon-reload systemctl enable --now docker - opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml index 921995455..04f39a87a 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml @@ -96,7 +96,6 @@ write_files: open-vm-tools \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true @@ -119,7 +118,6 @@ write_files: systemctl daemon-reload systemctl enable --now docker - opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml index eb4187f9b..fc64949e5 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml @@ -88,7 +88,6 @@ write_files: open-vm-tools \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true @@ -111,7 +110,6 @@ write_files: systemctl daemon-reload systemctl enable --now docker - opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin From 680ff9033851d90b5da1adec360c5c80f8113b7f Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Mon, 25 Oct 2021 19:53:46 +0200 Subject: [PATCH 006/489] Machine controller osm integration (#1074) * add --useosm * nodes joins successfully * wip * refactor osm integration Signed-off-by: Moath Qasim * fixing go imports Signed-off-by: Moath Qasim * add api token extraction Signed-off-by: Moath Qasim * refactor osm bootstraping in machine controller Signed-off-by: Moath Qasim * vendoring osm Signed-off-by: Moath Qasim * fixing lint Signed-off-by: Moath Qasim * fix osc secert generating bug Signed-off-by: Moath Qasim * remove useOSM parameter duplication Signed-off-by: Moath Qasim * resolve conflicts Signed-off-by: Moath Qasim * fix linting Signed-off-by: Moath Qasim Co-authored-by: moelsayed --- cmd/machine-controller/main.go | 14 + go.mod | 30 +- go.sum | 682 +++++++++++++++++- hack/run-machine-controller.sh | 1 + pkg/cloudprovider/provider/packet/provider.go | 1 - pkg/cloudprovider/util/cloud_init_settings.go | 22 + pkg/clusterinfo/configmap.go | 4 + pkg/controller/machine/machine_controller.go | 214 +++++- pkg/userdata/ubuntu/provider.go | 1 - 9 files changed, 912 insertions(+), 57 deletions(-) diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index f3730a010..ae4d2bc15 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -43,6 +43,7 @@ import ( machinesv1alpha1 "github.com/kubermatic/machine-controller/pkg/machines/v1alpha1" "github.com/kubermatic/machine-controller/pkg/node" "github.com/kubermatic/machine-controller/pkg/signals" + osmv1alpha1 "k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/types" @@ -71,6 +72,8 @@ var ( nodeCSRApprover bool caBundleFile string + useOSM bool + nodeHTTPProxy string nodeNoProxy string nodeInsecureRegistries string @@ -123,6 +126,8 @@ type controllerRunOptions struct { node machinecontroller.NodeSettings + useOSM bool + // Assigns the POD networks that will be allocated. podCidr string @@ -165,6 +170,8 @@ func main() { flag.StringVar(&podCidr, "pod-cidr", "172.25.0.0/16", "The network ranges from which POD networks are allocated") flag.StringVar(&nodePortRange, "node-port-range", "30000-32767", "A port range to reserve for services with NodePort visibility") + flag.BoolVar(&useOSM, "use-osm", false, "use osm controller for node bootstrap") + flag.Parse() kubeconfig = flag.Lookup("kubeconfig").Value.(flag.Getter).Get().(string) masterURL = flag.Lookup("master").Value.(flag.Getter).Get().(string) @@ -194,6 +201,11 @@ func main() { klog.Fatalf("failed to add clusterv1alpha1 api to scheme: %v", err) } + // needed for OSM + if err := osmv1alpha1.AddToScheme(scheme.Scheme); err != nil { + klog.Fatalf("failed to add osmv1alpha1 api to scheme: %v", err) + } + // Check if the hyperkube image has a tag set hyperkubeImageRef, err := reference.Parse(nodeHyperkubeImage) if err != nil { @@ -278,6 +290,7 @@ func main() { containerruntime.WithRegistryMirrors(registryMirrors), ), }, + useOSM: useOSM, podCidr: podCidr, nodePortRange: nodePortRange, } @@ -413,6 +426,7 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { bs.opt.bootstrapTokenServiceAccountName, bs.opt.skipEvictionAfter, bs.opt.node, + bs.opt.useOSM, bs.opt.podCidr, bs.opt.nodePortRange, ); err != nil { diff --git a/go.mod b/go.mod index 2f0264110..a826e9432 100644 --- a/go.mod +++ b/go.mod @@ -8,34 +8,25 @@ require ( github.com/Azure/azure-sdk-for-go v49.0.0+incompatible github.com/Azure/go-autorest/autorest/azure/auth v0.5.5 github.com/Azure/go-autorest/autorest/to v0.4.0 - github.com/Azure/go-autorest/autorest/validation v0.3.0 // indirect github.com/BurntSushi/toml v0.3.1 github.com/Masterminds/semver v1.5.0 - github.com/Masterminds/semver/v3 v3.1.0 - github.com/Masterminds/sprig/v3 v3.1.0 - github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083 // indirect - github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd // indirect + github.com/Masterminds/semver/v3 v3.1.1 + github.com/Masterminds/sprig/v3 v3.2.2 github.com/aliyun/alibaba-cloud-sdk-go v1.61.751 github.com/anexia-it/go-anxcloud v0.3.8 github.com/aws/aws-sdk-go v1.36.2 github.com/coreos/container-linux-config-transpiler v0.9.0 - github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect - github.com/coreos/ignition v0.35.0 // indirect github.com/davecgh/go-spew v1.1.1 github.com/digitalocean/godo v1.54.0 github.com/docker/distribution v2.7.1+incompatible - github.com/emicklei/go-restful v2.11.2+incompatible // indirect github.com/ghodss/yaml v1.0.0 - github.com/go-openapi/spec v0.19.6 // indirect - github.com/go-openapi/swag v0.19.7 // indirect github.com/go-test/deep v1.0.7 github.com/google/uuid v1.1.2 github.com/gophercloud/gophercloud v0.14.0 github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 github.com/hetznercloud/hcloud-go v1.25.0 github.com/linode/linodego v0.24.0 - github.com/mailru/easyjson v0.7.1 // indirect - github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888 + github.com/packethost/packngo v0.5.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 @@ -44,20 +35,18 @@ require ( github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7 github.com/sethvargo/go-password v0.2.0 github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda - github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb // indirect github.com/vmware/govmomi v0.23.1 - go4.org v0.0.0-20200104003542-c7e774b10ea0 // indirect - golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 + golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58 gomodules.xyz/jsonpatch/v2 v2.1.0 google.golang.org/api v0.36.0 google.golang.org/grpc v1.33.2 - gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 - k8s.io/api v0.19.4 + k8c.io/operating-system-manager v0.1.0 + k8s.io/api v0.20.2 k8s.io/apiextensions-apiserver v0.19.4 - k8s.io/apimachinery v0.19.4 + k8s.io/apimachinery v0.20.2 k8s.io/client-go v12.0.0+incompatible k8s.io/klog v1.0.0 k8s.io/kubelet v0.19.4 @@ -68,4 +57,7 @@ require ( sigs.k8s.io/yaml v1.2.0 ) -replace k8s.io/client-go => k8s.io/client-go v0.19.4 +replace ( + github.com/packethost/packngo => github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888 + k8s.io/client-go => k8s.io/client-go v0.20.2 +) diff --git a/go.sum b/go.sum index 02c6f87c2..4a7c704b8 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,15 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.30.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.47.0/go.mod h1:5p3Ky/7f3N10VBkhuR5LFtddroTiMyjZV/Kj5qOQFxU= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= @@ -27,6 +30,7 @@ cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM7 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/logging v1.0.0/go.mod h1:V1cc3ogwobYzQq5f2R7DS/GvRIrI4FKj01Gs5glwAls= cloud.google.com/go/logging v1.1.2 h1:KNALX0NZn8UJhqKnqoHxhMqyoZfBZoh5wF7CQJZ5XrU= cloud.google.com/go/logging v1.1.2/go.mod h1:KrljuAHIw631j9+QXsnq9vDwsrwmdxfGpivMR68M7DY= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -39,18 +43,40 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +code.cloudfoundry.org/go-pubsub v0.0.0-20180503211407-becd51dc37cb/go.mod h1:2mohpzdn59JWHT85lXjjglNpGLF51tk6hHqfxpc0utk= +contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= +contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= +contrib.go.opencensus.io/exporter/stackdriver v0.12.8/go.mod h1:XyyafDnFOsqoxHJgTFycKZMrRUrPThLh2iYTJF6uoO0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= +github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= +github.com/Azure/azure-sdk-for-go v28.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v35.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v38.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v46.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v49.0.0+incompatible h1:rvYYNgKNBwoxUaBFmd/+TpW3qrd805EHBBvUp5FmFso= github.com/Azure/azure-sdk-for-go v49.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.1.0/go.mod h1:AKyIcETwSUFxIcs/Wnq/C+kwCtlEYGUVd7FPNb2slmg= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= +github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= +github.com/Azure/go-autorest/autorest v0.9.5/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest v0.11.6/go.mod h1:V6p3pKZx1KKkJubbxnDWrzNhEIfOy/pTGasLqzHIPHs= github.com/Azure/go-autorest/autorest v0.11.13 h1:XKx/sB3bfadpXBBHPc7tP2XPKhzVyrdhxpDC3T0wqjs= github.com/Azure/go-autorest/autorest v0.11.13/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest/adal v0.1.0/go.mod h1:MeS4XhScH55IST095THyTxElntu7WqB7pNbZo8Q5G3E= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= +github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.4/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.8 h1:bW6ZdxqMYWsxGikpM62SSE3jnvOXVu9SXzJTuj1WM3Y= github.com/Azure/go-autorest/autorest/adal v0.9.8/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= @@ -65,35 +91,54 @@ github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSY github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/to v0.1.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc= +github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc= +github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= +github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI= github.com/Azure/go-autorest/autorest/validation v0.3.0 h1:3I9AAI63HfcLtphd9g39ruUwRI+Ca+z/f36KHPFRUss= github.com/Azure/go-autorest/autorest/validation v0.3.0/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.1.0/go.mod h1:ROEEAFwXycQw7Sn3DXNtEedEvdeRAgDr0izn4z5Ij88= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher v0.0.0-20191203181535-308b93ad1f39/go.mod h1:yfGmCjKuUzk9WzubMlW2zwjhCraIc/J+M40cufdemRM= +github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14= +github.com/GoogleCloudPlatform/testgrid v0.0.1-alpha.4/go.mod h1:f96W2HYy3tiBNV5zbbRc+NczwYHgG1PHXMQfoEWv680= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/Masterminds/goutils v1.1.0 h1:zukEsf/1JZwCMgHiK3GZftabmxiCw4apj3a28RPBiVg= +github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= +github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Masterminds/semver/v3 v3.1.0 h1:Y2lUDsFKVRSYGojLJ1yLxSXdMmMYTYls0rCvoqmMUQk= github.com/Masterminds/semver/v3 v3.1.0/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Masterminds/sprig/v3 v3.1.0 h1:j7GpgZ7PdFqNsmncycTHsLmVPf5/3wJtlgW9TNDYD9Y= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= +github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/sprig/v3 v3.1.0/go.mod h1:ONGMf7UfYGAbMXCZmQLy8x3lCDIPrEZE/rU8pmrbihA= +github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= +github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OneOfOne/xxhash v1.2.7/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= @@ -102,8 +147,11 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/Venafi/vcert/v4 v4.11.0/go.mod h1:OE+UZ0cj8qqVUuk0u7R4GIk4ZB6JMSf/WySqnBPNwws= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/a8m/expect v1.0.0/go.mod h1:4IwSCMumY49ScypDnjNbYEjgVeqy1/U2cEs3Lat96eA= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083 h1:uwcvnXW76Y0rHM+qs7y8iHknWUWXYFNlD6FEVhc47TU= @@ -120,12 +168,14 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/aliyun/alibaba-cloud-sdk-go v1.61.751 h1:PX0jCn9kBBgaybsFltpmQ8F7O74hQXY/3yNyCjInDag= github.com/aliyun/alibaba-cloud-sdk-go v1.61.751/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/andygrunwald/go-gerrit v0.0.0-20190120104749-174420ebee6c/go.mod h1:0iuRQp6WJ44ts+iihy5E/WlPqfg5RNeQxOmzRkxCdtk= github.com/anexia-it/go-anxcloud v0.3.8 h1:+ZOVqUHwINTm9Q68GPVh+Q/c794Fe+2GahIVagNLjDg= github.com/anexia-it/go-anxcloud v0.3.8/go.mod h1:cevqezsbOJ4GBlAWaztfLKl9w4VzxJBt4ipgHORi3gw= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apoydence/onpar v0.0.0-20200406201722-06f95a1c68e8/go.mod h1:maauOJD0kdDqIz4xmkunipFVbBoTM6pFSy0kkWBcIUY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -134,11 +184,23 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-k8s-tester v0.0.0-20190114231546-b411acf57dfe/go.mod h1:1ADF5tAtU1/mVtfMcHAYSm2fPw71DA7fFk0yed64/0I= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.16.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.27.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.27.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/aws/aws-sdk-go v1.34.30/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.36.2 h1:UAeFPct+jHqWM+tgiqDrC9/sfbWj6wkcvpsJ+zdcsvA= github.com/aws/aws-sdk-go v1.36.2/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/bazelbuild/buildtools v0.0.0-20190917191645-69366ca98f89/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -149,22 +211,31 @@ github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d/go.mod h1:IyUJYN1gvWjtLF5ZuygmxbnsAyP3aJS6cHzIuZY50B0= github.com/briandowns/spinner v1.8.0/go.mod h1://Zf9tMcxfRUA36V23M6YGEAv+kECGfvpnLTnb8n4XQ= +github.com/bwmarrin/snowflake v0.0.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clarketm/json v1.13.4/go.mod h1:ynr2LRfb0fQU34l07csRNBTcivjySLLiY1YzQqKVfdo= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudevents/sdk-go v0.0.0-20190509003705-56931988abe3/go.mod h1:j1nZWMLGg3om8SswStBoY6/SHvcLM19MuZqwDtMtmzs= +github.com/cloudflare/cloudflare-go v0.13.2/go.mod h1:27kfc1apuifUmJhp069y0+hwlKDg4bd8LWlu7oKeZvM= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= @@ -178,7 +249,10 @@ github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.17+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-oidc v0.0.0-20180117170138-065b426bd416/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.0.0-20180108230905-e214231b295a/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -186,44 +260,57 @@ github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/ignition v0.35.0 h1:UFodoYq1mOPrbEjtxIsZbThcDyQwAI1owczRDqWmKkQ= github.com/coreos/ignition v0.35.0/go.mod h1:WJQapxzEn9DE0ryxsGvm8QnBajm/XsS/PkrDqSpz+bA= +github.com/coreos/locksmith v0.6.2/go.mod h1:mSLRr7SVSEAIugjic7+TXif/+ZQQq0zCks1vptuj2fs= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/prometheus-operator v0.35.0/go.mod h1:XHYZUStZWcwd1yk/1DjZv/fywqKIyAJ6pSwvIr+v9BQ= +github.com/cpu/goacmedns v0.0.3/go.mod h1:4MipLkI+qScwqtVxcNO6okBhbgRrr7/tKXUSgSL0teQ= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cristim/ec2-instances-info v0.0.0-20201110114654-2dfcc09f67d4/go.mod h1:0yCjO4zBzlwWSGh/zGfW2Zq1NX605qCYVBHD1fPXKNs= github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE= +github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= +github.com/denisenkom/go-mssqldb v0.0.0-20190111225525-2fea367d496d/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/digitalocean/godo v1.44.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= github.com/digitalocean/godo v1.54.0 h1:KP0Nv87pgViR8k/7De3VrmflCL5pJqXbNnkcw0bwG10= github.com/digitalocean/godo v1.54.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8= github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= @@ -239,14 +326,23 @@ github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4s github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= +github.com/etcd-io/gofail v0.0.0-20190801230047-ad7f989257ca/go.mod h1:49H/RkXP8pKaZy4h0d+NW16rSLhyVBt4o6VLJbmOqDE= +github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.1.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -256,7 +352,9 @@ github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsouza/fake-gcs-server v0.0.0-20180612165233-e85be23bdaa8/go.mod h1:1/HufuJ+eaDf4KTnYdS6HJMGvMRU8d4cYTuu/1QaBbI= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -268,70 +366,118 @@ github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.62.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.2.1-0.20200730175230-ee2de8da5be6/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.3.0 h1:q4c+kbcR0d5rSurhBR8dIgieOaYpXtsdTYfx22Cu6rs= github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/zapr v0.2.0 h1:v6Ji8yBW77pva6NkJKQdHLAJKrIJKRHz0RXwPqCHSR4= +github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= +github.com/go-logr/zapr v0.1.1/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= +github.com/go-logr/zapr v0.3.0 h1:iyiCRZ29uPmbO7mWIjOEiYMXrTxZWTyK4tCatLyGpUY= +github.com/go-logr/zapr v0.3.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.17.2/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= +github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ= +github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk= github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.17.2/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.19.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= -github.com/go-openapi/jsonpointer v0.19.3 h1:gihV7YNZK1iK6Tgwwsxo2rJbD1GTbdm72325Bq8FI3w= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= -github.com/go-openapi/jsonreference v0.19.3 h1:5cxNfTy0UVC3X8JL5ymxzyoUZmo8iZb+jeTWn7tUa8o= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.4/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.17.2/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= +github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5WPaZvcvSI= github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= +github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY= +github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= +github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= +github.com/go-openapi/runtime v0.17.2/go.mod h1:QO936ZXeisByFmZEO1IS1Dqhtf4QV1sYYFtIq6Ld86Q= github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= +github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo= +github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98= +github.com/go-openapi/runtime v0.19.20/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= +github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.17.2/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.19.6 h1:rMMMj8cV38KVXK7SFc+I2MWClbEfbK705+j+dyqun5g= +github.com/go-openapi/spec v0.19.4/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/spec v0.19.15 h1:uxh8miNJEfMm8l8ekpY7i39LcORm1xSRtoipEGl1JPk= +github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= +github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.4/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.7 h1:VRuXN2EnMSsZdauzdss6JBC29YotDqG59BZ+tdlIL1s= github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= +github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= +github.com/go-openapi/swag v0.19.12 h1:Bc0bnY2c3AoF7Gc+IMIAQQsD8fLHjHpc19wXvYuayQI= +github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M= +github.com/go-openapi/validate v0.17.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= +github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= +github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8= +github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4= +github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= @@ -339,22 +485,55 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 h1:JVrqSeQfdhYRFk24TvhTZWU0q8lfCojxZQFi3Ou7+uY= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= +github.com/go-sql-driver/mysql v0.0.0-20160411075031-7ebe0a500653/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-swagger/go-swagger v0.25.0/go.mod h1:9639ioXrPX9E6BbnbaDklGXjNz7upAXoNBwL4Ok11Vk= +github.com/go-swagger/scan-repo-boundary v0.0.0-20180623220736-973b3573c013/go.mod h1:b65mBPzqzZWxOZGxSWrqs4GInLIn+u99Q9q7p+GKni0= +github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= +github.com/gobuffalo/envy v1.6.5/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/envy v1.7.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= github.com/gobuffalo/flect v0.1.5/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= +github.com/gobuffalo/flect v0.2.0/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kETFqjFoFlOc= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godror/godror v0.13.3/go.mod h1:2ouUT4kdhUBk7TAkHWD4SN0CdI0pgEQbo8FVHhbSKWg= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -369,11 +548,13 @@ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2V github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -382,6 +563,8 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v0.0.0-20181025225059-d3de96c4c28e/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= +github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -398,6 +581,12 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= +github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= +github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= +github.com/gomodule/redigo v1.7.0/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -410,13 +599,18 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-containerregistry v0.0.0-20200115214256-379933c9c22b/go.mod h1:Wtl/v6YdQxv397EREtzwgd9+Ud7Q5D8XMbi3Zazgkrs= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-licenses v0.0.0-20191112164736-212ea350c932/go.mod h1:16wa6pRqNDUIhOtwF0GcROVqMeXHZJ7H6eGDFUh5Pfk= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/licenseclassifier v0.0.0-20190926221455-842c0d70d702/go.mod h1:qsqn2hxC+vURpyBRygGUuinTO42MFRLcsmQ/P8v94+M= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -424,6 +618,7 @@ github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/ github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -434,50 +629,78 @@ github.com/google/pprof v0.0.0-20201117184057-ae444373da19/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v0.0.0-20170306145142-6a5e28554805/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= +github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.2.2/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1 h1:A8Yhf6EtqTv9RMsU6MQTyrtV1TjWlR6xU9BsZIwuTCM= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gophercloud/gophercloud v0.14.0 h1:c2Byo+YMxhHlTJ3TPptjQ4dOQ1YknTHDJ/9zClDH+84= github.com/gophercloud/gophercloud v0.14.0/go.mod h1:VX0Ibx85B60B5XOrZr6kaNwrmPUzcmMpwxvQ1WQIIWM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/csrf v1.6.2/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= +github.com/gorilla/mux v0.0.0-20181024020800-521ea7b17d02/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170330212424-2500245aa611/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.15.2 h1:HC+hWRWf+v5zTMPyoaYTKIJih+4sd4XRWmj0qlG87Co= github.com/grpc-ecosystem/grpc-gateway v1.15.2/go.mod h1:vO11I9oWA+KsxmfFQPhLnnIb1VDE24M+pdxZFiuZcA8= +github.com/h2non/gock v1.0.9/go.mod h1:CZMcB0Lg5IWnr9bF79pPMg9WeV6WumxQiUJ1UvdO1iE= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= +github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= +github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -486,6 +709,7 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= @@ -493,36 +717,60 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= +github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 h1:GT4RsKmHh1uZyhmTkWJTDALRjSHYQp6FRKrotf0zhAs= github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= +github.com/hetznercloud/hcloud-go v1.23.1/go.mod h1:xng8lbDUg+xM1dgc0yGHX5EeqbwIq7UYlMWMTx3SQVg= github.com/hetznercloud/hcloud-go v1.25.0 h1:QAaFKtGKWRxjwjKJWBGMxGYUxVEQmIkb35j/WXrsazY= github.com/hetznercloud/hcloud-go v1.25.0/go.mod h1:2C5uMtBiMoFr3m7lBFPf7wXTdh33CevmZpQIIDPGYJI= +github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= +github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/iancoleman/strcase v0.1.2/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.0.0-20171009183408-7fe0c75c13ab/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.10 h1:6q5mVkdH/vYmqngx7kZQTjJ5HRsx+ImorDIEQ+beJgc= +github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/improbable-eng/thanos v0.3.2/go.mod h1:GZewVGILKuJVPNRn7L4Zw+7X96qzFOwj63b22xYGXBE= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb v0.0.0-20161215172503-049f9b42e9a5/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= +github.com/jenkins-x/go-scm v1.5.65/go.mod h1:MgGRkJScE/rJ30J/bXYqduN5sDPZqZFITJopsnZmTOw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jetstack/cert-manager v1.1.0/go.mod h1:GULIHTGjSc2LjlgBCLhQ8u5WmQ95hk9FAiQbhjMthMk= +github.com/jinzhu/gorm v0.0.0-20170316141641-572d0a0ab1eb/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= +github.com/jinzhu/inflection v0.0.0-20190603042836-f5c5f50e6090/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.0.0-20141017032234-72f9bd7c4e0c/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -539,31 +787,47 @@ github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfV github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20191119172530-79f836b90111/go.mod h1:MP2HbArq3QT+oVp8pmtHNZnSnkhdkHtDnc7h6nJXmBU= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/knative/build v0.1.2/go.mod h1:/sU74ZQkwlYA5FwYDJhYTy61i/Kn+5eWfln2jDbw3Qo= +github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubermatic/machine-controller v1.23.1/go.mod h1:mXWbT7SjqpgFhzCFT3yMEHKdIlT+KkGy4KQCkNRM9Fc= +github.com/kubermatic/machine-controller v1.26.0/go.mod h1:dcJ+GdDSCxCwM0poxwOK8hVO7epiOORDmNMmb2veyw4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.1-0.20191011153232-f91d3411e481/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linode/linodego v0.24.0 h1:o6hNS0T7jeikOfUHoJhUhA/e2QTCsw9MGccVmRHRLE4= github.com/linode/linodego v0.24.0/go.mod h1:GSBKPpjoQfxEfryoCRcgkuUOCuVtGHWhzI8OMdycNTE= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -573,8 +837,13 @@ github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/markbates/inflect v1.0.4/go.mod h1:1fR9+pO2KHEO9ZRtto13gDwwZaAKstQzferVeWqbgNs= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= @@ -582,30 +851,46 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.6/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-oci8 v0.0.7/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= +github.com/mattn/go-runewidth v0.0.0-20181025052659-b20a3daf6a39/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.5/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-sqlite3 v0.0.0-20160514122348-38ee283dabf1/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= +github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/ioprogress v0.0.0-20180201004757-6a23b12fa88e/go.mod h1:waEya8ee1Ro/lgxpVhkJI4BVASzkm3UZqkx/cFJiYHM= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= +github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= +github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -614,48 +899,74 @@ github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lN github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/munnerz/crd-schema-fuzz v1.0.0/go.mod h1:4z/rcm37JxUkSsExFcLL6ZIT1SgDRdLiu7qq1evdVS0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= +github.com/nats-io/gnatsd v1.4.1/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ= +github.com/nats-io/go-nats v1.7.0/go.mod h1:+t7RHT5ApZebkrQdnn6AhQJmhJJiKAvJUio1PiiCtj0= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.0/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= +github.com/nelsam/hel v0.0.0-20200611165952-2d829bae0c66/go.mod h1:Rl/hm4V2s75ScsPmI9cNz87HLNg5MoFAMJwA90fzbkw= +github.com/nelsam/hel/v2 v2.3.2/go.mod h1:1ZTGfU2PFTOd5mx22i5O0Lc2GY933lQ2wb/ggy+rL3w= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= +github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.1 h1:jMU0WaQrP0a/YAEq8eJmJKjBoMs+pClEr1vDMlM/Do4= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= +github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= +github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/open-policy-agent/cert-controller v0.0.0-20200921224206-24b87bbc4b6e/go.mod h1:/y33mmiq3Cc0N+6cickevrLI/iBbWcUwcEVjSKHA0z0= +github.com/open-policy-agent/frameworks/constraint v0.0.0-20200929072634-d96896eff389/go.mod h1:Dr3QxvH+NTQcPPZWSt1ueNOsxW4VwgUltaLL7Ttnrac= +github.com/open-policy-agent/frameworks/constraint v0.0.0-20201118071520-0d37681951a4/go.mod h1:vvhkBONv7Uah2fvS/bQ/N1u0rSLvxZOs2ErR6m+4QtQ= +github.com/open-policy-agent/gatekeeper v0.0.0-20201111000257-4450f08fa95e/go.mod h1:IseSnWz7MX7IhEpZ4CLhA3NrMazc+T6a5rtSq9pOEc4= +github.com/open-policy-agent/opa v0.19.1/go.mod h1:rrwxoT/b011T0cyj+gg2VvxqTtn6N3gp/jzmr3fjW44= +github.com/open-policy-agent/opa v0.21.0/go.mod h1:cZaTfhxsj7QdIiUI0U9aBtOLLTqVNe+XE60+9kZKLHw= +github.com/open-policy-agent/opa v0.24.0/go.mod h1:qEyD/i8j+RQettHGp4f86yjrjvv+ZYia+JHCMv2G7wA= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= @@ -670,35 +981,57 @@ github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKw github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go v0.2.0/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888 h1:ARzSqjQJcSR9IXGMEkXN1IQEZChwd2MqXWm6YNGvO5o= github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888/go.mod h1:RQHg5xR1F614BwJyepfMqrKN+32IH0i7yX+ey43rEeQ= github.com/packethost/pkg v0.0.0-20200903155310-0433e0605550/go.mod h1:GSv7cTtIjns4yc0pyajaM1RE/KE4djJONoblFIRDrxA= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pavel-v-chernykh/keystore-go v2.1.0+incompatible/go.mod h1:xlUlxe/2ItGlQyMTstqeDv9r3U4obH7xYd26TbDQutY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= +github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.0.0-20181023235946-059132a15dd0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/poy/onpar v0.0.0-20200406201722-06f95a1c68e8/go.mod h1:nSbFQvMj97ZyhFRSJYtut+msi4sOY6zJDGCdSc+/rZU= +github.com/poy/onpar v1.0.1/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/prometheus/client_golang v0.0.0-20181025174421-f30f42803563/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= @@ -709,6 +1042,7 @@ github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeD github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.8.0 h1:zvJNkoCFAnYFNC24FV8nW4JdRJ3GIFcLbg65lL/JDcw= github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0/7TSWwj+ITvv0TnM= +github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -716,6 +1050,9 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -726,13 +1063,17 @@ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt2 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.14.0 h1:RHRyE8UocrbjU+6UvRzwi6HjiDfxrrBU91TtbKzkGp4= github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -740,33 +1081,55 @@ github.com/prometheus/prometheus v2.3.2+incompatible/go.mod h1:oAIUtOny2rjMX0OWN github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.8.0/go.mod h1:fSI0j+IUQrDd7+ZtR9WKIGtoYAYAJUKcKhYLG25tN4g= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20190706150252-9beb055b7962/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= +github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg= +github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7 h1:Do8ksLD4Nr3pA0x0hnLOLftZgkiTDvwPDShRTUxtXpE= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8= +github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI= github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shurcooL/githubv4 v0.0.0-20180925043049-51d7b505e2e9/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= +github.com/shurcooL/githubv4 v0.0.0-20190718010115-4ba037080260/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= +github.com/shurcooL/githubv4 v0.0.0-20191102174205-af46314aec7b/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= +github.com/shurcooL/graphql v0.0.0-20180924043259-e4a3a37e6d42/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= +github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -776,16 +1139,24 @@ github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.3.2/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.0-20180319062004-c439c4fa0937/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.0-20181021141114-fe5e611709b0/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.0.1-0.20200713175500-884edc58ad08/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v0.0.0-20181024212040-082b515c9490/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -810,45 +1181,71 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tektoncd/pipeline v0.10.1/go.mod h1:D2X0exT46zYx95BU7ByM8+erpjoN7thmUBvlKThOszU= +github.com/tektoncd/plumbing v0.0.0-20191216083742-847dcf196de9/go.mod h1:QZHgU07PRBTRF6N57w4+ApRu8OgfYLFNqCDlfEZaD9Y= +github.com/tektoncd/plumbing/pipelinerun-logs v0.0.0-20191206114338-712d544c2c21/go.mod h1:S62EUWtqmejjJgUMOGB1CCCHRp6C706laH06BoALkzU= github.com/testcontainers/testcontainers-go v0.9.0/go.mod h1:b22BFXhRbg4PJmeMVWh6ftqjyZHgiIl3w274e9r3C2E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda h1:uAHwUH+06gowZMVLqQXm7jN1y3Sl+CDJHThNiKyLHus= github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda/go.mod h1:s4k7CORR0OMWd4cYwBqNBFPSJZhnSQxeKdDtMa/aspk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJMir8RTqb4ayM= +github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/urfave/cli v1.18.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/vdemeester/k8s-pkg-credentialprovider v0.0.0-20200107171650-7c61ffa44238/go.mod h1:JwQJCMWpUDqjZrB5jpw0f5VbN7U95zxFy1ZDpoEarGo= +github.com/vdemeester/k8s-pkg-credentialprovider v1.13.12-1/go.mod h1:Fko0rTxEtDW2kju5Ky7yFJNS3IcNvW8IPsp4/e9oev0= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb h1:lyL3z7vYwTWXf4/bI+A01+cCSnfhKIBhy+SQ46Z/ml8= github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= +github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/vmware/govmomi v0.23.1 h1:vU09hxnNR/I7e+4zCJvW+5vHu5dO64Aoe2Lw7Yi/KRg= github.com/vmware/govmomi v0.23.1/go.mod h1:Y+Wq4lst78L85Ge/F8+ORXIWiKYqaro1vhAulACy9Lc= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yashtewari/glob-intersection v0.0.0-20180916065949-5c77d914dd0b/go.mod h1:HptNXiXVDcJjXe9SqMd0v2FsL9f8dz4GnXgltU6q/co= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +go.etcd.io/bbolt v1.3.1-etcd.7/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/etcd v0.0.0-20181031231232-83304cfc808c/go.mod h1:weASp41xM3dk0YHg1s/W8ecdGP5G4teSTMBPpYAaUgA= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= +go.etcd.io/etcd/v3 v3.3.0-rc.0.0.20200728214110-6c81b20ec8de/go.mod h1:UENlOa05tkNvLx9VnNziSerG4Ro74upGK6Apd4v6M/Y= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.mongodb.org/mongo-driver v1.3.5/go.mod h1:Ual6Gkco7ZGQw8wE1t4tLnvBsf6yVSM60qW6TgOeJ5c= +go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= +go.opencensus.io v0.17.0/go.mod h1:mp1VrMQxhlqqDpKvH4UcQUa4YwlzNmymAjPrDdfxNpI= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -872,14 +1269,19 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go4.org v0.0.0-20200104003542-c7e774b10ea0 h1:M6XsnQeLwG+rHQ+/rrGh3puBI3WZEy9TBWmf2H+enQA= go4.org v0.0.0-20200104003542-c7e774b10ea0/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -887,27 +1289,37 @@ golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 h1:xYJJ3S178yv++9zXV/hnr29plCAGO9vAFG9dorqaFQc= +golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -916,6 +1328,8 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181023182221-1baf3a9d7d67/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -929,6 +1343,7 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mobile v0.0.0-20190806162312-597adff16ade/go.mod h1:AlhUtkH4DA4asiFC5RgK7ZKmauvtkAVcy9L0epCzlWo= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -938,6 +1353,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -963,8 +1379,12 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190930134127-c5a3c61f89f3/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191119073136-fc4aabc6c914/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -977,15 +1397,20 @@ golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201026091529-146b70c837a4/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -998,6 +1423,7 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1005,6 +1431,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1015,32 +1442,46 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190219203350-90b0e4468f99/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1065,13 +1506,19 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= @@ -1080,8 +1527,9 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180810170437-e96c4e24768d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1096,33 +1544,45 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.0.0-20190807223507-b346f7fd45de/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191010171213-8abd42400456/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112005509-a3f652f18032/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200115165105-de0b1760071a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1133,16 +1593,21 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200313205530-4303120df7d8/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200616195046-dc31b401abb5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201017001424-6003fad69a88/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201030143252-cf7a54d06671/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201105220310-78b158585360/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1155,16 +1620,21 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.1.0 h1:Phva6wqu+xR//Njw6iorylFFgn/z547tw5Ne3HZPQ+k= gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.0.0-20181021000519-a2651947f503/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.6.1-0.20190607001116-5213b8090861/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.10.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= @@ -1187,20 +1657,28 @@ google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20170731182057-09f6ed296fc6/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180608181217-32ee49c4dd80/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181016170114-94acd270e44e/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190708153700-3bdd9d9f5532/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -1231,8 +1709,12 @@ google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201203001206-6486ece9c497 h1:jDYzwXmX9tLnuG4sL85HPmE1ruErXOopALp2i/0AHnI= google.golang.org/genproto v0.0.0-20201203001206-6486ece9c497/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.13.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1241,6 +1723,7 @@ google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -1267,8 +1750,11 @@ google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= @@ -1276,26 +1762,44 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= +gopkg.in/ini.v1 v1.46.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.52.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0-20150622162204-20b71e5b60d7/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5/go.mod h1:hiOFpYm0ZJbusNj2ywpbrXowU3G8U6GIQzqn2mw1UIE= +gopkg.in/square/go-jose.v2 v2.0.0-20180411045311-89060dee6a84/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.1.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1303,10 +1807,13 @@ gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20190709130402-674ba3eaed22/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20190905181640-827449938966/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v0.0.0-20181223230014-1083505acf35/go.mod h1:R//lfYlUuTOTfblYI3lGoAAAebUdzjvbmQsuB7Ykd90= @@ -1317,75 +1824,170 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.2/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8c.io/kubermatic/v2 v2.16.2/go.mod h1:NdW+2mq4ynRtfZs9yPnvcnFWQpzmM7ngntW6GeuQicU= +k8c.io/operating-system-manager v0.1.0 h1:2/vmpWHOLm1j3YZ0qrlaW+ucydXXS83FF7pISDoWlKs= +k8c.io/operating-system-manager v0.1.0/go.mod h1:ULyZQO1irKjsQTNjIdrHld7SZ+joHjmPnOEs5Db8G8M= +k8s.io/api v0.0.0-20181018013834-843ad2d9b9ae/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20181115043458-b799cb063522/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20190725062911-6607c48751ae/go.mod h1:1O0xzX/RAtnm7l+5VEUxZ1ysO2ghatfq/OZED4zM9kA= k8s.io/api v0.0.0-20190918155943-95b840bb6a1f/go.mod h1:uWuOHnjmNrtQomJrvEBg0c0HRNyQ+8KTEERVsK0PW48= +k8s.io/api v0.0.0-20190918195907-bd6ac527cfd2/go.mod h1:AOxZTnaXR/xiarlQL0JUfwQPxjmKDvVYoRp58cA7lUo= k8s.io/api v0.16.4/go.mod h1:AtzMnsR45tccQss5q8RnF+W8L81DH6XwXwo/joEx9u0= +k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= +k8s.io/api v0.17.2/go.mod h1:BS9fjjLc4CMuqfSO8vgbHPKMt5+SF0ET6u/RVDihTo4= +k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= +k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= +k8s.io/api v0.18.4/go.mod h1:lOIQAKYgai1+vz9J7YcDZwC26Z0zQewYOGWdyIPUUQ4= +k8s.io/api v0.18.5/go.mod h1:tN+e/2nbdGKOAH55NMV8oGrMG+3uRlA9GaRfvnCCSNk= +k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= +k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw= k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= -k8s.io/api v0.19.4 h1:I+1I4cgJYuCDgiLNjKx7SLmIbwgj9w7N7Zr5vSIdwpo= k8s.io/api v0.19.4/go.mod h1:SbtJ2aHCItirzdJ36YslycFNzWADYH3tgOhvBEFtZAk= +k8s.io/api v0.20.2 h1:y/HR22XDZY3pniu9hIFDLpUCPq2w5eQ6aV/VFQ7uJMw= +k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY= +k8s.io/apiextensions-apiserver v0.0.0-20190918201827-3de75813f604/go.mod h1:7H8sjDlWQu89yWB3FhZfsLyRCRLuoXoCoY5qtwW1q6I= k8s.io/apiextensions-apiserver v0.16.4/go.mod h1:HYQwjujEkXmQNhap2C9YDdIVOSskGZ3et0Mvjcyjbto= +k8s.io/apiextensions-apiserver v0.17.2/go.mod h1:4KdMpjkEjjDI2pPfBA15OscyNldHWdBCfsWMDWAmSTs= +k8s.io/apiextensions-apiserver v0.18.0/go.mod h1:18Cwn1Xws4xnWQNC00FLq1E350b9lUF+aOdIWDOZxgo= +k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY= +k8s.io/apiextensions-apiserver v0.18.4/go.mod h1:NYeyeYq4SIpFlPxSAB6jHPIdvu3hL0pc36wuRChybio= +k8s.io/apiextensions-apiserver v0.18.6/go.mod h1:lv89S7fUysXjLZO7ke783xOwVTm6lKizADfvUM/SS/M= +k8s.io/apiextensions-apiserver v0.19.0/go.mod h1:znfQxNpjqz/ZehvbfMg5N6fvBJW5Lqu5HVLTJQdP4Fs= k8s.io/apiextensions-apiserver v0.19.2/go.mod h1:EYNjpqIAvNZe+svXVx9j4uBaVhTB4C94HkY3w058qcg= k8s.io/apiextensions-apiserver v0.19.4 h1:D9ak9T012tb3vcGFWYmbQuj9SCC8YM4zhA4XZqsAQC4= k8s.io/apiextensions-apiserver v0.19.4/go.mod h1:B9rpH/nu4JBCtuUp3zTTk8DEjZUupZTBEec7/2zNRYw= +k8s.io/apimachinery v0.0.0-20181015213631-60666be32c5d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/apimachinery v0.0.0-20181110190943-2a7c93004028/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/apimachinery v0.0.0-20190703205208-4cfb76a8bf76/go.mod h1:M2fZgZL9DbLfeJaPBCDqSqNsdsmLN+V29knYJnIXlMA= k8s.io/apimachinery v0.0.0-20190719140911-bfcf53abc9f8/go.mod h1:sBJWIJZfxLhp7mRsRyuAE/NfKTr3kXGR1iaqg8O0gJo= +k8s.io/apimachinery v0.0.0-20190816221834-a9f1d8a9c101/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/apimachinery v0.0.0-20190817020851-f2f3a405f61d/go.mod h1:3jediapYqJ2w1BFw7lAZPCx7scubsTfosqHkhXCWJKw= k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4= k8s.io/apimachinery v0.16.4/go.mod h1:llRdnznGEAqC3DcNm6yEj472xaFVfLM7hnYofMb12tQ= +k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/apimachinery v0.17.1/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/apimachinery v0.17.2/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= +k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= +k8s.io/apimachinery v0.18.4/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= +k8s.io/apimachinery v0.18.5/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= +k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= +k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= -k8s.io/apimachinery v0.19.4 h1:+ZoddM7nbzrDCp0T3SWnyxqf8cbWPT2fkZImoyvHUG0= k8s.io/apimachinery v0.19.4/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= +k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg= +k8s.io/apiserver v0.0.0-20190918200908-1e17798da8c1/go.mod h1:4FuDU+iKPjdsdQSN3GsEKZLB/feQsj1y9dhhBDVV2Ns= k8s.io/apiserver v0.16.4/go.mod h1:kbLJOak655g6W7C+muqu1F76u9wnEycfKMqbVaXIdAc= +k8s.io/apiserver v0.17.0/go.mod h1:ABM+9x/prjINN6iiffRVNCBR2Wk7uY4z+EtEGZD48cg= +k8s.io/apiserver v0.17.2/go.mod h1:lBmw/TtQdtxvrTk0e2cgtOxHizXI+d0mmGQURIHQZlo= +k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw= +k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw= +k8s.io/apiserver v0.18.4/go.mod h1:q+zoFct5ABNnYkGIaGQ3bcbUNdmPyOCoEBcg51LChY8= +k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg= +k8s.io/apiserver v0.19.0/go.mod h1:XvzqavYj73931x7FLtyagh8WibHpePJ1QwWrSJs2CLk= k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= k8s.io/apiserver v0.19.4/go.mod h1:X8WRHCR1UGZDd7HpV0QDc1h/6VbbpAeAGyxSh8yzZXw= -k8s.io/client-go v0.19.4 h1:85D3mDNoLF+xqpyE9Dh/OtrJDyJrSRKkHmDXIbEzer8= -k8s.io/client-go v0.19.4/go.mod h1:ZrEy7+wj9PjH5VMBCuu/BDlvtUAku0oVFk4MmnW9mWA= +k8s.io/autoscaler v0.0.0-20190218140445-7f77136aeea4/go.mod h1:QEXezc9uKPT91dwqhSJq3GNI3B1HxFRQHiku9kmrsSA= +k8s.io/cli-runtime v0.19.0/go.mod h1:tun9l0eUklT8IHIM0jors17KmUjcrAxn0myoBYwuNuo= +k8s.io/cli-runtime v0.19.4/go.mod h1:m8G32dVbKOeaX1foGhleLEvNd6REvU7YnZyWn5//9rw= +k8s.io/client-go v0.20.2 h1:uuf+iIAbfnCSw8IGAv/Rg0giM+2bOzHLOsbbrwrdhNQ= +k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= +k8s.io/cloud-provider v0.17.0/go.mod h1:Ze4c3w2C0bRsjkBUoHpFi+qWe3ob1wI2/7cUn+YQIDE= k8s.io/code-generator v0.0.0-20181114232248-ae218e241252/go.mod h1:IPqxl/YHk05nodzupwjke6ctMjyNRdV2zZ5/j3/F204= +k8s.io/code-generator v0.0.0-20190612205613-18da4a14b22b/go.mod h1:G8bQwmHm2eafm5bgtX67XDZQ8CWKSGu9DekI+yN4Y5I= k8s.io/code-generator v0.0.0-20190717022600-77f3a1fe56bb/go.mod h1:cDx5jQmWH25Ff74daM7NVYty9JWw9dvIS9zT9eIubCY= k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE= k8s.io/code-generator v0.16.4/go.mod h1:mJUgkl06XV4kstAnLHAIzJPVCOzVR+ZcfPIv4fUsFCY= +k8s.io/code-generator v0.17.1/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= +k8s.io/code-generator v0.17.2/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= +k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= +k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= +k8s.io/code-generator v0.18.4/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= +k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= +k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= k8s.io/code-generator v0.19.4/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= +k8s.io/code-generator v0.20.4/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA= +k8s.io/component-base v0.0.0-20190918200425-ed2f0867c778/go.mod h1:DFWQCXgXVLiWtzFaS17KxHdlUeUymP7FLxZSkmL9/jU= k8s.io/component-base v0.16.4/go.mod h1:GYQ+4hlkEwdlpAp59Ztc4gYuFhdoZqiAJD1unYDJ3FM= +k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= +k8s.io/component-base v0.17.2/go.mod h1:zMPW3g5aH7cHJpKYQ/ZsGMcgbsA/VyhEugF3QT1awLs= +k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c= +k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM= +k8s.io/component-base v0.18.4/go.mod h1:7jr/Ef5PGmKwQhyAz/pjByxJbC58mhKAhiaDu0vXfPk= +k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14= +k8s.io/component-base v0.19.0/go.mod h1:dKsY8BxkA+9dZIAh2aWJLL/UdASFDNtGYTCItL4LM7Y= k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo= k8s.io/component-base v0.19.4 h1:HobPRToQ8KJ9ubRju6PUAk9I5V1GNMJZ4PyWbiWA0uI= k8s.io/component-base v0.19.4/go.mod h1:ZzuSLlsWhajIDEkKF73j64Gz/5o0AgON08FgRbEPI70= +k8s.io/csi-translation-lib v0.17.0/go.mod h1:HEF7MEz7pOLJCnxabi45IPkhSsE/KmxPQksuCrHKWls= k8s.io/gengo v0.0.0-20181106084056-51747d6e00da/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190907103519-ebc107f98eab/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20191108084044-e500ee069b5c/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.0.0-20190306015804-8e90cee79f82/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.3.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-aggregator v0.19.0/go.mod h1:1Ln45PQggFAG8xOqWPIYMxUq8WNtpPnYsbUJ39DpF/A= +k8s.io/kube-aggregator v0.19.4/go.mod h1:cTkvun110194d797AuThyydBBlgm+cKIFUeS2uzGJfU= k8s.io/kube-openapi v0.0.0-20181114233023-0317810137be/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= k8s.io/kube-openapi v0.0.0-20190709113604-33be087ad058/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4= k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ= +k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= +k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kubectl v0.19.0/go.mod h1:gPCjjsmE6unJzgaUNXIFGZGafiUp5jh0If3F/x7/rRg= +k8s.io/kubectl v0.19.4/go.mod h1:XPmlu4DJEYgD83pvZFeKF8+MSvGnYGqunbFSrJsqHv0= k8s.io/kubelet v0.19.4 h1:X5xd2BAJYz7i+arNgMlQSJl7r2xzpfducf4BYqn/Loo= k8s.io/kubelet v0.19.4/go.mod h1:zJnPeb7nJCRvtAwxJhe9fFCtMLXL3cXbQiczPmpDrLU= +k8s.io/legacy-cloud-providers v0.17.0/go.mod h1:DdzaepJ3RtRy+e5YhNtrCYwlgyK87j/5+Yfp0L9Syp8= +k8s.io/metrics v0.19.0/go.mod h1:WykpW8B60OeAJx1imdwUgyOID2kDljr/Q+1zrPJ98Wo= +k8s.io/metrics v0.19.4/go.mod h1:a0gvAzrxQPw2ouBqnXI7X9qlggpPkKAFgWU/Py+KZiU= +k8s.io/test-infra v0.0.0-20181019233642-2e10a0bbe9b3/go.mod h1:2NzXB13Ji0nqpyublHeiPC4FZwU0TknfvyaaNfl/BTA= +k8s.io/test-infra v0.0.0-20200220102703-18fae0a00a2c/go.mod h1:B9KsgNJiVixsZud99/ugFoQys8h9Tyv/A/eG5LMyrEE= +k8s.io/utils v0.0.0-20181019225348-5e321f9a457c/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= +k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= +k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +knative.dev/caching v0.0.0-20190719140829-2032732871ff/go.mod h1:dHXFU6CGlLlbzaWc32g80cR92iuBSpsslDNBWI8C7eg= +knative.dev/eventing-contrib v0.6.1-0.20190723221543-5ce18048c08b/go.mod h1:SnXZgSGgMSMLNFTwTnpaOH7hXDzTFtw0J8OmHflNx3g= +knative.dev/pkg v0.0.0-20191111150521-6d806b998379/go.mod h1:pgODObA1dTyhNoFxPZTTjNWfx6F0aKsKzn+vaT9XO/Q= kubevirt.io/client-go v0.30.0 h1:0jUvTa/Ev03lCN+Dr4mH22ipoJ9otAOkpFh6wA66b5M= kubevirt.io/client-go v0.30.0/go.mod h1:JY7hQq+SUT0aLvleXrW/+28fDfZ6BPe4E6f8FyC8jkY= kubevirt.io/containerized-data-importer v1.10.6 h1:xkqLb48pkbdoY8gB2VDP2o+KXpO18tgQuLjcXNn0qAI= @@ -1395,20 +1997,42 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= +mvdan.cc/xurls/v2 v2.0.0/go.mod h1:2/webFPYOXN9jp/lzuj0zuAVlF+9g4KPFJANH1oJhRU= +pack.ag/amqp v0.11.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= +sigs.k8s.io/controller-runtime v0.3.0/go.mod h1:Cw6PkEg0Sa7dAYovGT4R0tRkGhHXpYijwNxYhAnAZZk= +sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns= +sigs.k8s.io/controller-runtime v0.6.1/go.mod h1:XRYBPdbf5XJu9kpS84VJiZ7h/u1hF3gEORz0efEja7A= +sigs.k8s.io/controller-runtime v0.6.2/go.mod h1:vhcq/rlnENJ09SIRp3EveTaZ0yqH526hjf9iJdbUJ/E= +sigs.k8s.io/controller-runtime v0.6.3/go.mod h1:WlZNXcM0++oyaQt4B7C2lEE5JYRs8vJUzRP4N4JpdAY= sigs.k8s.io/controller-runtime v0.7.0 h1:bU20IBBEPccWz5+zXpLnpVsgBYxqclaHu1pVDl/gEt8= sigs.k8s.io/controller-runtime v0.7.0/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= sigs.k8s.io/controller-tools v0.2.4/go.mod h1:m/ztfQNocGYBgTTCmFdnK94uVvgxeZeE3LtJvd/jIzA= +sigs.k8s.io/controller-tools v0.2.9-0.20200414181213-645d44dca7c0/go.mod h1:YKE/iHvcKITCljdnlqHYe+kAt7ZldvtAwUzQff0k1T0= +sigs.k8s.io/controller-tools v0.4.1/go.mod h1:G9rHdZMVlBDocIxGkK3jHLWqcTMNvveypYJwrvYKjWU= +sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= +sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= +sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= sigs.k8s.io/structured-merge-diff v1.0.1 h1:LOs1LZWMsz1xs77Phr/pkB4LFaavH7IVq/3+WTN9XTA= sigs.k8s.io/structured-merge-diff v1.0.1/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= -sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= +sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +software.sslmate.com/src/go-pkcs12 v0.0.0-20180114231543-2291e8f0f237/go.mod h1:/xvNRWUqm0+/ZMiF4EX00vrSCMsE4/NHb+Pt3freEeQ= +software.sslmate.com/src/go-pkcs12 v0.0.0-20200830195227-52f69702a001/go.mod h1:/xvNRWUqm0+/ZMiF4EX00vrSCMsE4/NHb+Pt3freEeQ= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= +vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= +vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= diff --git a/hack/run-machine-controller.sh b/hack/run-machine-controller.sh index cf607c18c..8dea7808f 100755 --- a/hack/run-machine-controller.sh +++ b/hack/run-machine-controller.sh @@ -29,4 +29,5 @@ $(dirname $0)/../machine-controller \ -cluster-dns=172.16.0.10 \ -enable-profiling \ -metrics-address=0.0.0.0:8080 \ + -use-osm \ -health-probe-address=0.0.0.0:8085 diff --git a/pkg/cloudprovider/provider/packet/provider.go b/pkg/cloudprovider/provider/packet/provider.go index 055d5f95a..576aed263 100644 --- a/pkg/cloudprovider/provider/packet/provider.go +++ b/pkg/cloudprovider/provider/packet/provider.go @@ -260,7 +260,6 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.P } client := getClient(c.APIKey) - res, err := client.Devices.Delete(instance.(*packetDevice).device.ID) if err != nil { return false, packetErrorToTerminalError(err, res, "failed to delete the server") diff --git a/pkg/cloudprovider/util/cloud_init_settings.go b/pkg/cloudprovider/util/cloud_init_settings.go index 2b7e1654a..b4fa4c5b6 100644 --- a/pkg/cloudprovider/util/cloud_init_settings.go +++ b/pkg/cloudprovider/util/cloud_init_settings.go @@ -37,6 +37,28 @@ const ( jwtTokenNamePrefix = "cloud-init-getter-token" ) +func ExtractAPIServerToken(ctx context.Context, client ctrlruntimeclient.Client) (string, error) { + secretList := corev1.SecretList{} + if err := client.List(ctx, &secretList, &ctrlruntimeclient.ListOptions{Namespace: CloudInitNamespace}); err != nil { + return "", fmt.Errorf("failed to list secrets in namespace %s: %v", CloudInitNamespace, err) + } + + for _, secret := range secretList.Items { + if strings.HasPrefix(secret.Name, jwtTokenNamePrefix) { + if secret.Data != nil { + jwtToken := secret.Data["token"] + if jwtToken != nil { + token := string(jwtToken) + return token, nil + } + } + } + } + + return "", errors.New("failed to fetch api server token") + +} + func ExtractTokenAndAPIServer(ctx context.Context, userdata string, client ctrlruntimeclient.Client) (token string, apiServer string, err error) { secretList := corev1.SecretList{} if err := client.List(ctx, &secretList, &ctrlruntimeclient.ListOptions{Namespace: CloudInitNamespace}); err != nil { diff --git a/pkg/clusterinfo/configmap.go b/pkg/clusterinfo/configmap.go index 5fb6b380c..5d654fcae 100644 --- a/pkg/clusterinfo/configmap.go +++ b/pkg/clusterinfo/configmap.go @@ -135,3 +135,7 @@ func getCAData(config *rest.Config) ([]byte, error) { return ioutil.ReadFile(config.TLSClientConfig.CAFile) } + +func (p *KubeconfigProvider) GetBearerToken() string { + return p.clientConfig.BearerToken +} diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index f99e513f3..314055aac 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -17,12 +17,16 @@ limitations under the License. package controller import ( + "bytes" "context" + "encoding/base64" "errors" "fmt" "net" + "regexp" "strconv" "strings" + "text/template" "time" "github.com/heptiolabs/healthcheck" @@ -35,6 +39,7 @@ import ( cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/containerruntime" kuberneteshelper "github.com/kubermatic/machine-controller/pkg/kubernetes" "github.com/kubermatic/machine-controller/pkg/node/eviction" @@ -91,6 +96,8 @@ const ( // AnnotationAutoscalerIdentifier is used by the cluster-autoscaler // cluster-api provider to match Nodes to Machines AnnotationAutoscalerIdentifier = "cluster.k8s.io/machine" + + provisioningSuffix = "osc-provisioning" ) // Reconciler is the controller implementation for machine resources @@ -111,8 +118,10 @@ type Reconciler struct { nodeSettings NodeSettings redhatSubscriptionManager rhsm.RedHatSubscriptionManager satelliteSubscriptionManager rhsm.SatelliteSubscriptionManager - podCIDR string - nodePortRange string + + useOSM bool + podCIDR string + nodePortRange string } type NodeSettings struct { @@ -143,6 +152,7 @@ type NodeSettings struct { type KubeconfigProvider interface { GetKubeconfig(context.Context) (*clientcmdapi.Config, error) + GetBearerToken() string } // MetricsCollection is a struct of all metrics used in @@ -169,6 +179,7 @@ func Add( bootstrapTokenServiceAccountName *types.NamespacedName, skipEvictionAfter time.Duration, nodeSettings NodeSettings, + useOSM bool, podCIDR string, nodePortRange string, ) error { @@ -186,8 +197,10 @@ func Add( nodeSettings: nodeSettings, redhatSubscriptionManager: rhsm.NewRedHatSubscriptionManager(), satelliteSubscriptionManager: rhsm.NewSatelliteSubscriptionManager(), - podCIDR: podCIDR, - nodePortRange: nodePortRange, + + useOSM: useOSM, + podCIDR: podCIDR, + nodePortRange: nodePortRange, } m, err := userdatamanager.New() if err != nil { @@ -732,10 +745,41 @@ func (r *Reconciler) ensureInstanceExistsForMachine( PodCIDR: r.podCIDR, NodePortRange: r.nodePortRange, } + // Here we do stuff! + var userdata string - userdata, err := userdataPlugin.UserData(req) - if err != nil { - return nil, fmt.Errorf("failed get userdata: %v", err) + if r.useOSM { + referencedMachineDeployment, err := r.getMachineDeploymentNameForMachine(ctx, machine) + if err != nil { + return nil, fmt.Errorf("failed to find machine's MachineDployment: %v", err) + } + + cloudInitConfigSecretName := fmt.Sprintf("%s-%s", + referencedMachineDeployment, + provisioningSuffix) + + // It is important to check if the secret which holds the cloud init configurations + if err := r.client.Get(ctx, + types.NamespacedName{Name: cloudInitConfigSecretName, Namespace: "kube-system"}, + &corev1.Secret{}); err != nil { + klog.Errorf("Cloud init configurations for machine: %v is not ready yet", machine.Name) + return nil, err + } + + userdata, err = getOSMBootstrapUserdata(ctx, r.client, req, cloudInitConfigSecretName) + if err != nil { + return nil, fmt.Errorf("failed get OSM userdata: %v", err) + } + + userdata, err = cleanupTemplateOutput(userdata) + if err != nil { + return nil, fmt.Errorf("failed to cleanup user-data template: %v", err) + } + } else { + userdata, err = userdataPlugin.UserData(req) + if err != nil { + return nil, fmt.Errorf("failed get userdata: %v", err) + } } // Create the instance @@ -1042,3 +1086,159 @@ func (r *Reconciler) updateNode(ctx context.Context, node *corev1.Node, modifier return r.client.Update(ctx, node) }) } + +func (r *Reconciler) getMachineDeploymentNameForMachine(ctx context.Context, machine *clusterv1alpha1.Machine) (string, error) { + var ( + machineSetName string + machineDeploymentName string + ) + for _, ownerRef := range machine.OwnerReferences { + if ownerRef.Kind == "MachineSet" { + machineSetName = ownerRef.Name + } + } + + if machineSetName != "" { + machineSet := &clusterv1alpha1.MachineSet{} + if err := r.client.Get(ctx, types.NamespacedName{Name: machineSetName, Namespace: "kube-system"}, machineSet); err != nil { + return "", err + } + + for _, ownerRef := range machineSet.OwnerReferences { + if ownerRef.Kind == "MachineDeployment" { + machineDeploymentName = ownerRef.Name + } + } + + if machineDeploymentName != "" { + return machineDeploymentName, nil + } + } + + return "", fmt.Errorf("failed to find machine deployment reference for the machine %s", machine.Name) +} + +func getOSMBootstrapUserdata(ctx context.Context, client ctrlruntimeclient.Client, req plugin.UserDataRequest, secretName string) (string, error) { + + var clusterName string + for key := range req.Kubeconfig.Clusters { + clusterName = key + } + + token, err := util.ExtractAPIServerToken(ctx, client) + if err != nil { + return "", fmt.Errorf("failed to fetch api-server token: %v", err) + } + + data := struct { + Token string + SecretName string + ServerURL string + MachineName string + }{ + Token: token, + SecretName: secretName, + ServerURL: req.Kubeconfig.Clusters[clusterName].Server, + MachineName: req.MachineSpec.Name, + } + bsScript, err := template.New("bootstrap-cloud-init").Parse(bootstrapBinContentTemplate) + if err != nil { + return "", fmt.Errorf("failed to parse bootstrapBinContentTemplate template: %v", err) + } + script := &bytes.Buffer{} + err = bsScript.Execute(script, data) + if err != nil { + return "", fmt.Errorf("failed to execute bootstrapBinContentTemplate template: %v", err) + } + bsCloudInit, err := template.New("bootstrap-cloud-init").Parse(cloudInitTemplate) + if err != nil { + return "", fmt.Errorf("failed to parse download-binaries template: %v", err) + } + pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) + if err != nil { + return "", fmt.Errorf("failed to get providerSpec: %v", err) + } + + cloudInit := &bytes.Buffer{} + err = bsCloudInit.Execute(cloudInit, struct { + Script string + Service string + plugin.UserDataRequest + ProviderSpec *providerconfigtypes.Config + }{ + Script: base64.StdEncoding.EncodeToString(script.Bytes()), + Service: base64.StdEncoding.EncodeToString([]byte(bootstrapServiceContentTemplate)), + UserDataRequest: req, + ProviderSpec: pconfig, + }) + if err != nil { + return "", fmt.Errorf("failed to execute cloudInitTemplate template: %v", err) + } + return cloudInit.String(), nil +} + +// cleanupTemplateOutput postprocesses the output of the template processing. Those +// may exist due to the working of template functions like those of the sprig package +// or template condition. +func cleanupTemplateOutput(output string) (string, error) { + // Valid YAML files are not allowed to have empty lines containing spaces or tabs. + // So far only cleanup. + woBlankLines := regexp.MustCompile(`(?m)^[ \t]+$`).ReplaceAllString(output, "") + return woBlankLines, nil +} + +const ( + bootstrapBinContentTemplate = `#!/bin/bash +set -xeuo pipefail +apt update && apt install -y curl jq +curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-init"]' -r| base64 -d > /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg +cloud-init clean +cloud-init --file /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg init +systemctl daemon-reload +systemctl restart setup.service +systemctl restart kubelet.service +systemctl restart kubelet-healthcheck.service + ` + + bootstrapServiceContentTemplate = `[Install] +WantedBy=multi-user.target + +[Unit] +Requires=network-online.target +After=network-online.target +[Service] +Type=oneshot +RemainAfterExit=true +ExecStart=/opt/bin/bootstrap + ` + + cloudInitTemplate = `#cloud-config +{{ if ne .CloudProviderName "aws" }} +hostname: {{ .MachineSpec.Name }} +{{- /* Never set the hostname on AWS nodes. Kubernetes(kube-proxy) requires the hostname to be the private dns name */}} +{{ end }} +ssh_pwauth: no + +{{- if .ProviderSpec.SSHPublicKeys }} +ssh_authorized_keys: +{{- range .ProviderSpec.SSHPublicKeys }} +- "{{ . }}" +{{- end }} +{{- end }} + +write_files: +- path: /opt/bin/bootstrap + permissions: '0755' + encoding: b64 + content: | + {{ .Script }} +- path: /etc/systemd/system/bootstrap.service + permissions: '0644' + encoding: b64 + content: | + {{ .Service }} +runcmd: +- systemctl restart bootstrap.service +- systemctl daemon-reload +` +) diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index 1143cde34..c11e33171 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -91,7 +91,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { if err != nil { return "", fmt.Errorf("failed to generate container runtime config: %w", err) } - data := struct { plugin.UserDataRequest ProviderSpec *providerconfigtypes.Config From 6e521e51a6ee9d8db7d617992e1c5655cc116fe5 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Mon, 25 Oct 2021 23:36:43 +0200 Subject: [PATCH 007/489] enable vsphere e2e tests (#1079) Signed-off-by: Moath Qasim --- .prow.yaml | 4 ++-- test/e2e/provisioning/all_e2e_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.prow.yaml b/.prow.yaml index f749ee1ca..c1bdfbd91 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -545,7 +545,7 @@ presubmits: cpu: 500m - name: pull-machine-controller-e2e-vsphere - always_run: false + always_run: true decorate: true error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" @@ -790,7 +790,7 @@ presubmits: cpu: 500m - name: pull-machine-controller-e2e-vsphere-resource-pool - always_run: false + always_run: true decorate: true error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 8e50109eb..8b852edfd 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -766,7 +766,7 @@ func getVSphereTestParams(t *testing.T) []string { func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() - selector := Not(OsSelector("sles", "rhel", "amzn2")) + selector := Not(OsSelector("sles", "amzn2")) params := getVSphereTestParams(t) runScenarios(t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) From ae9eaded595d3a32d6ffd8a9ee80fa4cb82bdfbf Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Fri, 29 Oct 2021 15:50:35 +0200 Subject: [PATCH 008/489] Upgrade e2e tests (#1080) * make containerd the default cri Signed-off-by: Moath Qasim * upgrading k8s master node in e2e tests Signed-off-by: Moath Qasim * upgrading k8s versions in e2e tests Signed-off-by: Moath Qasim * debug Signed-off-by: Moath Qasim * configure containerd Signed-off-by: Moath Qasim * add buildah Signed-off-by: Moath Qasim * run containerd as cri in e2e tests Signed-off-by: Moath Qasim * debug Signed-off-by: Moath Qasim * debug Signed-off-by: Moath Qasim * update fixtures Signed-off-by: Moath Qasim * revert debug Signed-off-by: Moath Qasim * update fixtures Signed-off-by: Moath Qasim * fixing defaults Signed-off-by: Moath Qasim * fix container runtime defaulting bug Signed-off-by: Moath Qasim * update flatcar image in azure Signed-off-by: Moath Qasim * fix custom tests Signed-off-by: Moath Qasim --- hack/ci-e2e-test.sh | 2 +- pkg/cloudprovider/provider/azure/provider.go | 2 +- pkg/containerruntime/containerd.go | 4 + pkg/containerruntime/containerruntime.go | 14 ++-- pkg/containerruntime/docker.go | 4 + pkg/userdata/amzn2/provider.go | 4 +- pkg/userdata/centos/provider.go | 4 +- pkg/userdata/rhel/provider.go | 4 +- pkg/userdata/ubuntu/provider.go | 4 +- test/e2e/provisioning/all_e2e_test.go | 6 +- test/e2e/provisioning/helper.go | 13 ++- test/tools/integration/hetzner.tf | 2 +- .../integration/master_install_script.sh | 80 +++++++++++++++++-- 13 files changed, 112 insertions(+), 31 deletions(-) diff --git a/hack/ci-e2e-test.sh b/hack/ci-e2e-test.sh index beff0a76d..24b6c366a 100755 --- a/hack/ci-e2e-test.sh +++ b/hack/ci-e2e-test.sh @@ -40,7 +40,7 @@ trap cleanup EXIT echo "Installing dependencies..." apt update && apt install -y jq rsync unzip genisoimage curl --retry 5 --location --remote-name \ - https://storage.googleapis.com/kubernetes-release/release/v1.12.4/bin/linux/amd64/kubectl && + https://storage.googleapis.com/kubernetes-release/release/v1.22.2/bin/linux/amd64/kubectl && chmod +x kubectl && mv kubectl /usr/local/bin diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index f5517952a..558d42f99 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -140,7 +140,7 @@ var imageReferences = map[providerconfigtypes.OperatingSystem]compute.ImageRefer Publisher: to.StringPtr("kinvolk"), Offer: to.StringPtr("flatcar-container-linux"), Sku: to.StringPtr("stable"), - Version: to.StringPtr("2345.3.0"), + Version: to.StringPtr("2905.2.5"), }, } diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index e0e49106c..e42bff608 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -151,3 +151,7 @@ systemctl daemon-reload systemctl enable --now containerd `)) ) + +func (eng *Containerd) String() string { + return containerdName +} diff --git a/pkg/containerruntime/containerruntime.go b/pkg/containerruntime/containerruntime.go index aba82cf19..f15d9ee21 100644 --- a/pkg/containerruntime/containerruntime.go +++ b/pkg/containerruntime/containerruntime.go @@ -23,7 +23,8 @@ import ( ) const ( - Default = "docker" + dockerName = "docker" + containerdName = "containerd" ) type Engine interface { @@ -31,6 +32,7 @@ type Engine interface { ScriptFor(os types.OperatingSystem) (string, error) ConfigFileName() string Config() (string, error) + String() string } type Opt func(*Config) @@ -51,10 +53,10 @@ func Get(containerRuntimeName string, opts ...Opt) Config { cfg := Config{} switch containerRuntimeName { - case "docker": + case dockerName: cfg.Docker = &Docker{} cfg.Containerd = nil - case "containerd": + case containerdName: cfg.Containerd = &Containerd{} cfg.Docker = nil default: @@ -79,12 +81,12 @@ type Config struct { func (cfg Config) String() string { switch { case cfg.Containerd != nil: - return "containerd" + return containerdName case cfg.Docker != nil: - return "docker" + return dockerName } - return Default + return dockerName } func (cfg Config) Engine(kubeletVersion *semver.Version) Engine { diff --git a/pkg/containerruntime/docker.go b/pkg/containerruntime/docker.go index 580af1a68..fc9d8b5b2 100644 --- a/pkg/containerruntime/docker.go +++ b/pkg/containerruntime/docker.go @@ -163,3 +163,7 @@ systemctl daemon-reload systemctl enable --now docker `)) ) + +func (eng *Docker) String() string { + return dockerName +} diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index dc3aee339..1adbf8127 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -105,6 +105,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ContainerRuntimeScript string ContainerRuntimeConfigFileName string ContainerRuntimeConfig string + ContainerRuntimeName string }{ UserDataRequest: req, ProviderSpec: pconfig, @@ -118,6 +119,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ContainerRuntimeScript: crScript, ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), ContainerRuntimeConfig: crConfig, + ContainerRuntimeName: crEngine.String(), } buf := strings.Builder{} @@ -244,7 +246,7 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntime.String .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index a0de9221f..02e2c5e8f 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -105,6 +105,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ContainerRuntimeScript string ContainerRuntimeConfigFileName string ContainerRuntimeConfig string + ContainerRuntimeName string }{ UserDataRequest: req, ProviderSpec: pconfig, @@ -118,6 +119,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ContainerRuntimeScript: crScript, ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), ContainerRuntimeConfig: crConfig, + ContainerRuntimeName: crEngine.String(), } buf := strings.Builder{} @@ -244,7 +246,7 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntime.String .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 52f1ee66f..c7e5c10fb 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -105,6 +105,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ContainerRuntimeScript string ContainerRuntimeConfigFileName string ContainerRuntimeConfig string + ContainerRuntimeName string }{ UserDataRequest: req, ProviderSpec: pconfig, @@ -118,6 +119,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ContainerRuntimeScript: crScript, ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), ContainerRuntimeConfig: crConfig, + ContainerRuntimeName: crEngine.String(), } var buf strings.Builder @@ -256,7 +258,7 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntime.String .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index c11e33171..3765e5917 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -104,6 +104,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ContainerRuntimeScript string ContainerRuntimeConfigFileName string ContainerRuntimeConfig string + ContainerRuntimeName string }{ UserDataRequest: req, ProviderSpec: pconfig, @@ -117,6 +118,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ContainerRuntimeScript: crScript, ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), ContainerRuntimeConfig: crConfig, + ContainerRuntimeName: crEngine.String(), } var buf strings.Builder @@ -245,7 +247,7 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntime.String .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} - path: "/etc/systemd/system/kubelet.service.d/extras.conf" content: | diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 8b852edfd..dbf8520be 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -312,7 +312,7 @@ func TestKubevirtDNSConfigProvisioningE2E(t *testing.T) { name: "Kubevirt with dns config", osName: "ubuntu", containerRuntime: "docker", - kubernetesVersion: "v1.17.0", + kubernetesVersion: "v1.22.2", executor: verifyCreateAndDelete, } @@ -794,7 +794,7 @@ func TestVsphereResourcePoolProvisioningE2E(t *testing.T) { name: "vSphere resource pool provisioning", osName: "flatcar", containerRuntime: "docker", - kubernetesVersion: "1.17.0", + kubernetesVersion: "1.22.2", executor: verifyCreateAndDelete, } @@ -868,7 +868,7 @@ func TestUbuntuProvisioningWithUpgradeE2E(t *testing.T) { name: "Ubuntu upgrade", osName: "ubuntu", containerRuntime: "docker", - kubernetesVersion: "1.16.2", + kubernetesVersion: "1.22.2", executor: verifyCreateAndDelete, } diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 4cc671c14..bc6fe278d 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,10 +33,10 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.17.16"), - semver.MustParse("v1.18.14"), - semver.MustParse("v1.19.4"), - semver.MustParse("v1.20.1"), + semver.MustParse("v1.19.15"), + semver.MustParse("v1.20.11"), + semver.MustParse("v1.21.5"), + semver.MustParse("v1.22.2"), } operatingSystems = []providerconfigtypes.OperatingSystem{ @@ -52,7 +52,7 @@ var ( string(providerconfigtypes.OperatingSystemUbuntu): "machine-controller-e2e-ubuntu-20-04", string(providerconfigtypes.OperatingSystemCentOS): "machine-controller-e2e-centos", string(providerconfigtypes.OperatingSystemRHEL): "machine-controller-e2e-rhel", - string(providerconfigtypes.OperatingSystemFlatcar): "machine-controller-e2e-flatcar", + string(providerconfigtypes.OperatingSystemFlatcar): "Flatcar Stable (2021-10-27)", } ) @@ -207,8 +207,7 @@ func buildScenarios() []scenario { for _, version := range versions { for _, operatingSystem := range operatingSystems { s := scenario{ - name: fmt.Sprintf("%s-%s", operatingSystem, version), - // We only support docker + name: fmt.Sprintf("%s-%s", operatingSystem, version), containerRuntime: "docker", kubernetesVersion: version.String(), osName: string(operatingSystem), diff --git a/test/tools/integration/hetzner.tf b/test/tools/integration/hetzner.tf index 064ad2eb1..badaf2b39 100644 --- a/test/tools/integration/hetzner.tf +++ b/test/tools/integration/hetzner.tf @@ -9,7 +9,7 @@ resource "hcloud_ssh_key" "default" { resource "hcloud_server" "machine-controller-test" { name = var.hcloud_test_server_name - image = "ubuntu-18.04" + image = "ubuntu-20.04" server_type = "cx21" ssh_keys = [hcloud_ssh_key.default.id] location = "nbg1" diff --git a/test/tools/integration/master_install_script.sh b/test/tools/integration/master_install_script.sh index 12f2c4d26..fb44dc534 100644 --- a/test/tools/integration/master_install_script.sh +++ b/test/tools/integration/master_install_script.sh @@ -17,23 +17,83 @@ set -euo pipefail set -x -K8S_VERSION=1.20.1 +K8S_VERSION=1.22.2 echo "$LC_E2E_SSH_PUBKEY" >> .ssh/authorized_keys # Hetzner's Ubuntu Bionic comes with swap pre-configured, so we force it off. systemctl mask swap.target swapoff -a +if ! which buildah; then + sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" + wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_20.04/Release.key -O Release.key + apt-key add - < Release.key + apt-get update + apt-get -y install buildah +fi if ! which make; then apt update apt install make fi -if ! which docker; then +if ! which containerd; then apt update - apt install -y docker.io - systemctl enable docker.service - systemctl start docker - systemctl status docker + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + cat < Date: Mon, 1 Nov 2021 12:19:38 +0100 Subject: [PATCH 009/489] disable e2e tests for vSphere cloud provider temporarily (#1086) Signed-off-by: Moath Qasim --- test/e2e/provisioning/all_e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index dbf8520be..21c5c2258 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -766,7 +766,7 @@ func getVSphereTestParams(t *testing.T) []string { func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() - selector := Not(OsSelector("sles", "amzn2")) + selector := Not(OsSelector("sles", "amzn2", "rhel")) params := getVSphereTestParams(t) runScenarios(t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) From 75fea2b1c3cdb93536af45374df6a924e38ad8a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Mon, 1 Nov 2021 13:51:38 +0100 Subject: [PATCH 010/489] Provide hostname to kubelet for AWS instances with external CCM (#1087) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Provide hostname to kubelet for AWS instances with external CCM Signed-off-by: Marko Mudrinić * Update fixtures Signed-off-by: Marko Mudrinić --- .../amzn2/testdata/containerd-kubelet-v1.20-aws.yaml | 7 +++++-- pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml | 7 +++++-- .../amzn2/testdata/kubelet-v1.17-aws-external.yaml | 8 ++++++-- pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml | 7 +++++-- .../amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml | 7 +++++-- .../amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml | 7 +++++-- pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml | 7 +++++-- pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml | 7 +++++-- pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml | 7 +++++-- pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml | 7 +++++-- .../centos/testdata/kubelet-containerd-v1.20-aws.yaml | 7 +++++-- .../centos/testdata/kubelet-v1.17-aws-external.yaml | 8 ++++++-- pkg/userdata/centos/testdata/kubelet-v1.17-aws.yaml | 7 +++++-- .../centos/testdata/kubelet-v1.17-vsphere-mirrors.yaml | 7 +++++-- .../centos/testdata/kubelet-v1.17-vsphere-proxy.yaml | 7 +++++-- pkg/userdata/centos/testdata/kubelet-v1.17-vsphere.yaml | 7 +++++-- pkg/userdata/centos/testdata/kubelet-v1.18-aws.yaml | 7 +++++-- pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml | 7 +++++-- pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml | 7 +++++-- pkg/userdata/flatcar/testdata/cloud-init_v1.17.16.yaml | 7 +++++-- pkg/userdata/flatcar/testdata/cloud-init_v1.18.14.yaml | 7 +++++-- pkg/userdata/flatcar/testdata/cloud-init_v1.19.4.yaml | 7 +++++-- pkg/userdata/flatcar/testdata/cloud-init_v1.20.1.yaml | 7 +++++-- pkg/userdata/flatcar/testdata/containerd.yaml | 7 +++++-- pkg/userdata/flatcar/testdata/ignition_v1.17.16.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.18.14.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.19.4.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.20.1.json | 2 +- pkg/userdata/helper/helper.go | 7 +++++-- pkg/userdata/helper/kubelet.go | 2 ++ .../rhel/testdata/kubelet-containerd-v1.20-aws.yaml | 7 +++++-- pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml | 7 +++++-- pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml | 7 +++++-- pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml | 7 +++++-- .../rhel/testdata/kubelet-v1.20-aws-external.yaml | 8 ++++++-- pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml | 7 +++++-- .../rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml | 7 +++++-- .../rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml | 7 +++++-- pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml | 7 +++++-- pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml | 7 +++++-- .../sles/testdata/kubelet-version-without-v-prefix.yaml | 7 +++++-- pkg/userdata/sles/testdata/multiple-dns-servers.yaml | 7 +++++-- pkg/userdata/sles/testdata/multiple-ssh-keys.yaml | 7 +++++-- .../sles/testdata/openstack-overwrite-cloud-config.yaml | 7 +++++-- pkg/userdata/sles/testdata/openstack.yaml | 7 +++++-- pkg/userdata/sles/testdata/version-1.17.16.yaml | 7 +++++-- pkg/userdata/sles/testdata/version-1.18.14.yaml | 7 +++++-- pkg/userdata/sles/testdata/version-1.19.4.yaml | 7 +++++-- pkg/userdata/sles/testdata/version-1.20.1.yaml | 7 +++++-- pkg/userdata/sles/testdata/vsphere-mirrors.yaml | 7 +++++-- pkg/userdata/sles/testdata/vsphere-proxy.yaml | 7 +++++-- pkg/userdata/sles/testdata/vsphere.yaml | 7 +++++-- pkg/userdata/ubuntu/testdata/containerd.yaml | 7 +++++-- pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml | 7 +++++-- .../ubuntu/testdata/kubelet-version-without-v-prefix.yaml | 7 +++++-- pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml | 7 +++++-- pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml | 7 +++++-- .../ubuntu/testdata/openstack-overwrite-cloud-config.yaml | 7 +++++-- pkg/userdata/ubuntu/testdata/openstack.yaml | 7 +++++-- pkg/userdata/ubuntu/testdata/version-1.17.16.yaml | 7 +++++-- pkg/userdata/ubuntu/testdata/version-1.18.14.yaml | 7 +++++-- pkg/userdata/ubuntu/testdata/version-1.19.4.yaml | 7 +++++-- pkg/userdata/ubuntu/testdata/version-1.20.1.yaml | 7 +++++-- pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml | 7 +++++-- pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml | 7 +++++-- pkg/userdata/ubuntu/testdata/vsphere.yaml | 7 +++++-- 66 files changed, 314 insertions(+), 126 deletions(-) diff --git a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml index 8fcc3ef81..82861b1d5 100644 --- a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml @@ -239,6 +239,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -249,13 +252,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml index d34965c59..b6fdc0281 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml @@ -235,6 +235,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -245,13 +248,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws-external.yaml index 8cdcdf37c..6d4a64faa 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws-external.yaml @@ -208,6 +208,7 @@ write_files: --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=external \ + --hostname-override=${KUBELET_HOSTNAME} \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ @@ -235,6 +236,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -245,13 +249,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml index 61f028f5e..bfa80e007 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml @@ -236,6 +236,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -246,13 +249,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml index 8361f5913..f666497bd 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml @@ -253,6 +253,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -263,13 +266,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml index 43609e989..544fed03f 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml @@ -253,6 +253,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -263,13 +266,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml index d23372dc3..47efe5b05 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml @@ -244,6 +244,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -254,13 +257,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml index 492c6fede..f0ee44f7f 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml @@ -236,6 +236,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -246,13 +249,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml index 4f5d19864..eafd53633 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml @@ -236,6 +236,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -246,13 +249,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml index 032503d06..1491d5b10 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml @@ -236,6 +236,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -246,13 +249,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml index bbb05a404..666aae1ae 100644 --- a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml @@ -240,6 +240,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -250,13 +253,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/centos/testdata/kubelet-v1.17-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.17-aws-external.yaml index acde5cdf0..40dd03244 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.17-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.17-aws-external.yaml @@ -213,6 +213,7 @@ write_files: --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=external \ + --hostname-override=${KUBELET_HOSTNAME} \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ @@ -240,6 +241,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -250,13 +254,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/centos/testdata/kubelet-v1.17-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.17-aws.yaml index 90059f710..a2df0a5d1 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.17-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.17-aws.yaml @@ -241,6 +241,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -251,13 +254,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-mirrors.yaml index 6cf534fad..ae4af5a39 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-mirrors.yaml @@ -258,6 +258,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -268,13 +271,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-proxy.yaml index 1afd44c7b..b29f18ec6 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-proxy.yaml @@ -258,6 +258,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -268,13 +271,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere.yaml index cf5fd540c..e7aa0ecfd 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere.yaml @@ -249,6 +249,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -259,13 +262,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/centos/testdata/kubelet-v1.18-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.18-aws.yaml index 956b7970b..0bb90884f 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.18-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.18-aws.yaml @@ -241,6 +241,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -251,13 +254,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml index 5d851d59a..0121f01d2 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml @@ -241,6 +241,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -251,13 +254,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml index b5e6818fd..72a1d8ccf 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml @@ -241,6 +241,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -251,13 +254,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.17.16.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.17.16.yaml index e3b53ac44..6ea8f78b0 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.17.16.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.17.16.yaml @@ -293,6 +293,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -303,13 +306,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.18.14.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.18.14.yaml index f9e97e5a6..c0571b49d 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.18.14.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.18.14.yaml @@ -293,6 +293,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -303,13 +306,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.19.4.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.19.4.yaml index f1df637c9..4d26a1431 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.19.4.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.19.4.yaml @@ -293,6 +293,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -303,13 +306,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.1.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.1.yaml index 8df596dbc..09985805c 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.1.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.1.yaml @@ -293,6 +293,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -303,13 +306,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index 54d5bdfd5..dcaa2f0c0 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -276,6 +276,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -286,13 +289,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.17.16.json b/pkg/userdata/flatcar/testdata/ignition_v1.17.16.json index 314fa8c8e..943d8c6d5 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.17.16.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.17.16.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.17.16%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-less-then-1.19/hyperkubeImage:v1.17.16 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.17.16%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-less-then-1.19/hyperkubeImage:v1.17.16 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.18.14.json b/pkg/userdata/flatcar/testdata/ignition_v1.18.14.json index 738cb7840..04be7405a 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.18.14.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.18.14.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.18.14%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-less-then-1.19/hyperkubeImage:v1.18.14 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.18.14%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-less-then-1.19/hyperkubeImage:v1.18.14 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.19.4.json b/pkg/userdata/flatcar/testdata/ignition_v1.19.4.json index 0296ba206..28f40552c 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.19.4.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.19.4.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.19.4%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-more-then-1.19/kubeletImage:v1.19.4 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.19.4%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-more-then-1.19/kubeletImage:v1.19.4 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.20.1.json b/pkg/userdata/flatcar/testdata/ignition_v1.20.1.json index f96c8f604..3f4e1be81 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.20.1.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.20.1.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.1%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-more-then-1.19/kubeletImage:v1.20.1 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.1%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-more-then-1.19/kubeletImage:v1.20.1 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/helper.go b/pkg/userdata/helper/helper.go index 730710aef..3323c68dc 100644 --- a/pkg/userdata/helper/helper.go +++ b/pkg/userdata/helper/helper.go @@ -244,6 +244,9 @@ echodate() { # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") +# get the full hostname +FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -254,13 +257,13 @@ fi # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi ` } diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index a912ebd63..108e5bb56 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -42,6 +42,8 @@ const ( {{- end }} {{- if and (.Hostname) (ne .CloudProvider "aws") }} --hostname-override={{ .Hostname }} \ +{{- else if and (eq .CloudProvider "aws") (.IsExternal) }} +--hostname-override=${KUBELET_HOSTNAME} \ {{- end }} --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ diff --git a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml index ab0678015..fd5a6421e 100644 --- a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml @@ -240,6 +240,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -250,13 +253,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml index 611f21b0b..0d19bead9 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml @@ -241,6 +241,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -251,13 +254,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml index 610b2f705..2373334aa 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml @@ -241,6 +241,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -251,13 +254,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml index da2a8f20b..ab4754e68 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml @@ -241,6 +241,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -251,13 +254,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml index ffc30f63f..2f29993ef 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml @@ -213,6 +213,7 @@ write_files: --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=external \ + --hostname-override=${KUBELET_HOSTNAME} \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ @@ -240,6 +241,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -250,13 +254,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml index a32bc4b33..9c3280ad9 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml @@ -241,6 +241,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -251,13 +254,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml index 1d9e483cc..abe900f0f 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml @@ -259,6 +259,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -269,13 +272,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml index 04f39a87a..cd4a98c6d 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml @@ -259,6 +259,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -269,13 +272,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml index fc64949e5..5d0e5044a 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml @@ -250,6 +250,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -260,13 +263,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml index 8864a7998..f2512d40b 100644 --- a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml @@ -206,6 +206,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -216,13 +219,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml index ff4482d6a..b59b484dd 100644 --- a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml @@ -204,6 +204,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -214,13 +217,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml index 35624d1b5..65345261a 100644 --- a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml @@ -204,6 +204,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -214,13 +217,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml index 3c91cb4c9..f2ca6a03a 100644 --- a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml @@ -206,6 +206,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -216,13 +219,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml index 40e4c8c2a..488abc728 100644 --- a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml @@ -208,6 +208,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -218,13 +221,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/sles/testdata/openstack.yaml b/pkg/userdata/sles/testdata/openstack.yaml index 75e73678d..6205f65ab 100644 --- a/pkg/userdata/sles/testdata/openstack.yaml +++ b/pkg/userdata/sles/testdata/openstack.yaml @@ -206,6 +206,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -216,13 +219,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/sles/testdata/version-1.17.16.yaml b/pkg/userdata/sles/testdata/version-1.17.16.yaml index 6e9e2fd73..1b39e68bf 100644 --- a/pkg/userdata/sles/testdata/version-1.17.16.yaml +++ b/pkg/userdata/sles/testdata/version-1.17.16.yaml @@ -204,6 +204,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -214,13 +217,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/sles/testdata/version-1.18.14.yaml b/pkg/userdata/sles/testdata/version-1.18.14.yaml index fcc89dd8b..11f869073 100644 --- a/pkg/userdata/sles/testdata/version-1.18.14.yaml +++ b/pkg/userdata/sles/testdata/version-1.18.14.yaml @@ -204,6 +204,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -214,13 +217,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/sles/testdata/version-1.19.4.yaml b/pkg/userdata/sles/testdata/version-1.19.4.yaml index 3fc104f87..dac8abba5 100644 --- a/pkg/userdata/sles/testdata/version-1.19.4.yaml +++ b/pkg/userdata/sles/testdata/version-1.19.4.yaml @@ -204,6 +204,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -214,13 +217,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/sles/testdata/version-1.20.1.yaml b/pkg/userdata/sles/testdata/version-1.20.1.yaml index ff4482d6a..b59b484dd 100644 --- a/pkg/userdata/sles/testdata/version-1.20.1.yaml +++ b/pkg/userdata/sles/testdata/version-1.20.1.yaml @@ -204,6 +204,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -214,13 +217,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml index 4464c5b68..7c838d297 100644 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml @@ -219,6 +219,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -229,13 +232,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml index fba9f967d..4a7734f1f 100644 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/sles/testdata/vsphere-proxy.yaml @@ -219,6 +219,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -229,13 +232,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml index 5717c6a0d..ca099709c 100644 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ b/pkg/userdata/sles/testdata/vsphere.yaml @@ -209,6 +209,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -219,13 +222,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index 16c3a4e07..dd462c871 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -252,6 +252,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -262,13 +265,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index fbccfa0d6..6775c4227 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -252,6 +252,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -262,13 +265,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index 64b02b0bd..2d8c72f21 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -250,6 +250,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -260,13 +263,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index ba3b58964..c4c286d50 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -250,6 +250,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -260,13 +263,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 31a45b767..736d42862 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -252,6 +252,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -262,13 +265,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index a31a8a522..8da93a43e 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -254,6 +254,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -264,13 +267,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index f355ecbcf..826b030e7 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -252,6 +252,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -262,13 +265,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/ubuntu/testdata/version-1.17.16.yaml b/pkg/userdata/ubuntu/testdata/version-1.17.16.yaml index 7f8fb5142..6a170b46a 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.17.16.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.17.16.yaml @@ -250,6 +250,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -260,13 +263,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/ubuntu/testdata/version-1.18.14.yaml b/pkg/userdata/ubuntu/testdata/version-1.18.14.yaml index 82fc6ba3a..afaf93c84 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.18.14.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.18.14.yaml @@ -250,6 +250,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -260,13 +263,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/ubuntu/testdata/version-1.19.4.yaml b/pkg/userdata/ubuntu/testdata/version-1.19.4.yaml index 0eaef6879..0950810d5 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.19.4.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.19.4.yaml @@ -250,6 +250,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -260,13 +263,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/ubuntu/testdata/version-1.20.1.yaml b/pkg/userdata/ubuntu/testdata/version-1.20.1.yaml index ff0d98592..7d0c206b2 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.20.1.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.20.1.yaml @@ -250,6 +250,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -260,13 +263,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index b66b6707c..6682efc38 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -265,6 +265,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -275,13 +278,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 3eef17ae4..9fbe91618 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -265,6 +265,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -275,13 +278,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index eb8423114..0009ad2f2 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -255,6 +255,9 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + if [ -z "${DEFAULT_IFC_IP}" ] then echodate "Failed to get IP address for the default route interface" @@ -265,13 +268,13 @@ write_files: # we need the line below because flatcar has the same string "coreos" in that file if grep -q coreos /etc/os-release then - echo "KUBELET_NODE_IP=${DEFAULT_IFC_IP}" > /etc/kubernetes/nodeip.conf + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf elif [ ! -d /etc/systemd/system/kubelet.service.d ] then echodate "Can't find kubelet service extras directory" exit 1 else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf fi From 4659b47679d1ef969a5f3bc2e88dfdb1d6588a5d Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 1 Nov 2021 20:33:38 +0500 Subject: [PATCH 011/489] Update TLS Cipher Suites config for kubelet and restrict the allowed values (#1077) * Update TLS Cipher Suites config for kubelet and restrict the allowed values * Update fixtures * Update allowed semver for e2e provisioing tests * Update fixtures * Fix formatting error for kubelet.go * Improve code styling * Update kubectl version for ci e2e environment * Update e2e cluster configuration to use k8s v1.22.2 with containerd as CRI * Templatize container runtime health check to pick container runtime dynamically * Update test fixture for flatcard instance with cri containerd * Update script to install master node for integration tests * Remove invalid trailing slash from run-machine-controller.sh * Fix linting errors * Remove deprecated commands for terraform --- hack/ci-e2e-test.sh | 2 +- hack/run-machine-controller.sh | 3 +- .../containerd-kubelet-v1.20-aws.yaml | 10 ++++ .../amzn2/testdata/kubelet-v1.16-aws.yaml | 10 ++++ .../testdata/kubelet-v1.17-aws-external.yaml | 10 ++++ .../amzn2/testdata/kubelet-v1.17-aws.yaml | 10 ++++ .../kubelet-v1.17-vsphere-mirrors.yaml | 10 ++++ .../testdata/kubelet-v1.17-vsphere-proxy.yaml | 10 ++++ .../amzn2/testdata/kubelet-v1.17-vsphere.yaml | 10 ++++ .../amzn2/testdata/kubelet-v1.18-aws.yaml | 10 ++++ .../amzn2/testdata/kubelet-v1.19-aws.yaml | 10 ++++ .../amzn2/testdata/kubelet-v1.20-aws.yaml | 10 ++++ .../kubelet-containerd-v1.20-aws.yaml | 10 ++++ .../testdata/kubelet-v1.17-aws-external.yaml | 10 ++++ .../centos/testdata/kubelet-v1.17-aws.yaml | 10 ++++ .../kubelet-v1.17-vsphere-mirrors.yaml | 10 ++++ .../testdata/kubelet-v1.17-vsphere-proxy.yaml | 10 ++++ .../testdata/kubelet-v1.17-vsphere.yaml | 10 ++++ .../centos/testdata/kubelet-v1.18-aws.yaml | 10 ++++ .../centos/testdata/kubelet-v1.19-aws.yaml | 10 ++++ .../centos/testdata/kubelet-v1.20-aws.yaml | 10 ++++ pkg/userdata/flatcar/provider.go | 4 +- .../flatcar/testdata/cloud-init_v1.17.16.yaml | 10 ++++ .../flatcar/testdata/cloud-init_v1.18.14.yaml | 10 ++++ .../flatcar/testdata/cloud-init_v1.19.4.yaml | 10 ++++ .../flatcar/testdata/cloud-init_v1.20.1.yaml | 10 ++++ pkg/userdata/flatcar/testdata/containerd.yaml | 14 +++++- .../flatcar/testdata/ignition_v1.17.16.json | 2 +- .../flatcar/testdata/ignition_v1.18.14.json | 2 +- .../flatcar/testdata/ignition_v1.19.4.json | 2 +- .../flatcar/testdata/ignition_v1.20.1.json | 2 +- pkg/userdata/helper/kubelet.go | 49 ++++++++++++++++--- .../kubelet-containerd-v1.20-aws.yaml | 10 ++++ .../rhel/testdata/kubelet-v1.17-aws.yaml | 10 ++++ .../rhel/testdata/kubelet-v1.18-aws.yaml | 10 ++++ .../rhel/testdata/kubelet-v1.19-aws.yaml | 10 ++++ .../testdata/kubelet-v1.20-aws-external.yaml | 10 ++++ .../rhel/testdata/kubelet-v1.20-aws.yaml | 10 ++++ .../kubelet-v1.20-vsphere-mirrors.yaml | 10 ++++ .../testdata/kubelet-v1.20-vsphere-proxy.yaml | 10 ++++ .../rhel/testdata/kubelet-v1.20-vsphere.yaml | 10 ++++ pkg/userdata/sles/provider.go | 2 +- .../sles/testdata/dist-upgrade-on-boot.yaml | 10 ++++ .../kubelet-version-without-v-prefix.yaml | 10 ++++ .../sles/testdata/multiple-dns-servers.yaml | 10 ++++ .../sles/testdata/multiple-ssh-keys.yaml | 10 ++++ .../openstack-overwrite-cloud-config.yaml | 10 ++++ pkg/userdata/sles/testdata/openstack.yaml | 10 ++++ .../sles/testdata/version-1.17.16.yaml | 10 ++++ .../sles/testdata/version-1.18.14.yaml | 10 ++++ .../sles/testdata/version-1.19.4.yaml | 10 ++++ .../sles/testdata/version-1.20.1.yaml | 10 ++++ .../sles/testdata/vsphere-mirrors.yaml | 10 ++++ pkg/userdata/sles/testdata/vsphere-proxy.yaml | 10 ++++ pkg/userdata/sles/testdata/vsphere.yaml | 10 ++++ pkg/userdata/ubuntu/testdata/containerd.yaml | 10 ++++ .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 10 ++++ .../kubelet-version-without-v-prefix.yaml | 10 ++++ .../ubuntu/testdata/multiple-dns-servers.yaml | 10 ++++ .../ubuntu/testdata/multiple-ssh-keys.yaml | 10 ++++ .../openstack-overwrite-cloud-config.yaml | 10 ++++ pkg/userdata/ubuntu/testdata/openstack.yaml | 10 ++++ .../ubuntu/testdata/version-1.17.16.yaml | 10 ++++ .../ubuntu/testdata/version-1.18.14.yaml | 10 ++++ .../ubuntu/testdata/version-1.19.4.yaml | 10 ++++ .../ubuntu/testdata/version-1.20.1.yaml | 10 ++++ .../ubuntu/testdata/vsphere-mirrors.yaml | 10 ++++ .../ubuntu/testdata/vsphere-proxy.yaml | 10 ++++ pkg/userdata/ubuntu/testdata/vsphere.yaml | 10 ++++ test/e2e/provisioning/helper.go | 2 +- test/tools/integration/Makefile | 4 +- .../integration/master_install_script.sh | 2 +- 72 files changed, 656 insertions(+), 24 deletions(-) diff --git a/hack/ci-e2e-test.sh b/hack/ci-e2e-test.sh index 24b6c366a..3e68be68d 100755 --- a/hack/ci-e2e-test.sh +++ b/hack/ci-e2e-test.sh @@ -28,7 +28,7 @@ function cleanup { for try in {1..20}; do # Clean up master echo "Cleaning up controller, attempt ${try}" - terraform destroy -force + terraform apply -destroy -auto-approve if [[ $? == 0 ]]; then break; fi echo "Sleeping for $try seconds" sleep ${try}s diff --git a/hack/run-machine-controller.sh b/hack/run-machine-controller.sh index 8dea7808f..58b7edb29 100755 --- a/hack/run-machine-controller.sh +++ b/hack/run-machine-controller.sh @@ -30,4 +30,5 @@ $(dirname $0)/../machine-controller \ -enable-profiling \ -metrics-address=0.0.0.0:8080 \ -use-osm \ - -health-probe-address=0.0.0.0:8085 + -health-probe-address=0.0.0.0:8085 \ + -node-container-runtime=containerd diff --git a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml index 82861b1d5..5063d7ae3 100644 --- a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml @@ -325,6 +325,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml index b6fdc0281..ed528241f 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml @@ -321,6 +321,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws-external.yaml index 6d4a64faa..6158cb09f 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws-external.yaml @@ -322,6 +322,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml index bfa80e007..7f92bbbaf 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml @@ -322,6 +322,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml index f666497bd..4524e51b4 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml @@ -339,6 +339,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml index 544fed03f..7fd925557 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml @@ -339,6 +339,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml index 47efe5b05..e59c043ae 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml @@ -330,6 +330,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml index f0ee44f7f..e871e5a81 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml @@ -322,6 +322,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml index eafd53633..8c4444b01 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml @@ -322,6 +322,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml index 1491d5b10..0c06373b9 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml @@ -322,6 +322,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml index 666aae1ae..f73b37d08 100644 --- a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml @@ -326,6 +326,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.17-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.17-aws-external.yaml index 40dd03244..029ceee33 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.17-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.17-aws-external.yaml @@ -327,6 +327,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.17-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.17-aws.yaml index a2df0a5d1..4643d78bd 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.17-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.17-aws.yaml @@ -327,6 +327,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-mirrors.yaml index ae4af5a39..0021eff52 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-mirrors.yaml @@ -344,6 +344,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-proxy.yaml index b29f18ec6..8c4ff3d5d 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-proxy.yaml @@ -344,6 +344,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere.yaml index e7aa0ecfd..c40382e96 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere.yaml @@ -335,6 +335,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.18-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.18-aws.yaml index 0bb90884f..47ea81655 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.18-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.18-aws.yaml @@ -327,6 +327,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml index 0121f01d2..d283eef2a 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml @@ -327,6 +327,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml index 72a1d8ccf..9271c0cdf 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml @@ -327,6 +327,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index 139d2d377..144445f9f 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -225,7 +225,7 @@ systemd: Requires=download-script.service After=download-script.service contents: | -{{ containerRuntimeHealthCheckSystemdUnit | indent 10 }} +{{ containerRuntimeHealthCheckSystemdUnit .ContainerRuntime.String | indent 10 }} - name: kubelet-healthcheck.service enabled: true @@ -537,7 +537,7 @@ coreos: Requires=download-script.service After=download-script.service content: | -{{ containerRuntimeHealthCheckSystemdUnit | indent 6 }} +{{ containerRuntimeHealthCheckSystemdUnit .ContainerRuntime.String | indent 6 }} - name: kubelet-healthcheck.service enable: true diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.17.16.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.17.16.yaml index 6ea8f78b0..9a57b726c 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.17.16.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.17.16.yaml @@ -248,6 +248,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.18.14.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.18.14.yaml index c0571b49d..88bb347ed 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.18.14.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.18.14.yaml @@ -248,6 +248,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.19.4.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.19.4.yaml index 4d26a1431..b8f74d6ca 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.19.4.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.19.4.yaml @@ -248,6 +248,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.1.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.1.yaml index 09985805c..814f4b0d9 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.1.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.1.yaml @@ -248,6 +248,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index dcaa2f0c0..216628e61 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -40,8 +40,8 @@ coreos: After=download-script.service content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] ExecStart=/opt/bin/health-monitor.sh container-runtime @@ -231,6 +231,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.17.16.json b/pkg/userdata/flatcar/testdata/ignition_v1.17.16.json index 943d8c6d5..47e7c30d0 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.17.16.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.17.16.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.17.16%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-less-then-1.19/hyperkubeImage:v1.17.16 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.17.16%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-less-then-1.19/hyperkubeImage:v1.17.16 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.18.14.json b/pkg/userdata/flatcar/testdata/ignition_v1.18.14.json index 04be7405a..3b9d39940 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.18.14.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.18.14.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.18.14%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-less-then-1.19/hyperkubeImage:v1.18.14 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.18.14%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-less-then-1.19/hyperkubeImage:v1.18.14 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.19.4.json b/pkg/userdata/flatcar/testdata/ignition_v1.19.4.json index 28f40552c..8b7e61ac5 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.19.4.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.19.4.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.19.4%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-more-then-1.19/kubeletImage:v1.19.4 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.19.4%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-more-then-1.19/kubeletImage:v1.19.4 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.20.1.json b/pkg/userdata/flatcar/testdata/ignition_v1.20.1.json index 3f4e1be81..2350caa4b 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.20.1.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.20.1.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.1%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-more-then-1.19/kubeletImage:v1.20.1 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.1%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-more-then-1.19/kubeletImage:v1.20.1 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index 108e5bb56..e07f48412 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -80,6 +80,16 @@ ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ {{ kubeletFlags .KubeletVersion .CloudProvider .Hostname .ClusterDNSIPs .IsExternal .PauseImage .InitialTaints .ExtraKubeletFlags | indent 2 }} +[Install] +WantedBy=multi-user.target` + + containerRuntimeHealthCheckSystemdUnitTpl = `[Unit] +Requires={{ .ContainerRuntime }}.service +After={{ .ContainerRuntime }}.service + +[Service] +ExecStart=/opt/bin/health-monitor.sh container-runtime + [Install] WantedBy=multi-user.target` ) @@ -87,6 +97,21 @@ WantedBy=multi-user.target` const cpFlags = `--cloud-provider=%s \ --cloud-config=/etc/kubernetes/cloud-config` +// List of allowed TLS cipher suites for kubelet +var kubeletTLSCipherSuites = []string{ + // TLS 1.3 cipher suites + "TLS_AES_128_GCM_SHA256", + "TLS_AES_256_GCM_SHA384", + "TLS_CHACHA20_POLY1305_SHA256", + // TLS 1.0 - 1.2 cipher suites + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", +} + // CloudProviderFlags returns --cloud-provider and --cloud-config flags func CloudProviderFlags(cpName string, external bool) (string, error) { if cpName == "" && !external { @@ -174,6 +199,7 @@ func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGate KubeReserved: map[string]string{"cpu": "200m", "memory": "200Mi", "ephemeral-storage": "1Gi"}, SystemReserved: map[string]string{"cpu": "200m", "memory": "200Mi", "ephemeral-storage": "1Gi"}, VolumePluginDir: "/var/lib/kubelet/volumeplugins", + TLSCipherSuites: kubeletTLSCipherSuites, } buf, err := kyaml.Marshal(cfg) @@ -254,14 +280,21 @@ WantedBy=multi-user.target } // ContainerRuntimeHealthCheckSystemdUnit container-runtime health checking systemd unit -func ContainerRuntimeHealthCheckSystemdUnit() string { - return `[Unit] -Requires=docker.service -After=docker.service +func ContainerRuntimeHealthCheckSystemdUnit(containerRuntime string) (string, error) { + tmpl, err := template.New("container-runtime-healthcheck-systemd-unit").Funcs(TxtFuncMap()).Parse(containerRuntimeHealthCheckSystemdUnitTpl) + if err != nil { + return "", fmt.Errorf("failed to parse container-runtime-healthcheck-systemd-unit template: %v", err) + } -[Service] -ExecStart=/opt/bin/health-monitor.sh container-runtime + data := struct { + ContainerRuntime string + }{ + ContainerRuntime: containerRuntime, + } -[Install] -WantedBy=multi-user.target` + var buf strings.Builder + if err = tmpl.Execute(&buf, data); err != nil { + return "", fmt.Errorf("failed to execute container-runtime-healthcheck-systemd-unit template: %w", err) + } + return buf.String(), nil } diff --git a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml index fd5a6421e..7ff1baea6 100644 --- a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml @@ -326,6 +326,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml index 0d19bead9..97453c210 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml @@ -327,6 +327,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml index 2373334aa..42a983f67 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml @@ -327,6 +327,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml index ab4754e68..77a735084 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml @@ -327,6 +327,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml index 2f29993ef..5803a68c7 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml @@ -327,6 +327,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml index 9c3280ad9..9978ca7ea 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml @@ -327,6 +327,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml index abe900f0f..203691eb3 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml @@ -345,6 +345,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml index cd4a98c6d..939fa108f 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml @@ -345,6 +345,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml index 5d0e5044a..70caacc6b 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml @@ -336,6 +336,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/sles/provider.go b/pkg/userdata/sles/provider.go index c8209004c..ae70e9e13 100644 --- a/pkg/userdata/sles/provider.go +++ b/pkg/userdata/sles/provider.go @@ -266,7 +266,7 @@ write_files: - path: /etc/systemd/system/docker-healthcheck.service permissions: "0644" content: | -{{ containerRuntimeHealthCheckSystemdUnit | indent 4 }} +{{ containerRuntimeHealthCheckSystemdUnit .ContainerRuntime.String | indent 4 }} - path: /etc/systemd/system/docker.service.d/environment.conf permissions: "0644" diff --git a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml index f2512d40b..51b57ce8e 100644 --- a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml @@ -339,6 +339,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml index b59b484dd..37a37ba89 100644 --- a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml @@ -337,6 +337,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml index 65345261a..68d141f77 100644 --- a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml @@ -339,6 +339,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml index f2ca6a03a..c2afa8429 100644 --- a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml @@ -339,6 +339,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml index 488abc728..8d395bb84 100644 --- a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml @@ -341,6 +341,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/sles/testdata/openstack.yaml b/pkg/userdata/sles/testdata/openstack.yaml index 6205f65ab..cd514b379 100644 --- a/pkg/userdata/sles/testdata/openstack.yaml +++ b/pkg/userdata/sles/testdata/openstack.yaml @@ -341,6 +341,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/sles/testdata/version-1.17.16.yaml b/pkg/userdata/sles/testdata/version-1.17.16.yaml index 1b39e68bf..6212014cc 100644 --- a/pkg/userdata/sles/testdata/version-1.17.16.yaml +++ b/pkg/userdata/sles/testdata/version-1.17.16.yaml @@ -337,6 +337,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/sles/testdata/version-1.18.14.yaml b/pkg/userdata/sles/testdata/version-1.18.14.yaml index 11f869073..da32e589c 100644 --- a/pkg/userdata/sles/testdata/version-1.18.14.yaml +++ b/pkg/userdata/sles/testdata/version-1.18.14.yaml @@ -337,6 +337,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/sles/testdata/version-1.19.4.yaml b/pkg/userdata/sles/testdata/version-1.19.4.yaml index dac8abba5..386b7cdd5 100644 --- a/pkg/userdata/sles/testdata/version-1.19.4.yaml +++ b/pkg/userdata/sles/testdata/version-1.19.4.yaml @@ -337,6 +337,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/sles/testdata/version-1.20.1.yaml b/pkg/userdata/sles/testdata/version-1.20.1.yaml index b59b484dd..37a37ba89 100644 --- a/pkg/userdata/sles/testdata/version-1.20.1.yaml +++ b/pkg/userdata/sles/testdata/version-1.20.1.yaml @@ -337,6 +337,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml index 7c838d297..c01577581 100644 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml @@ -352,6 +352,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml index 4a7734f1f..b6c5418e9 100644 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/sles/testdata/vsphere-proxy.yaml @@ -352,6 +352,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml index ca099709c..c2f9a381b 100644 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ b/pkg/userdata/sles/testdata/vsphere.yaml @@ -342,6 +342,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index dd462c871..29a51e884 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -412,6 +412,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 6775c4227..55b3479ff 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -395,6 +395,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index 2d8c72f21..8ecc20d08 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -393,6 +393,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index c4c286d50..9174901cc 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -395,6 +395,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 736d42862..d652ec4d3 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -395,6 +395,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index 8da93a43e..efed76eb8 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -397,6 +397,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 826b030e7..b2a2b0918 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -397,6 +397,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/ubuntu/testdata/version-1.17.16.yaml b/pkg/userdata/ubuntu/testdata/version-1.17.16.yaml index 6a170b46a..93b489504 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.17.16.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.17.16.yaml @@ -393,6 +393,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/ubuntu/testdata/version-1.18.14.yaml b/pkg/userdata/ubuntu/testdata/version-1.18.14.yaml index afaf93c84..f61a7788b 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.18.14.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.18.14.yaml @@ -393,6 +393,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/ubuntu/testdata/version-1.19.4.yaml b/pkg/userdata/ubuntu/testdata/version-1.19.4.yaml index 0950810d5..5bfc31cd7 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.19.4.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.19.4.yaml @@ -393,6 +393,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/ubuntu/testdata/version-1.20.1.yaml b/pkg/userdata/ubuntu/testdata/version-1.20.1.yaml index 7d0c206b2..2c09702eb 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.20.1.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.20.1.yaml @@ -393,6 +393,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index 6682efc38..e388d76c3 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -408,6 +408,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 9fbe91618..3e22bb8b8 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -408,6 +408,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 0009ad2f2..7760df17f 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -398,6 +398,16 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 volumePluginDir: /var/lib/kubelet/volumeplugins volumeStatsAggPeriod: 0s diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index bc6fe278d..d956623b6 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -219,7 +219,7 @@ func buildScenarios() []scenario { all = append(all, scenario{ name: "migrateUID", - containerRuntime: "docker", + containerRuntime: "containerd", osName: "ubuntu", executor: verifyMigrateUID, }) diff --git a/test/tools/integration/Makefile b/test/tools/integration/Makefile index 77116484a..c35759c48 100644 --- a/test/tools/integration/Makefile +++ b/test/tools/integration/Makefile @@ -22,14 +22,12 @@ export PATH := $(shell pwd):$(PATH) ifeq ($(MAKECMDGOALS),apply) EXTRA_ARG = -auto-approve -else ifeq ($(MAKECMDGOALS),destroy) - EXTRA_ARG = -force endif .PHONY: terraform terraform: @if ! which terraform; then \ - curl https://releases.hashicorp.com/terraform/0.13.5/terraform_0.13.5_linux_amd64.zip \ + curl https://releases.hashicorp.com/terraform/1.0.9/terraform_1.0.9_linux_amd64.zip \ --retry 5 \ -o /tmp/terraform.zip && \ unzip -n /tmp/terraform.zip terraform && \ diff --git a/test/tools/integration/master_install_script.sh b/test/tools/integration/master_install_script.sh index fb44dc534..5cdb88336 100644 --- a/test/tools/integration/master_install_script.sh +++ b/test/tools/integration/master_install_script.sh @@ -159,4 +159,4 @@ done echo "Error: machine-controller didn't come up within 100 seconds!" echo "Logs:" kubectl logs -n kube-system $(kubectl get pods -n kube-system|egrep '^machine-controller'|awk '{ print $1}') -exit 1 +exit 1 \ No newline at end of file From 89f5a9b3bc848e875d6c8859621556512a13ad04 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 2 Nov 2021 00:19:38 +0100 Subject: [PATCH 012/489] enable e2e tests for vSphere cloud provider (#1088) Signed-off-by: Moath Qasim --- test/e2e/provisioning/all_e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 21c5c2258..dbf8520be 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -766,7 +766,7 @@ func getVSphereTestParams(t *testing.T) []string { func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() - selector := Not(OsSelector("sles", "amzn2", "rhel")) + selector := Not(OsSelector("sles", "amzn2")) params := getVSphereTestParams(t) runScenarios(t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) From 432037016d3c2781a36973497b2381f25c84ddfc Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Tue, 2 Nov 2021 13:36:45 +0100 Subject: [PATCH 013/489] Use CentOS 7 images issued by CentOS CPE team as default (#1089) Signed-off-by: Marvin Beckers --- pkg/cloudprovider/provider/aws/provider.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 9b7e50fe7..8a5c6a2e4 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -96,14 +96,18 @@ var ( ) amiFilters = map[providerconfigtypes.OperatingSystem]map[awstypes.CPUArchitecture]amiFilter{ + // Source: https://wiki.centos.org/Cloud/AWS providerconfigtypes.OperatingSystemCentOS: { awstypes.CPUArchitectureX86_64: { - description: "CentOS Linux 7 x86_64 HVM EBS*", - // The AWS marketplace ID from AWS - owner: "679593333241", - productCode: "aw0evgkw8e5c1q413zgy5pjce", + description: "CentOS 7* x86_64", + // The AWS marketplace ID from CentOS Community Platform Engineering (CPE) + owner: "125523088429", + }, + awstypes.CPUArchitectureARM64: { + description: "CentOS 7* aarch64", + // The AWS marketplace ID from CentOS Community Platform Engineering (CPE) + owner: "125523088429", }, - // 2021-10-14 - No CentOS 7 ARM64 image available under legacy product code }, providerconfigtypes.OperatingSystemAmazonLinux2: { awstypes.CPUArchitectureX86_64: { From 3b83d7727b3b7a0bead2cd033872a0ffeeac1560 Mon Sep 17 00:00:00 2001 From: Florin Peter <2911849+FlorinPeter@users.noreply.github.com> Date: Tue, 2 Nov 2021 18:16:45 +0100 Subject: [PATCH 014/489] fix failing download script on flatcar due to read-only filesystem (#1066) Signed-off-by: Florin Peter Co-authored-by: Florin Peter --- pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.17-aws-external.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.17-aws-external.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.17-aws.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-mirrors.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-proxy.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.17-vsphere.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.18-aws.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml | 2 +- pkg/userdata/flatcar/testdata/cloud-init_v1.17.16.yaml | 2 +- pkg/userdata/flatcar/testdata/cloud-init_v1.18.14.yaml | 2 +- pkg/userdata/flatcar/testdata/cloud-init_v1.19.4.yaml | 2 +- pkg/userdata/flatcar/testdata/cloud-init_v1.20.1.yaml | 2 +- pkg/userdata/flatcar/testdata/containerd.yaml | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.17.16.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.18.14.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.19.4.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.20.1.json | 2 +- pkg/userdata/helper/download_binaries_script.go | 2 +- .../helper/testdata/safe_download_binaries_v1.20.1.golden | 2 +- pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml | 2 +- pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml | 2 +- .../sles/testdata/kubelet-version-without-v-prefix.yaml | 2 +- pkg/userdata/sles/testdata/multiple-dns-servers.yaml | 2 +- pkg/userdata/sles/testdata/multiple-ssh-keys.yaml | 2 +- .../sles/testdata/openstack-overwrite-cloud-config.yaml | 2 +- pkg/userdata/sles/testdata/openstack.yaml | 2 +- pkg/userdata/sles/testdata/version-1.17.16.yaml | 2 +- pkg/userdata/sles/testdata/version-1.18.14.yaml | 2 +- pkg/userdata/sles/testdata/version-1.19.4.yaml | 2 +- pkg/userdata/sles/testdata/version-1.20.1.yaml | 2 +- pkg/userdata/sles/testdata/vsphere-mirrors.yaml | 2 +- pkg/userdata/sles/testdata/vsphere-proxy.yaml | 2 +- pkg/userdata/sles/testdata/vsphere.yaml | 2 +- pkg/userdata/ubuntu/testdata/containerd.yaml | 2 +- pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml | 2 +- .../ubuntu/testdata/kubelet-version-without-v-prefix.yaml | 2 +- pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml | 2 +- pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml | 2 +- .../ubuntu/testdata/openstack-overwrite-cloud-config.yaml | 2 +- pkg/userdata/ubuntu/testdata/openstack.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.17.16.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.18.14.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.19.4.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.20.1.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere.yaml | 2 +- 66 files changed, 66 insertions(+), 66 deletions(-) diff --git a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml index 5063d7ae3..c942b97ac 100644 --- a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml @@ -141,7 +141,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml index ed528241f..231b4f456 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml @@ -137,7 +137,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.16.16}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws-external.yaml index 6158cb09f..ee92a26bc 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws-external.yaml @@ -138,7 +138,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml index 7f92bbbaf..30c2bc6d3 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml @@ -138,7 +138,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml index 4524e51b4..e036edb8d 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml @@ -151,7 +151,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml index 7fd925557..bce80e1c9 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml @@ -151,7 +151,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml index e59c043ae..026b0874c 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml @@ -143,7 +143,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml index e871e5a81..7ccb62d5f 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml @@ -138,7 +138,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.18.14}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml index 8c4444b01..9b8931534 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml @@ -138,7 +138,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.19.4}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml index 0c06373b9..379798b1f 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml @@ -138,7 +138,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml index f73b37d08..3e1e757cd 100644 --- a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml @@ -142,7 +142,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.17-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.17-aws-external.yaml index 029ceee33..9ab965a77 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.17-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.17-aws-external.yaml @@ -143,7 +143,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.17-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.17-aws.yaml index 4643d78bd..af66d24d1 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.17-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.17-aws.yaml @@ -143,7 +143,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-mirrors.yaml index 0021eff52..23792e571 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-mirrors.yaml @@ -156,7 +156,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-proxy.yaml index 8c4ff3d5d..1f4b151cf 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere-proxy.yaml @@ -156,7 +156,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere.yaml index c40382e96..8537deb29 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.17-vsphere.yaml @@ -148,7 +148,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.18-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.18-aws.yaml index 47ea81655..ad483335f 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.18-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.18-aws.yaml @@ -143,7 +143,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.18.14}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml index d283eef2a..3ce9046d7 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml @@ -143,7 +143,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.19.4}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml index 9271c0cdf..aa50373f1 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml @@ -143,7 +143,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.17.16.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.17.16.yaml index 9a57b726c..880d23760 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.17.16.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.17.16.yaml @@ -448,7 +448,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.18.14.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.18.14.yaml index 88bb347ed..673fc5ab2 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.18.14.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.18.14.yaml @@ -448,7 +448,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.18.14}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.19.4.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.19.4.yaml index b8f74d6ca..2d4cac1c9 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.19.4.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.19.4.yaml @@ -448,7 +448,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.19.4}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.1.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.1.yaml index 814f4b0d9..9d77e733a 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.1.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.1.yaml @@ -448,7 +448,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index 216628e61..2645bb80a 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -431,7 +431,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.17.16.json b/pkg/userdata/flatcar/testdata/ignition_v1.17.16.json index 47e7c30d0..90054cb35 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.17.16.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.17.16.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.17.16%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-less-then-1.19/hyperkubeImage:v1.17.16 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.17.16%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-less-then-1.19/hyperkubeImage:v1.17.16 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.18.14.json b/pkg/userdata/flatcar/testdata/ignition_v1.18.14.json index 3b9d39940..bf13aad95 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.18.14.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.18.14.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.18.14%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-less-then-1.19/hyperkubeImage:v1.18.14 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.18.14%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-less-then-1.19/hyperkubeImage:v1.18.14 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.19.4.json b/pkg/userdata/flatcar/testdata/ignition_v1.19.4.json index 8b7e61ac5..33d8fbc2c 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.19.4.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.19.4.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.19.4%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-more-then-1.19/kubeletImage:v1.19.4 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.19.4%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-more-then-1.19/kubeletImage:v1.19.4 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.20.1.json b/pkg/userdata/flatcar/testdata/ignition_v1.20.1.json index 2350caa4b..42bad48ef 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.20.1.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.20.1.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.1%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-more-then-1.19/kubeletImage:v1.20.1 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.1%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"enabled":true,"name":"docker.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=docker.service\nAfter=docker.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh container-runtime\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"docker-healthcheck.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-docker.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nDescription=Kubernetes Kubelet\nRequires=docker.service\nAfter=docker.service\n[Service]\nTimeoutStartSec=5min\nCPUAccounting=true\nMemoryAccounting=true\nEnvironmentFile=-/etc/environment\nEnvironmentFile=/etc/kubernetes/nodeip.conf\nEnvironment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStartPre=/bin/mkdir -p /var/lib/calico\nExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests\nExecStartPre=/bin/mkdir -p /etc/cni/net.d\nExecStartPre=/bin/mkdir -p /opt/cni/bin\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/sh -c '/usr/bin/env \u003e /tmp/environment'\nExecStart=/usr/bin/docker run --name %n \\\n --rm --tty --restart no \\\n --network host \\\n --pid host \\\n --env-file /tmp/environment \\\n --privileged \\\n --cgroup-parent system.slice \\\n --entrypoint kubelet \\\n -v /dev:/dev \\\n -v /etc/cni/net.d:/etc/cni/net.d \\\n -v /etc/kubernetes:/etc/kubernetes \\\n -v /etc/machine-id:/etc/machine-id:ro \\\n -v /etc/os-release:/etc/os-release:ro \\\n -v /etc/resolv.conf:/etc/resolv.conf:ro \\\n -v /lib/modules:/lib/modules \\\n -v /mnt:/mnt:rshared \\\n -v /opt/cni/bin:/opt/cni/bin:ro \\\n -v /run:/run \\\n -v /sys:/sys \\\n -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \\\n -v /var/lib/calico:/var/lib/calico:ro \\\n -v /var/lib/cni:/var/lib/cni \\\n -v /var/lib/docker:/var/lib/docker \\\n -v /var/lib/kubelet:/var/lib/kubelet:rshared \\\n -v /var/log/pods:/var/log/pods \\\n for-kubernetes-more-then-1.19/kubeletImage:v1.20.1 \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\nExecStop=-/usr/bin/docker stop %n\nRestart=always\nRestartSec=10\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"kubelet.service"},{"dropins":[{"contents":"[Service]\nEnvironmentFile=-/etc/environment\n","name":"10-environment.conf"}],"enabled":true,"name":"docker.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/download_binaries_script.go b/pkg/userdata/helper/download_binaries_script.go index 4e5b74a0e..358ceb3e1 100644 --- a/pkg/userdata/helper/download_binaries_script.go +++ b/pkg/userdata/helper/download_binaries_script.go @@ -89,7 +89,7 @@ sha256sum -c <<<"$cri_tools_sum" {{- /* unpack cri-tools and symlink to path so it's available to all users */}} tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" -ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl +ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - {{- /* kubelet */}} diff --git a/pkg/userdata/helper/testdata/safe_download_binaries_v1.20.1.golden b/pkg/userdata/helper/testdata/safe_download_binaries_v1.20.1.golden index a7ea86f83..c8465e026 100644 --- a/pkg/userdata/helper/testdata/safe_download_binaries_v1.20.1.golden +++ b/pkg/userdata/helper/testdata/safe_download_binaries_v1.20.1.golden @@ -37,7 +37,7 @@ cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" -ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl +ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml index 7ff1baea6..3a336c5e5 100644 --- a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml @@ -142,7 +142,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml index 97453c210..4740a7e7d 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.17-aws.yaml @@ -143,7 +143,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml index 42a983f67..d96de8228 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.18-aws.yaml @@ -143,7 +143,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.18.14}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml index 77a735084..a3674f9b3 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml @@ -143,7 +143,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.19.4}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml index 5803a68c7..b2fe4f8f8 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws-external.yaml @@ -143,7 +143,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml index 9978ca7ea..951ea16b7 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml @@ -143,7 +143,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml index 203691eb3..ff439e480 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-mirrors.yaml @@ -157,7 +157,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml index 939fa108f..ea1f409b0 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere-proxy.yaml @@ -157,7 +157,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml index 70caacc6b..0072de1da 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-vsphere.yaml @@ -149,7 +149,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml index 51b57ce8e..271014bad 100644 --- a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml @@ -104,7 +104,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml index 37a37ba89..7d5eac0fa 100644 --- a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml @@ -102,7 +102,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml index 68d141f77..eae6bc4a0 100644 --- a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml @@ -102,7 +102,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml index c2afa8429..ec49fc6cd 100644 --- a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml @@ -104,7 +104,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml index 8d395bb84..0cb53cf81 100644 --- a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml @@ -102,7 +102,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/sles/testdata/openstack.yaml b/pkg/userdata/sles/testdata/openstack.yaml index cd514b379..ba5b0d36f 100644 --- a/pkg/userdata/sles/testdata/openstack.yaml +++ b/pkg/userdata/sles/testdata/openstack.yaml @@ -102,7 +102,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/sles/testdata/version-1.17.16.yaml b/pkg/userdata/sles/testdata/version-1.17.16.yaml index 6212014cc..f3a48fac3 100644 --- a/pkg/userdata/sles/testdata/version-1.17.16.yaml +++ b/pkg/userdata/sles/testdata/version-1.17.16.yaml @@ -102,7 +102,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/sles/testdata/version-1.18.14.yaml b/pkg/userdata/sles/testdata/version-1.18.14.yaml index da32e589c..beef3fa82 100644 --- a/pkg/userdata/sles/testdata/version-1.18.14.yaml +++ b/pkg/userdata/sles/testdata/version-1.18.14.yaml @@ -102,7 +102,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.18.14}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/sles/testdata/version-1.19.4.yaml b/pkg/userdata/sles/testdata/version-1.19.4.yaml index 386b7cdd5..a953509fd 100644 --- a/pkg/userdata/sles/testdata/version-1.19.4.yaml +++ b/pkg/userdata/sles/testdata/version-1.19.4.yaml @@ -102,7 +102,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.19.4}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/sles/testdata/version-1.20.1.yaml b/pkg/userdata/sles/testdata/version-1.20.1.yaml index 37a37ba89..7d5eac0fa 100644 --- a/pkg/userdata/sles/testdata/version-1.20.1.yaml +++ b/pkg/userdata/sles/testdata/version-1.20.1.yaml @@ -102,7 +102,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml index c01577581..7a440b07c 100644 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml @@ -112,7 +112,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml index b6c5418e9..ab3b64a30 100644 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/sles/testdata/vsphere-proxy.yaml @@ -112,7 +112,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml index c2f9a381b..ba4f9a2ab 100644 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ b/pkg/userdata/sles/testdata/vsphere.yaml @@ -103,7 +103,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index 29a51e884..cf87902d4 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -151,7 +151,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 55b3479ff..2a547db4f 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -151,7 +151,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index 8ecc20d08..7b8267594 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -149,7 +149,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.3}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 9174901cc..c0b1d399b 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -149,7 +149,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index d652ec4d3..40746fffb 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -151,7 +151,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.3}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index efed76eb8..c46117936 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -149,7 +149,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.3}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index b2a2b0918..a425b8750 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -149,7 +149,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/ubuntu/testdata/version-1.17.16.yaml b/pkg/userdata/ubuntu/testdata/version-1.17.16.yaml index 93b489504..b0fe5f044 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.17.16.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.17.16.yaml @@ -149,7 +149,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/ubuntu/testdata/version-1.18.14.yaml b/pkg/userdata/ubuntu/testdata/version-1.18.14.yaml index f61a7788b..e8ee40b88 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.18.14.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.18.14.yaml @@ -149,7 +149,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.18.14}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/ubuntu/testdata/version-1.19.4.yaml b/pkg/userdata/ubuntu/testdata/version-1.19.4.yaml index 5bfc31cd7..bf0c7c014 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.19.4.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.19.4.yaml @@ -149,7 +149,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.19.4}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/ubuntu/testdata/version-1.20.1.yaml b/pkg/userdata/ubuntu/testdata/version-1.20.1.yaml index 2c09702eb..4d4a552a2 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.20.1.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.20.1.yaml @@ -149,7 +149,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index e388d76c3..db6afef78 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -159,7 +159,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.3}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 3e22bb8b8..4f289d5e7 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -159,7 +159,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.3}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 7760df17f..2ed5b95ed 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -150,7 +150,7 @@ write_files: sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - KUBE_VERSION="${KUBE_VERSION:-v1.17.3}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" From 4a78bd9685e93b0b7490611602dd5b496cfae27e Mon Sep 17 00:00:00 2001 From: Simon Wessel <9195792+simon-wessel@users.noreply.github.com> Date: Sat, 6 Nov 2021 14:28:48 +0100 Subject: [PATCH 015/489] Add support for AWS AssumeRole functionality (#1090) Signed-off-by: Simon Wessel --- .prow.yaml | 22 ++++ pkg/cloudprovider/provider/aws/provider.go | 116 +++++++++++++----- pkg/cloudprovider/provider/aws/types/types.go | 3 + test/e2e/provisioning/all_e2e_test.go | 30 +++++ test/e2e/provisioning/helper.go | 4 + .../testdata/machinedeployment-aws.yaml | 2 + 6 files changed, 147 insertions(+), 30 deletions(-) diff --git a/.prow.yaml b/.prow.yaml index c1bdfbd91..5eb765939 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -834,6 +834,28 @@ presubmits: memory: 1Gi cpu: 500m + - name: pull-machine-controller-e2e-aws-assume-role + always_run: false + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-aws-assume-role: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.17.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestAWSAssumeRoleProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + postsubmits: - name: ci-push-machine-controller-image always_run: true diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 8a5c6a2e4..ec2778c35 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -31,6 +31,7 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/iam" + "github.com/aws/aws-sdk-go/service/sts" gocache "github.com/patrickmn/go-cache" "github.com/prometheus/client_golang/prometheus" @@ -203,6 +204,9 @@ type Config struct { SpotMaxPrice *string SpotPersistentRequest *bool SpotInterruptionBehavior *string + + AssumeRoleARN string + AssumeRoleExternalID string } type amiFilter struct { @@ -447,28 +451,76 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt } c.SpotInterruptionBehavior = pointer.StringPtr(interruptionBehavior) } + assumeRoleARN, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AssumeRoleARN, "AWS_ASSUME_ROLE_ARN") + if err != nil { + return nil, nil, nil, err + } + c.AssumeRoleARN = assumeRoleARN + assumeRoleExternalID, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AssumeRoleExternalID, "AWS_ASSUME_ROLE_EXTERNAL_ID") + if err != nil { + return nil, nil, nil, err + } + c.AssumeRoleExternalID = assumeRoleExternalID return &c, &pconfig, &rawConfig, err } -func getSession(id, secret, token, region string) (*session.Session, error) { +func getSession(id, secret, token, region, assumeRoleARN, assumeRoleExternalID string) (*session.Session, error) { config := aws.NewConfig() config = config.WithRegion(region) config = config.WithCredentials(credentials.NewStaticCredentials(id, secret, token)) config = config.WithMaxRetries(maxRetries) - return session.NewSession(config) + awsSession, err := session.NewSession(config) + if err != nil { + return nil, fmt.Errorf("failed to create AWS session: %v", err) + } + + // Assume IAM role of e.g. external AWS account if configured + if assumeRoleARN != "" { + awsSession, err = getAssumeRoleSession(awsSession, assumeRoleARN, assumeRoleExternalID, region) + if err != nil { + return nil, fmt.Errorf("failed to create temporary AWS session for assumed role: %v", err) + } + } + + return awsSession, err +} + +func getAssumeRoleSession(awsSession *session.Session, assumeRoleARN, assumeRoleExternalID, region string) (*session.Session, error) { + assumeRoleOutput, err := getAssumeRoleCredentials(awsSession, assumeRoleARN, assumeRoleExternalID) + if err != nil { + return nil, awsErrorToTerminalError(err, "unable to initialize aws external id session") + } + + assumedRoleConfig := aws.NewConfig() + assumedRoleConfig = assumedRoleConfig.WithRegion(region) + assumedRoleConfig = assumedRoleConfig.WithCredentials(credentials.NewStaticCredentials(*assumeRoleOutput.Credentials.AccessKeyId, + *assumeRoleOutput.Credentials.SecretAccessKey, + *assumeRoleOutput.Credentials.SessionToken)) + assumedRoleConfig = assumedRoleConfig.WithMaxRetries(maxRetries) + return session.NewSession(assumedRoleConfig) +} + +func getAssumeRoleCredentials(session *session.Session, assumeRoleARN, assumeRoleExternalID string) (*sts.AssumeRoleOutput, error) { + stsSession := sts.New(session) + sessionName := "kubermatic-machine-controller" + return stsSession.AssumeRole(&sts.AssumeRoleInput{ + ExternalId: &assumeRoleExternalID, + RoleArn: &assumeRoleARN, + RoleSessionName: &sessionName, + }) } -func getIAMclient(id, secret, region string) (*iam.IAM, error) { - sess, err := getSession(id, secret, "", region) +func getIAMclient(id, secret, region, assumeRoleArn, assumeRoleExternalID string) (*iam.IAM, error) { + sess, err := getSession(id, secret, "", region, assumeRoleArn, assumeRoleExternalID) if err != nil { return nil, awsErrorToTerminalError(err, "failed to get aws session") } return iam.New(sess), nil } -func getEC2client(id, secret, region string) (*ec2.EC2, error) { - sess, err := getSession(id, secret, "", region) +func getEC2client(id, secret, region, assumeRoleArn, assumeRoleExternalID string) (*ec2.EC2, error) { + sess, err := getSession(id, secret, "", region, assumeRoleArn, assumeRoleExternalID) if err != nil { return nil, awsErrorToTerminalError(err, "failed to get aws session") } @@ -520,7 +572,7 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { return fmt.Errorf("diskSize must be specified and > 0") } - ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region) + ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { return fmt.Errorf("failed to create ec2 client: %v", err) } @@ -557,7 +609,7 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { return fmt.Errorf("failed to validate security group id's: %v", err) } - iamClient, err := getIAMclient(config.AccessKeyID, config.SecretAccessKey, config.Region) + iamClient, err := getIAMclient(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { return fmt.Errorf("failed to create iam client: %v", err) } @@ -599,7 +651,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr } } - ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region) + ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { return nil, err } @@ -757,7 +809,7 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, _ *cloudprovidertypes.Prov } } - ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region) + ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { return false, err } @@ -805,7 +857,7 @@ func (p *provider) get(machine *v1alpha1.Machine) (*awsInstance, error) { } } - ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region) + ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { return nil, err } @@ -881,7 +933,7 @@ func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]s } func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { - instance, err := p.get(machine) + machineInstance, err := p.get(machine) if err != nil { if err == cloudprovidererrors.ErrInstanceNotFound { return nil @@ -897,13 +949,13 @@ func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { } } - ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region) + ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { return fmt.Errorf("failed to get EC2 client: %v", err) } _, err = ec2Client.CreateTags(&ec2.CreateTagsInput{ - Resources: aws.StringSlice([]string{instance.ID()}), + Resources: aws.StringSlice([]string{machineInstance.ID()}), Tags: []*ec2.Tag{{Key: aws.String(machineUIDTag), Value: aws.String(string(new))}}}) if err != nil { return fmt.Errorf("failed to update instance with new machineUIDTag: %v", err) @@ -1032,38 +1084,42 @@ func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { } type ec2Credentials struct { - acccessKeyID string - secretAccessKey string - region string + acccessKeyID string + secretAccessKey string + region string + assumeRoleARN string + assumeRoleExternalID string } - var errors []error - credentials := map[string]ec2Credentials{} + var machineErrors []error + machineEc2Credentials := map[string]ec2Credentials{} for _, machine := range machines.Items { config, _, _, err := p.getConfig(machines.Items[0].Spec.ProviderSpec) if err != nil { - errors = append(errors, fmt.Errorf("failed to parse MachineSpec of machine %s/%s, due to %v", machine.Namespace, machine.Name, err)) + machineErrors = append(machineErrors, fmt.Errorf("failed to parse MachineSpec of machine %s/%s, due to %v", machine.Namespace, machine.Name, err)) continue } // Very simple and very stupid - credentials[fmt.Sprintf("%s/%s/%s", config.AccessKeyID, config.SecretAccessKey, config.Region)] = ec2Credentials{ - acccessKeyID: config.AccessKeyID, - secretAccessKey: config.SecretAccessKey, - region: config.Region, + machineEc2Credentials[fmt.Sprintf("%s/%s/%s/%s/%s", config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID)] = ec2Credentials{ + acccessKeyID: config.AccessKeyID, + secretAccessKey: config.SecretAccessKey, + region: config.Region, + assumeRoleARN: config.AssumeRoleARN, + assumeRoleExternalID: config.AssumeRoleExternalID, } } allReservations := []*ec2.Reservation{} - for _, cred := range credentials { - ec2Client, err := getEC2client(cred.acccessKeyID, cred.secretAccessKey, cred.region) + for _, cred := range machineEc2Credentials { + ec2Client, err := getEC2client(cred.acccessKeyID, cred.secretAccessKey, cred.region, cred.assumeRoleARN, cred.assumeRoleExternalID) if err != nil { - errors = append(errors, fmt.Errorf("failed to get EC2 client: %v", err)) + machineErrors = append(machineErrors, fmt.Errorf("failed to get EC2 client: %v", err)) continue } inOut, err := ec2Client.DescribeInstances(&ec2.DescribeInstancesInput{}) if err != nil { - errors = append(errors, fmt.Errorf("failed to get EC2 instances: %v", err)) + machineErrors = append(machineErrors, fmt.Errorf("failed to get EC2 instances: %v", err)) continue } allReservations = append(allReservations, inOut.Reservations...) @@ -1074,8 +1130,8 @@ func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { getIntanceCountForMachine(machine, allReservations)) } - if len(errors) > 0 { - return fmt.Errorf("errors: %v", errors) + if len(machineErrors) > 0 { + return fmt.Errorf("errors: %v", machineErrors) } return nil diff --git a/pkg/cloudprovider/provider/aws/types/types.go b/pkg/cloudprovider/provider/aws/types/types.go index 75565c078..8f2072325 100644 --- a/pkg/cloudprovider/provider/aws/types/types.go +++ b/pkg/cloudprovider/provider/aws/types/types.go @@ -24,6 +24,9 @@ type RawConfig struct { AccessKeyID providerconfigtypes.ConfigVarString `json:"accessKeyId,omitempty"` SecretAccessKey providerconfigtypes.ConfigVarString `json:"secretAccessKey,omitempty"` + AssumeRoleARN providerconfigtypes.ConfigVarString `json:"assumeRoleARN,omitempty"` + AssumeRoleExternalID providerconfigtypes.ConfigVarString `json:"assumeRoleExternalID,omitempty"` + Region providerconfigtypes.ConfigVarString `json:"region"` AvailabilityZone providerconfigtypes.ConfigVarString `json:"availabilityZone,omitempty"` VpcID providerconfigtypes.ConfigVarString `json:"vpcId"` diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index dbf8520be..a5bce6dc6 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -388,6 +388,36 @@ func TestAWSProvisioningE2E(t *testing.T) { runScenarios(t, selector, params, AWSManifest, fmt.Sprintf("aws-%s", *testRunIdentifier)) } +// TestAWSAssumeRoleProvisioning - a test suite that exercises AWS provider +// by requesting nodes using an assumed role. +func TestAWSAssumeRoleProvisioningE2E(t *testing.T) { + t.Parallel() + + // test data + awsKeyID := os.Getenv("AWS_E2E_TESTS_KEY_ID") + awsSecret := os.Getenv("AWS_E2E_TESTS_SECRET") + awsAssumeRoleARN := os.Getenv("AWS_ASSUME_ROLE_ARN") + awsAssumeRoleExternalID := os.Getenv("AWS_ASSUME_ROLE_EXTERNAL_ID") + if len(awsKeyID) == 0 || len(awsSecret) == 0 || len(awsAssumeRoleARN) == 0 || len(awsAssumeRoleExternalID) == 0 { + t.Fatal("unable to run the test suite, environment variables AWS_E2E_TESTS_KEY_ID, AWS_E2E_TESTS_SECRET, AWS_E2E_ASSUME_ROLE_ARN and AWS_E2E_ASSUME_ROLE_EXTERNAL_ID cannot be empty") + } + + // act + params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), + fmt.Sprintf("<< AWS_SECRET_ACCESS_KEY >>=%s", awsSecret), + fmt.Sprintf("<< PROVISIONING_UTILITY >>=%s", flatcar.CloudInit), + } + + scenario := scenario{ + name: "AWS with AssumeRole", + osName: "ubuntu", + containerRuntime: "docker", + kubernetesVersion: "1.19.9", + executor: verifyCreateAndDelete, + } + testScenario(t, scenario, *testRunIdentifier, params, AWSManifest, false) +} + // TestAWSSpotInstanceProvisioning - a test suite that exercises AWS provider // by requesting spot nodes with different combination of container runtime type, container runtime version and the OS flavour. func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index d956623b6..67d882bb2 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -174,6 +174,10 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.02")) } + // only used by assume role scenario, otherwise empty (disabled) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< AWS_ASSUME_ROLE_ARN >>=%s", os.Getenv("AWS_ASSUME_ROLE_ARN"))) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< AWS_ASSUME_ROLE_EXTERNAL_ID >>=%s", os.Getenv("AWS_ASSUME_ROLE_EXTERNAL_ID"))) + // only used by OpenStack scenarios scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_IMAGE >>=%s", openStackImages[testCase.osName])) diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws.yaml index dad7489b6..93e050d1f 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws.yaml @@ -26,6 +26,8 @@ spec: cloudProviderSpec: accessKeyId: << AWS_ACCESS_KEY_ID >> secretAccessKey: << AWS_SECRET_ACCESS_KEY >> + assumeRoleARN: "<< AWS_ASSUME_ROLE_ARN >>" + assumeRoleExternalID: "<< AWS_ASSUME_ROLE_EXTERNAL_ID >>" region: "eu-central-1" availabilityZone: "eu-central-1a" vpcId: "vpc-819f62e9" From 8eeec9bd2fd829ddd2b8a649ed0a8d97cd171fa1 Mon Sep 17 00:00:00 2001 From: Happy2C0de <46957159+Happy2C0de@users.noreply.github.com> Date: Sat, 6 Nov 2021 15:57:48 +0100 Subject: [PATCH 016/489] Change getImageByName to use the Image client (#1085) Signed-off-by: Happy2C0de <46957159+Happy2C0de@users.noreply.github.com> --- .../provider/openstack/helper.go | 10 +-- .../provider/openstack/provider.go | 20 ++++- .../provider/openstack/provider_test.go | 76 +++++++++++++------ 3 files changed, 73 insertions(+), 33 deletions(-) diff --git a/pkg/cloudprovider/provider/openstack/helper.go b/pkg/cloudprovider/provider/openstack/helper.go index 957e0f57c..05901f1bc 100644 --- a/pkg/cloudprovider/provider/openstack/helper.go +++ b/pkg/cloudprovider/provider/openstack/helper.go @@ -27,8 +27,8 @@ import ( goopenstack "github.com/gophercloud/gophercloud/openstack" osavailabilityzones "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/availabilityzones" osflavors "github.com/gophercloud/gophercloud/openstack/compute/v2/flavors" - osimages "github.com/gophercloud/gophercloud/openstack/compute/v2/images" osregions "github.com/gophercloud/gophercloud/openstack/identity/v3/regions" + osimagesv2 "github.com/gophercloud/gophercloud/openstack/imageservice/v2/images" osfloatingips "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" ossecuritygroups "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/groups" osecruritygrouprules "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/security/rules" @@ -120,11 +120,11 @@ func getAvailabilityZone(computeClient *gophercloud.ServiceClient, c *Config) (* return nil, errNotFound } -func getImageByName(computeClient *gophercloud.ServiceClient, c *Config) (*osimages.Image, error) { - var allImages []osimages.Image - pager := osimages.ListDetail(computeClient, osimages.ListOpts{Name: c.Image}) +func getImageByName(imageClient *gophercloud.ServiceClient, c *Config) (*osimagesv2.Image, error) { + var allImages []osimagesv2.Image + pager := osimagesv2.List(imageClient, osimagesv2.ListOpts{Name: c.Image}) err := pager.EachPage(func(page pagination.Page) (bool, error) { - images, err := osimages.ExtractImages(page) + images, err := osimagesv2.ExtractImages(page) if err != nil { return false, err } diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 47929c6d4..d96225818 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -445,14 +445,20 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { return fmt.Errorf("failed to get compute client: %v", err) } - image, err := getImageByName(computeClient, c) + // Get OS Image Client + imageClient, err := goopenstack.NewImageServiceV2(client, gophercloud.EndpointOpts{Region: c.Region}) + if err != nil { + return fmt.Errorf("failed to get image client: %v", err) + } + + image, err := getImageByName(imageClient, c) if err != nil { return fmt.Errorf("failed to get image %q: %v", c.Image, err) } if c.RootDiskSizeGB != nil { - if *c.RootDiskSizeGB < image.MinDisk { + if *c.RootDiskSizeGB < image.MinDiskGigabytes { return fmt.Errorf("rootDiskSize %d is smaller than minimum disk size for image %q(%d)", - *c.RootDiskSizeGB, image.Name, image.MinDisk) + *c.RootDiskSizeGB, image.Name, image.MinDiskGigabytes) } } @@ -526,7 +532,13 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr return nil, osErrorToTerminalError(err, fmt.Sprintf("failed to get flavor %s", c.Flavor)) } - image, err := getImageByName(computeClient, c) + // Get OS Image Client + imageClient, err := goopenstack.NewImageServiceV2(client, gophercloud.EndpointOpts{Region: c.Region}) + if err != nil { + return nil, osErrorToTerminalError(err, "failed to get a image client") + } + + image, err := getImageByName(imageClient, c) if err != nil { return nil, osErrorToTerminalError(err, fmt.Sprintf("failed to get image %s", c.Image)) } diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index 5c60d1729..72e3c8559 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -43,7 +43,7 @@ const expectedServerRequest = `{ "server": { "availability_zone": "eu-de-01", "flavorRef": "1", - "imageRef": "f3e4a95d-1f4f-4989-97ce-f3a1fb8c04d7", + "imageRef": "1bea47ed-f6a9-463b-b423-14b9cca9ad27", "metadata": { "kubernetes-cluster": "xyz", "machine-uid": "", @@ -75,7 +75,7 @@ const expectedBlockDeviceBootRequest = `{ "delete_on_termination": true, "destination_type": "volume", "source_type": "image", - "uuid": "f3e4a95d-1f4f-4989-97ce-f3a1fb8c04d7", + "uuid": "1bea47ed-f6a9-463b-b423-14b9cca9ad27", "volume_size": 10 } ], @@ -111,7 +111,7 @@ const expectedBlockDeviceBootVolumeTypeRequest = `{ "delete_on_termination": true, "destination_type": "volume", "source_type": "image", - "uuid": "f3e4a95d-1f4f-4989-97ce-f3a1fb8c04d7", + "uuid": "1bea47ed-f6a9-463b-b423-14b9cca9ad27", "volume_size": 10, "volume_type": "ssd" } @@ -309,7 +309,7 @@ func ExpectServerCreated(t *testing.T, expectedServer string) { if err != nil { t.Fatalf("Error occurred while unmarshaling the expected server manifest.") } - res.Server.ID = "f3e4a95d-1f4f-4989-97ce-f3a1fb8c04d7" + res.Server.ID = "1bea47ed-f6a9-463b-b423-14b9cca9ad27" srvRes, err := json.Marshal(res) if err != nil { t.Fatalf("Error occurred while marshaling the server response manifest.") @@ -335,41 +335,69 @@ func ExpectServerCreated(t *testing.T, expectedServer string) { fmt.Fprintf(w, string(srvRes)) }) - // Handle listing images. - th.Mux.HandleFunc("/images/detail", func(w http.ResponseWriter, r *http.Request) { + // Handle listing images v2. + th.Mux.HandleFunc("/v2/images", func(w http.ResponseWriter, r *http.Request) { th.TestMethod(t, r, "GET") th.TestHeader(t, r, "X-Auth-Token", client.TokenID) w.Header().Add("Content-Type", "application/json") + // Example ref: https://docs.openstack.org/api-ref/image/v2/index.html?expanded=list-images-detail#list-images fmt.Fprintf(w, ` { "images": [ { - "status": "ACTIVE", - "updated": "2014-09-23T12:54:56Z", - "id": "f3e4a95d-1f4f-4989-97ce-f3a1fb8c04d7", - "OS-EXT-IMG-SIZE:size": 476704768, - "name": "F17-x86_64-cfntools", - "created": "2014-09-23T12:54:52Z", - "minDisk": 0, - "progress": 100, - "minRam": 0 + "status": "active", + "name": "cirros-0.3.2-x86_64-disk", + "tags": [], + "container_format": "bare", + "created_at": "2014-11-07T17:07:06Z", + "disk_format": "qcow2", + "updated_at": "2014-11-07T17:19:09Z", + "visibility": "public", + "self": "/v2/images/1bea47ed-f6a9-463b-b423-14b9cca9ad27", + "min_disk": 0, + "protected": false, + "id": "1bea47ed-f6a9-463b-b423-14b9cca9ad27", + "file": "/v2/images/1bea47ed-f6a9-463b-b423-14b9cca9ad27/file", + "checksum": "64d7c1cd2b6f60c92c14662941cb7913", + "os_hash_algo": "sha512", + "os_hash_value": "073b4523583784fbe01daff81eba092a262ec37ba6d04dd3f52e4cd5c93eb8258af44881345ecda0e49f3d8cc6d2df6b050ff3e72681d723234aff9d17d0cf09", + "os_hidden": false, + "owner": "5ef70662f8b34079a6eddb8da9d75fe8", + "size": 13167616, + "min_ram": 0, + "schema": "/v2/schemas/image", + "virtual_size": null }, { - "status": "ACTIVE", - "updated": "2014-09-23T12:51:43Z", - "id": "f90f6034-2570-4974-8351-6b49732ef2eb", - "OS-EXT-IMG-SIZE:size": 13167616, - "name": "cirros-0.3.2-x86_64-disk", - "created": "2014-09-23T12:51:42Z", - "minDisk": 0, - "progress": 100, - "minRam": 0 + "status": "active", + "name": "F17-x86_64-cfntools", + "tags": [], + "container_format": "bare", + "created_at": "2014-10-30T08:23:39Z", + "disk_format": "qcow2", + "updated_at": "2014-11-03T16:40:10Z", + "visibility": "public", + "self": "/v2/images/781b3762-9469-4cec-b58d-3349e5de4e9c", + "min_disk": 0, + "protected": false, + "id": "781b3762-9469-4cec-b58d-3349e5de4e9c", + "file": "/v2/images/781b3762-9469-4cec-b58d-3349e5de4e9c/file", + "checksum": "afab0f79bac770d61d24b4d0560b5f70", + "os_hash_algo": "sha512", + "os_hash_value": "ea3e20140df1cc65f53d4c5b9ee3b38d0d6868f61bbe2230417b0f98cef0e0c7c37f0ebc5c6456fa47f013de48b452617d56c15fdba25e100379bd0e81ee15ec", + "os_hidden": false, + "owner": "5ef70662f8b34079a6eddb8da9d75fe8", + "size": 476704768, + "min_ram": 0, + "schema": "/v2/schemas/image", + "virtual_size": null } ] } `) }) + // Handle listing flavours. th.Mux.HandleFunc("/flavors/detail", func(w http.ResponseWriter, r *http.Request) { th.TestMethod(t, r, "GET") From a5b2d57b475da95de59368f20cfe936dac55d0a5 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Mon, 8 Nov 2021 22:53:49 +0100 Subject: [PATCH 017/489] fix migrate uid test (#1093) Signed-off-by: Moath Qasim --- test/e2e/provisioning/migrateuidscenario.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/e2e/provisioning/migrateuidscenario.go b/test/e2e/provisioning/migrateuidscenario.go index 977ac71ff..8f1e20f60 100644 --- a/test/e2e/provisioning/migrateuidscenario.go +++ b/test/e2e/provisioning/migrateuidscenario.go @@ -60,13 +60,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time machine.Name = machineDeployment.Name machine.Namespace = metav1.NamespaceSystem machine.Spec.Name = machine.Name - fakeClient := fakectrlruntimeclient.NewFakeClient( - &v1alpha1.Machine{ - ObjectMeta: metav1.ObjectMeta{ - Name: machineDeployment.Name, - Namespace: metav1.NamespaceSystem, - }, - }) + fakeClient := fakectrlruntimeclient.NewFakeClient(machine) providerData := &cloudprovidertypes.ProviderData{ Update: cloudprovidertypes.GetMachineUpdater(context.Background(), fakeClient), From 01959c2271f8fafc27a524e9d0f08005d74b4878 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Wed, 10 Nov 2021 17:48:06 +0100 Subject: [PATCH 018/489] update to controller-runtime 0.10 and k8s 1.22, remove compat with k8s <1.19 (#1060) * it compiles! * it still compiles * remove semver v2 because we already use v3 * it still compiles * it still compiles * update golden testdata fixture files * remove 1.18 compat for CSR's * fix typo * temporarily disable containerd for 1.22, just to see what happens * make terraform look prettier * make sure to compile e2e tests when creating the gocache * post-rebase go mod tidy * update generated userdata * fix yamllint * CR 0.10.2 * update fixtures * Revert "temporarily disable containerd for 1.22, just to see what happens" This reverts commit 51a4f4ef28a60f71df56d3adeab461c8d96c3c92. --- Makefile | 1 + cmd/machine-controller/main.go | 18 +- docs/network-restrictions.md | 11 +- examples/alibaba-machinedeployment.yaml | 2 +- examples/aws-machinedeployment.yaml | 4 +- examples/digitalocean-machinedeployment.yaml | 2 +- examples/hetzner-machinedeployment.yaml | 2 +- examples/linode-machinedeployment.yaml | 2 +- examples/openstack-machinedeployment.yaml | 14 +- examples/packet-machinedeployment.yaml | 2 +- examples/scaleway-machinedeployment.yaml | 2 +- go.mod | 36 +- go.sum | 478 ++++++++++++++---- hack/ci-e2e-test.sh | 3 + .../machinesv1alpha1machine/openstack.yaml | 10 +- pkg/containerruntime/containerruntime.go | 6 +- pkg/containerruntime/docker.go | 8 +- pkg/controller/machine/machine_controller.go | 2 +- .../nodecsrapprover/node_csr_approver.go | 70 +-- .../node_csr_approver_v1beta1.go | 232 --------- pkg/userdata/amzn2/provider.go | 4 +- pkg/userdata/amzn2/provider_test.go | 69 ++- .../containerd-kubelet-v1.20-aws.yaml | 7 +- .../amzn2/testdata/kubelet-v1.16-aws.yaml | 408 --------------- .../amzn2/testdata/kubelet-v1.19-aws.yaml | 7 +- .../amzn2/testdata/kubelet-v1.20-aws.yaml | 7 +- ...l.yaml => kubelet-v1.21-aws-external.yaml} | 7 +- ...-v1.18-aws.yaml => kubelet-v1.21-aws.yaml} | 7 +- ...aml => kubelet-v1.21-vsphere-mirrors.yaml} | 7 +- ....yaml => kubelet-v1.21-vsphere-proxy.yaml} | 7 +- ...sphere.yaml => kubelet-v1.21-vsphere.yaml} | 7 +- ...-v1.17-aws.yaml => kubelet-v1.22-aws.yaml} | 53 +- pkg/userdata/centos/provider.go | 4 +- pkg/userdata/centos/provider_test.go | 62 +-- .../kubelet-containerd-v1.20-aws.yaml | 7 +- .../centos/testdata/kubelet-v1.19-aws.yaml | 7 +- .../centos/testdata/kubelet-v1.20-aws.yaml | 7 +- ...l.yaml => kubelet-v1.21-aws-external.yaml} | 7 +- ...-v1.17-aws.yaml => kubelet-v1.21-aws.yaml} | 7 +- ...aml => kubelet-v1.21-vsphere-mirrors.yaml} | 7 +- ....yaml => kubelet-v1.21-vsphere-proxy.yaml} | 7 +- ...sphere.yaml => kubelet-v1.21-vsphere.yaml} | 7 +- ...-v1.18-aws.yaml => kubelet-v1.22-aws.yaml} | 53 +- pkg/userdata/flatcar/provider.go | 2 +- pkg/userdata/flatcar/provider_test.go | 40 +- ...v1.18.14.yaml => cloud-init_v1.19.15.yaml} | 7 +- ..._v1.19.4.yaml => cloud-init_v1.20.11.yaml} | 7 +- ..._v1.17.16.yaml => cloud-init_v1.21.5.yaml} | 7 +- ...t_v1.20.1.yaml => cloud-init_v1.22.2.yaml} | 11 +- pkg/userdata/flatcar/testdata/containerd.yaml | 7 +- .../flatcar/testdata/ignition_v1.17.16.json | 1 - .../flatcar/testdata/ignition_v1.18.14.json | 1 - .../flatcar/testdata/ignition_v1.19.15.json | 1 + .../flatcar/testdata/ignition_v1.19.4.json | 1 - .../flatcar/testdata/ignition_v1.19.6.json | 1 - .../flatcar/testdata/ignition_v1.20.1.json | 1 - .../flatcar/testdata/ignition_v1.20.11.json | 1 + .../flatcar/testdata/ignition_v1.21.5.json | 1 + .../flatcar/testdata/ignition_v1.22.2.json | 1 + pkg/userdata/helper/common_test.go | 10 +- pkg/userdata/helper/kubelet_test.go | 2 +- ...lden => download_binaries_v1.19.15.golden} | 2 +- ...lden => download_binaries_v1.20.11.golden} | 2 +- ...olden => download_binaries_v1.21.5.golden} | 2 +- ...olden => download_binaries_v1.22.2.golden} | 2 +- ...emd_unit_version-v1.19.15-external.golden} | 0 ...blet_systemd_unit_version-v1.19.15.golden} | 0 ...emd_unit_version-v1.20.11-external.golden} | 0 ...blet_systemd_unit_version-v1.20.11.golden} | 0 ...temd_unit_version-v1.21.5-external.golden} | 0 ...ublet_systemd_unit_version-v1.21.5.golden} | 0 ...temd_unit_version-v1.22.2-external.golden} | 0 ...ublet_systemd_unit_version-v1.22.2.golden} | 0 pkg/userdata/rhel/provider.go | 4 +- pkg/userdata/rhel/provider_test.go | 36 +- ...yaml => kubelet-containerd-v1.19-aws.yaml} | 53 +- .../rhel/testdata/kubelet-v1.19-aws.yaml | 7 +- .../rhel/testdata/kubelet-v1.20-aws.yaml | 7 +- ...-v1.18-aws.yaml => kubelet-v1.21-aws.yaml} | 7 +- ...l.yaml => kubelet-v1.22-aws-external.yaml} | 53 +- ...-v1.20-aws.yaml => kubelet-v1.22-aws.yaml} | 7 +- ...aml => kubelet-v1.22-vsphere-mirrors.yaml} | 53 +- ....yaml => kubelet-v1.22-vsphere-proxy.yaml} | 57 ++- ...sphere.yaml => kubelet-v1.22-vsphere.yaml} | 53 +- pkg/userdata/sles/provider.go | 4 +- pkg/userdata/sles/provider_test.go | 12 +- .../sles/testdata/dist-upgrade-on-boot.yaml | 7 +- .../kubelet-version-without-v-prefix.yaml | 7 +- .../sles/testdata/multiple-dns-servers.yaml | 7 +- .../sles/testdata/multiple-ssh-keys.yaml | 7 +- .../openstack-overwrite-cloud-config.yaml | 7 +- pkg/userdata/sles/testdata/openstack.yaml | 7 +- ...sion-1.17.16.yaml => version-1.19.15.yaml} | 7 +- ...rsion-1.19.4.yaml => version-1.20.11.yaml} | 7 +- ...rsion-1.18.14.yaml => version-1.21.5.yaml} | 7 +- ...ersion-1.20.1.yaml => version-1.22.2.yaml} | 7 +- .../sles/testdata/vsphere-mirrors.yaml | 7 +- pkg/userdata/sles/testdata/vsphere-proxy.yaml | 7 +- pkg/userdata/sles/testdata/vsphere.yaml | 7 +- pkg/userdata/ubuntu/provider.go | 4 +- pkg/userdata/ubuntu/provider_test.go | 24 +- pkg/userdata/ubuntu/testdata/containerd.yaml | 7 +- .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 7 +- .../kubelet-version-without-v-prefix.yaml | 7 +- .../ubuntu/testdata/multiple-dns-servers.yaml | 7 +- .../ubuntu/testdata/multiple-ssh-keys.yaml | 7 +- .../openstack-overwrite-cloud-config.yaml | 7 +- pkg/userdata/ubuntu/testdata/openstack.yaml | 7 +- ...sion-1.18.14.yaml => version-1.19.15.yaml} | 7 +- ...rsion-1.19.4.yaml => version-1.20.11.yaml} | 7 +- ...rsion-1.17.16.yaml => version-1.21.5.yaml} | 7 +- ...ersion-1.20.1.yaml => version-1.22.2.yaml} | 52 +- .../ubuntu/testdata/vsphere-mirrors.yaml | 7 +- .../ubuntu/testdata/vsphere-proxy.yaml | 7 +- pkg/userdata/ubuntu/testdata/vsphere.yaml | 7 +- test/e2e/provisioning/all_e2e_test.go | 8 +- test/e2e/provisioning/helper.go | 2 +- ...deployment-aws-ebs-encryption-enabled.yaml | 4 +- .../machinedeployment-aws-spot-instances.yaml | 4 +- .../testdata/machinedeployment-aws.yaml | 4 +- ...hinedeployment-azure-redhat-satellite.yaml | 2 +- 121 files changed, 1165 insertions(+), 1254 deletions(-) delete mode 100644 pkg/controller/nodecsrapprover/node_csr_approver_v1beta1.go delete mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml rename pkg/userdata/amzn2/testdata/{kubelet-v1.17-aws-external.yaml => kubelet-v1.21-aws-external.yaml} (98%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.18-aws.yaml => kubelet-v1.21-aws.yaml} (98%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.17-vsphere-mirrors.yaml => kubelet-v1.21-vsphere-mirrors.yaml} (99%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.17-vsphere-proxy.yaml => kubelet-v1.21-vsphere-proxy.yaml} (99%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.17-vsphere.yaml => kubelet-v1.21-vsphere.yaml} (98%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.17-aws.yaml => kubelet-v1.22-aws.yaml} (91%) rename pkg/userdata/centos/testdata/{kubelet-v1.17-aws-external.yaml => kubelet-v1.21-aws-external.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.17-aws.yaml => kubelet-v1.21-aws.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.17-vsphere-mirrors.yaml => kubelet-v1.21-vsphere-mirrors.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.17-vsphere-proxy.yaml => kubelet-v1.21-vsphere-proxy.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.17-vsphere.yaml => kubelet-v1.21-vsphere.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.18-aws.yaml => kubelet-v1.22-aws.yaml} (91%) rename pkg/userdata/flatcar/testdata/{cloud-init_v1.18.14.yaml => cloud-init_v1.19.15.yaml} (98%) rename pkg/userdata/flatcar/testdata/{cloud-init_v1.19.4.yaml => cloud-init_v1.20.11.yaml} (98%) rename pkg/userdata/flatcar/testdata/{cloud-init_v1.17.16.yaml => cloud-init_v1.21.5.yaml} (98%) rename pkg/userdata/flatcar/testdata/{cloud-init_v1.20.1.yaml => cloud-init_v1.22.2.yaml} (98%) delete mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.17.16.json delete mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.18.14.json create mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.19.15.json delete mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.19.4.json delete mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.19.6.json delete mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.20.1.json create mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.20.11.json create mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.21.5.json create mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.22.2.json rename pkg/userdata/helper/testdata/{download_binaries_v1.17.16.golden => download_binaries_v1.19.15.golden} (92%) rename pkg/userdata/helper/testdata/{download_binaries_v1.18.14.golden => download_binaries_v1.20.11.golden} (92%) rename pkg/userdata/helper/testdata/{download_binaries_v1.19.4.golden => download_binaries_v1.21.5.golden} (92%) rename pkg/userdata/helper/testdata/{download_binaries_v1.20.1.golden => download_binaries_v1.22.2.golden} (92%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.17.16-external.golden => kublet_systemd_unit_version-v1.19.15-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.17.16.golden => kublet_systemd_unit_version-v1.19.15.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.18.14-external.golden => kublet_systemd_unit_version-v1.20.11-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.18.14.golden => kublet_systemd_unit_version-v1.20.11.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.19.4-external.golden => kublet_systemd_unit_version-v1.21.5-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.19.4.golden => kublet_systemd_unit_version-v1.21.5.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.20.1-external.golden => kublet_systemd_unit_version-v1.22.2-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.20.1.golden => kublet_systemd_unit_version-v1.22.2.golden} (100%) rename pkg/userdata/rhel/testdata/{kubelet-v1.17-aws.yaml => kubelet-containerd-v1.19-aws.yaml} (91%) rename pkg/userdata/rhel/testdata/{kubelet-v1.18-aws.yaml => kubelet-v1.21-aws.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.20-aws-external.yaml => kubelet-v1.22-aws-external.yaml} (91%) rename pkg/userdata/rhel/testdata/{kubelet-containerd-v1.20-aws.yaml => kubelet-v1.22-aws.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.20-vsphere-proxy.yaml => kubelet-v1.22-vsphere-mirrors.yaml} (92%) rename pkg/userdata/rhel/testdata/{kubelet-v1.20-vsphere-mirrors.yaml => kubelet-v1.22-vsphere-proxy.yaml} (90%) rename pkg/userdata/rhel/testdata/{kubelet-v1.20-vsphere.yaml => kubelet-v1.22-vsphere.yaml} (91%) rename pkg/userdata/sles/testdata/{version-1.17.16.yaml => version-1.19.15.yaml} (98%) rename pkg/userdata/sles/testdata/{version-1.19.4.yaml => version-1.20.11.yaml} (98%) rename pkg/userdata/sles/testdata/{version-1.18.14.yaml => version-1.21.5.yaml} (98%) rename pkg/userdata/sles/testdata/{version-1.20.1.yaml => version-1.22.2.yaml} (98%) rename pkg/userdata/ubuntu/testdata/{version-1.18.14.yaml => version-1.19.15.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.19.4.yaml => version-1.20.11.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.17.16.yaml => version-1.21.5.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.20.1.yaml => version-1.22.2.yaml} (91%) diff --git a/Makefile b/Makefile index 98d9c4335..9cec18e19 100644 --- a/Makefile +++ b/Makefile @@ -100,6 +100,7 @@ test-unit: .PHONY: build-tests build-tests: go test -run nope ./... + go test -tags e2e -run nope ./... .PHONY: e2e-cluster e2e-cluster: machine-controller webhook diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index ae4d2bc15..33590d281 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -79,7 +79,6 @@ var ( nodeInsecureRegistries string nodeRegistryMirrors string nodePauseImage string - nodeHyperkubeImage string nodeKubeletRepository string nodeContainerRuntime string podCidr string @@ -162,8 +161,7 @@ func main() { flag.StringVar(&nodeInsecureRegistries, "node-insecure-registries", "", "Comma separated list of registries which should be configured as insecure on the container runtime") flag.StringVar(&nodeRegistryMirrors, "node-registry-mirrors", "", "Comma separated list of Docker image mirrors") flag.StringVar(&nodePauseImage, "node-pause-image", "", "Image for the pause container including tag. If not set, the kubelet default will be used: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/") - flag.StringVar(&nodeHyperkubeImage, "node-hyperkube-image", "k8s.gcr.io/hyperkube-amd64", "Image for the hyperkube container excluding tag. Only has effect on Flatcar Linux, and for kubernetes < 1.18.") - flag.StringVar(&nodeKubeletRepository, "node-kubelet-repository", "quay.io/kubermatic/kubelet", "Repository for the kubelet container. Only has effect on Flatcar Linux, and for kubernetes >= 1.18.") + flag.StringVar(&nodeKubeletRepository, "node-kubelet-repository", "quay.io/kubermatic/kubelet", "Repository for the kubelet container. Only has effect on Flatcar Linux.") flag.StringVar(&nodeContainerRuntime, "node-container-runtime", "docker", "container-runtime to deploy") flag.StringVar(&caBundleFile, "ca-bundle", "", "path to a file containing all PEM-encoded CA certificates (will be used instead of the host's certificates if set)") flag.BoolVar(&nodeCSRApprover, "node-csr-approver", true, "Enable NodeCSRApprover controller to automatically approve node serving certificate requests") @@ -206,22 +204,13 @@ func main() { klog.Fatalf("failed to add osmv1alpha1 api to scheme: %v", err) } - // Check if the hyperkube image has a tag set - hyperkubeImageRef, err := reference.Parse(nodeHyperkubeImage) - if err != nil { - klog.Fatalf("failed to parse -node-hyperkube-image %s: %v", nodeHyperkubeImage, err) - } - if _, ok := hyperkubeImageRef.(reference.NamedTagged); ok { - klog.Fatalf("-node-hyperkube-image must not contain a tag. The tag will be dynamically set for each Machine.") - } - // Check if the kubelet image has a tag set kubeletRepoRef, err := reference.Parse(nodeKubeletRepository) if err != nil { - klog.Fatalf("failed to parse -node-hyperkube-image %s: %v", nodeHyperkubeImage, err) + klog.Fatalf("failed to parse -node-kubelet-repository %s: %v", nodeKubeletRepository, err) } if _, ok := kubeletRepoRef.(reference.NamedTagged); ok { - klog.Fatalf("-node-kubelet-image must not contain a tag. The tag will be dynamically set for each Machine.") + klog.Fatalf("-node-kubelet-repository must not contain a tag. The tag will be dynamically set for each Machine.") } cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) @@ -280,7 +269,6 @@ func main() { node: machinecontroller.NodeSettings{ ClusterDNSIPs: clusterDNSIPs, HTTPProxy: nodeHTTPProxy, - HyperkubeImage: nodeHyperkubeImage, KubeletRepository: nodeKubeletRepository, NoProxy: nodeNoProxy, PauseImage: nodePauseImage, diff --git a/docs/network-restrictions.md b/docs/network-restrictions.md index 52377c0f6..f049c853b 100644 --- a/docs/network-restrictions.md +++ b/docs/network-restrictions.md @@ -31,17 +31,14 @@ If that image won't be accessible from the node, a custom image can be specified ## Kubelet images ### Flatcar Linux -For Flatcar Linux nodes, the [hyperkube][1] or [kubelet][3] image must be accessible as well. This is due to the fact -that kubelet is running as a docker container. For kubelet version `< 1.18` hyperkube will be used, otherwise `kubelet` -image. +For Flatcar Linux nodes, [kubelet][3] image must be accessible as well. This is due to the fact +that kubelet is running as a docker container. By default the image `quay.io/kubermatic/kubelet` will be used. If that image won't be accessible from the node, a custom image can be specified on the machine-controller: + ```bash # Do not set a tag. The tag depends on the used Kubernetes version of a machine. -# Example: -# A Node using v1.14.2 would use 192.168.1.1:5000/kubernetes/hyperkube-amd64:v1.14.2 --node-hyperkube-image="192.168.1.1:5000/kubernetes/hyperkube-amd64" -node-kubelet-image="192.168.1.1:5000/my-custom/kubelet-amd64" ``` @@ -49,7 +46,7 @@ image can be specified on the machine-controller: If nodes require access to insecure registries, all registries must be specified via a flag: ```bash ---node-insecure-registries="192.168.1.1:5000,10.0.0.1:5000" +-node-insecure-registries="192.168.1.1:5000,10.0.0.1:5000" ``` [1]: https://console.cloud.google.com/gcr/images/google-containers/GLOBAL/hyperkube diff --git a/examples/alibaba-machinedeployment.yaml b/examples/alibaba-machinedeployment.yaml index 603092b31..1736b4730 100644 --- a/examples/alibaba-machinedeployment.yaml +++ b/examples/alibaba-machinedeployment.yaml @@ -38,7 +38,7 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "alibaba" cloudProviderSpec: - # If empty, can be set via ALIBABA_ACCESS_KEY_ID env var + # If empty, can be set via ALIBABA_ACCESS_KEY_ID env var accessKeyID: secretKeyRef: namespace: kube-system diff --git a/examples/aws-machinedeployment.yaml b/examples/aws-machinedeployment.yaml index 32beabe68..5d0060f21 100644 --- a/examples/aws-machinedeployment.yaml +++ b/examples/aws-machinedeployment.yaml @@ -38,13 +38,13 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "aws" cloudProviderSpec: - # If empty, can be set via AWS_ACCESS_KEY_ID env var + # If empty, can be set via AWS_ACCESS_KEY_ID env var accessKeyId: secretKeyRef: namespace: kube-system name: machine-controller-aws key: accessKeyId - # If empty, can be set via AWS_SECRET_ACCESS_KEY env var + # If empty, can be set via AWS_SECRET_ACCESS_KEY env var secretAccessKey: secretKeyRef: namespace: kube-system diff --git a/examples/digitalocean-machinedeployment.yaml b/examples/digitalocean-machinedeployment.yaml index 5b7bf9dba..ebde3cf31 100644 --- a/examples/digitalocean-machinedeployment.yaml +++ b/examples/digitalocean-machinedeployment.yaml @@ -37,7 +37,7 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "digitalocean" cloudProviderSpec: - # If empty, can be set via DO_TOKEN env var + # If empty, can be set via DO_TOKEN env var token: secretKeyRef: namespace: kube-system diff --git a/examples/hetzner-machinedeployment.yaml b/examples/hetzner-machinedeployment.yaml index 9ab5373aa..51b7c1cf4 100644 --- a/examples/hetzner-machinedeployment.yaml +++ b/examples/hetzner-machinedeployment.yaml @@ -37,7 +37,7 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "hetzner" cloudProviderSpec: - # If empty, can be set via HZ_TOKEN env var + # If empty, can be set via HZ_TOKEN env var token: secretKeyRef: namespace: kube-system diff --git a/examples/linode-machinedeployment.yaml b/examples/linode-machinedeployment.yaml index 9d0c735c7..88a510ebe 100644 --- a/examples/linode-machinedeployment.yaml +++ b/examples/linode-machinedeployment.yaml @@ -37,7 +37,7 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "linode" cloudProviderSpec: - # If empty, can be set via LINODE_TOKEN env var + # If empty, can be set via LINODE_TOKEN env var token: secretKeyRef: namespace: kube-system diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index 8320c16da..de2552703 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -52,43 +52,43 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "openstack" cloudProviderSpec: - # If empty, can be set via OS_AUTH_URL env var + # If empty, can be set via OS_AUTH_URL env var identityEndpoint: secretKeyRef: namespace: kube-system name: machine-controller-openstack key: identityEndpoint - # If empty, can be set via OS_APPLICATION_CREDENTIAL_ID env var + # If empty, can be set via OS_APPLICATION_CREDENTIAL_ID env var applicationCredentialID: secretKeyRef: namespace: kube-system name: machine-controller-openstack key: applicationCredentialID - # If empty, can be set via OS_APPLICATION_CREDENTIAL_SECRET env var + # If empty, can be set via OS_APPLICATION_CREDENTIAL_SECRET env var applicationCredentialSecret: secretKeyRef: namespace: kube-system name: machine-controller-openstack key: applicationCredentialSecret - # If empty, can be set via OS_USER_NAME env var + # If empty, can be set via OS_USER_NAME env var username: secretKeyRef: namespace: kube-system name: machine-controller-openstack key: username - # If empty, can be set via OS_PASSWORD env var + # If empty, can be set via OS_PASSWORD env var password: secretKeyRef: namespace: kube-system name: machine-controller-openstack key: password - # If empty, can be set via OS_DOMAIN_NAME env var + # If empty, can be set via OS_DOMAIN_NAME env var domainName: secretKeyRef: namespace: kube-system name: machine-controller-openstack key: domainName - # If empty, can be set via OS_TENANT_NAME env var + # If empty, can be set via OS_TENANT_NAME env var tenantName: secretKeyRef: namespace: kube-system diff --git a/examples/packet-machinedeployment.yaml b/examples/packet-machinedeployment.yaml index bf5d175b6..1831f37ef 100644 --- a/examples/packet-machinedeployment.yaml +++ b/examples/packet-machinedeployment.yaml @@ -37,7 +37,7 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "packet" cloudProviderSpec: - # If empty, can be set via PACKET_API_KEY env var + # If empty, can be set via PACKET_API_KEY env var apiKey: secretKeyRef: namespace: kube-system diff --git a/examples/scaleway-machinedeployment.yaml b/examples/scaleway-machinedeployment.yaml index c9c65720b..4b79fed20 100644 --- a/examples/scaleway-machinedeployment.yaml +++ b/examples/scaleway-machinedeployment.yaml @@ -61,4 +61,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.18.8 + kubelet: 1.21.5 diff --git a/go.mod b/go.mod index a826e9432..95ffd2bbb 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,8 @@ require ( github.com/Azure/azure-sdk-for-go v49.0.0+incompatible github.com/Azure/go-autorest/autorest/azure/auth v0.5.5 github.com/Azure/go-autorest/autorest/to v0.4.0 + github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/BurntSushi/toml v0.3.1 - github.com/Masterminds/semver v1.5.0 github.com/Masterminds/semver/v3 v3.1.1 github.com/Masterminds/sprig/v3 v3.2.2 github.com/aliyun/alibaba-cloud-sdk-go v1.61.751 @@ -31,33 +31,41 @@ require ( github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 - github.com/prometheus/client_golang v1.8.0 + github.com/prometheus/client_golang v1.11.0 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7 github.com/sethvargo/go-password v0.2.0 github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda github.com/vmware/govmomi v0.23.1 - golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 + golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58 - gomodules.xyz/jsonpatch/v2 v2.1.0 + gomodules.xyz/jsonpatch/v2 v2.2.0 google.golang.org/api v0.36.0 - google.golang.org/grpc v1.33.2 + google.golang.org/grpc v1.38.0 gopkg.in/gcfg.v1 v1.2.3 - gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b k8c.io/operating-system-manager v0.1.0 - k8s.io/api v0.20.2 - k8s.io/apiextensions-apiserver v0.19.4 - k8s.io/apimachinery v0.20.2 + k8s.io/api v0.22.2 + k8s.io/apiextensions-apiserver v0.22.2 + k8s.io/apimachinery v0.22.2 k8s.io/client-go v12.0.0+incompatible k8s.io/klog v1.0.0 - k8s.io/kubelet v0.19.4 - k8s.io/utils v0.0.0-20201110183641-67b214c5f920 + k8s.io/kubelet v0.22.2 + k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a kubevirt.io/client-go v0.30.0 - kubevirt.io/containerized-data-importer v1.10.6 - sigs.k8s.io/controller-runtime v0.7.0 + kubevirt.io/containerized-data-importer v1.40.0 + sigs.k8s.io/controller-runtime v0.10.2 sigs.k8s.io/yaml v1.2.0 ) replace ( + github.com/openshift/api => github.com/openshift/api v0.0.0-20210428205234-a8389931bee7 + github.com/openshift/client-go => github.com/openshift/client-go v0.0.0-20210112165513-ebc401615f47 + github.com/openshift/library-go => github.com/mhenriks/library-go v0.0.0-20210511195009-51ba86622560 + github.com/operator-framework/operator-lifecycle-manager => github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190128024246-5eb7ae5bdb7a + // the following replacements are only here to make kubevirt.io/containerized-data-importer work github.com/packethost/packngo => github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888 - k8s.io/client-go => k8s.io/client-go v0.20.2 + + k8s.io/client-go => k8s.io/client-go v0.22.2 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.22.2 + k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd ) diff --git a/go.sum b/go.sum index 4a7c704b8..14a291796 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,7 @@ contrib.go.opencensus.io/exporter/stackdriver v0.12.8/go.mod h1:XyyafDnFOsqoxHJg dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-sdk-for-go v28.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -60,26 +61,28 @@ github.com/Azure/azure-sdk-for-go v49.0.0+incompatible h1:rvYYNgKNBwoxUaBFmd/+Tp github.com/Azure/azure-sdk-for-go v49.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.1.0/go.mod h1:AKyIcETwSUFxIcs/Wnq/C+kwCtlEYGUVd7FPNb2slmg= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= github.com/Azure/go-autorest/autorest v0.9.5/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= -github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= github.com/Azure/go-autorest/autorest v0.11.6/go.mod h1:V6p3pKZx1KKkJubbxnDWrzNhEIfOy/pTGasLqzHIPHs= -github.com/Azure/go-autorest/autorest v0.11.13 h1:XKx/sB3bfadpXBBHPc7tP2XPKhzVyrdhxpDC3T0wqjs= github.com/Azure/go-autorest/autorest v0.11.13/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= +github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest/adal v0.1.0/go.mod h1:MeS4XhScH55IST095THyTxElntu7WqB7pNbZo8Q5G3E= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.4/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/adal v0.9.8 h1:bW6ZdxqMYWsxGikpM62SSE3jnvOXVu9SXzJTuj1WM3Y= github.com/Azure/go-autorest/autorest/adal v0.9.8/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/azure/auth v0.5.5 h1:7HT2JTm2BOsBMPrT1/iWZW4+XmRvyICcbCejf9BkmYU= github.com/Azure/go-autorest/autorest/azure/auth v0.5.5/go.mod h1:ptW4D47I+eIUe/lulFLYTVfG4rAARZoXIe1vmTQ+ol8= github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY= @@ -91,7 +94,6 @@ github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSY github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= -github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/to v0.1.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc= @@ -101,11 +103,13 @@ github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+X github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI= -github.com/Azure/go-autorest/autorest/validation v0.3.0 h1:3I9AAI63HfcLtphd9g39ruUwRI+Ca+z/f36KHPFRUss= github.com/Azure/go-autorest/autorest/validation v0.3.0/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= +github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.1.0/go.mod h1:ROEEAFwXycQw7Sn3DXNtEedEvdeRAgDr0izn4z5Ij88= github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= @@ -134,24 +138,27 @@ github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmy github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.7/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/Venafi/vcert/v4 v4.11.0/go.mod h1:OE+UZ0cj8qqVUuk0u7R4GIk4ZB6JMSf/WySqnBPNwws= +github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/a8m/expect v1.0.0/go.mod h1:4IwSCMumY49ScypDnjNbYEjgVeqy1/U2cEs3Lat96eA= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083 h1:uwcvnXW76Y0rHM+qs7y8iHknWUWXYFNlD6FEVhc47TU= @@ -176,6 +183,7 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apoydence/onpar v0.0.0-20200406201722-06f95a1c68e8/go.mod h1:maauOJD0kdDqIz4xmkunipFVbBoTM6pFSy0kkWBcIUY= +github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30/go.mod h1:4AJxUpXUhv4N+ziTvIcWWXgeorXpxPZOfk9HdEVr96M= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -189,6 +197,7 @@ github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:o github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-k8s-tester v0.0.0-20190114231546-b411acf57dfe/go.mod h1:1ADF5tAtU1/mVtfMcHAYSm2fPw71DA7fFk0yed64/0I= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.15.77/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= github.com/aws/aws-sdk-go v1.16.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -201,6 +210,9 @@ github.com/aws/aws-sdk-go v1.36.2 h1:UAeFPct+jHqWM+tgiqDrC9/sfbWj6wkcvpsJ+zdcsvA github.com/aws/aws-sdk-go v1.36.2/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/bazelbuild/buildtools v0.0.0-20190917191645-69366ca98f89/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU= +github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -217,6 +229,9 @@ github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= @@ -232,13 +247,32 @@ github.com/cloudevents/sdk-go v0.0.0-20190509003705-56931988abe3/go.mod h1:j1nZW github.com/cloudflare/cloudflare-go v0.13.2/go.mod h1:27kfc1apuifUmJhp069y0+hwlKDg4bd8LWlu7oKeZvM= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/container-storage-interface/spec v1.1.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= +github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= +github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containers/image/v5 v5.5.1/go.mod h1:4PyNYR0nwlGq/ybVJD9hWlhmIsNra4Q8uOQX2s6E2uM= +github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= +github.com/containers/ocicrypt v1.0.2/go.mod h1:nsOhbP19flrX6rE7ieGFvBlr7modwmNjsqWarIUce4M= +github.com/containers/storage v1.20.2/go.mod h1:oOB9Ie8OVPojvoaKWEGSEtHbXUAs+tSyr7RO7ZGteMc= github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -257,10 +291,12 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/ignition v0.35.0 h1:UFodoYq1mOPrbEjtxIsZbThcDyQwAI1owczRDqWmKkQ= github.com/coreos/ignition v0.35.0/go.mod h1:WJQapxzEn9DE0ryxsGvm8QnBajm/XsS/PkrDqSpz+bA= github.com/coreos/locksmith v0.6.2/go.mod h1:mSLRr7SVSEAIugjic7+TXif/+ZQQq0zCks1vptuj2fs= @@ -274,8 +310,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cristim/ec2-instances-info v0.0.0-20201110114654-2dfcc09f67d4/go.mod h1:0yCjO4zBzlwWSGh/zGfW2Zq1NX605qCYVBHD1fPXKNs= -github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -294,18 +330,26 @@ github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dteve github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v0.0.0-20180920194744-16128bbac47f/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v1.4.2-0.20191219165747-a9416c67da9f/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= +github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libnetwork v0.0.0-20190731215715-7f13a5c99f4b/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -316,26 +360,36 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/elazarl/goproxy v0.0.0-20190911111923-ecfe977594f1/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= +github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.8.1+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.10.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.11.2+incompatible h1:Z4Z0K2AuOw+QtgwkkJnwpT165MBr12qS8rnBwjP/Pzs= github.com/emicklei/go-restful v2.11.2+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful-openapi v1.2.0/go.mod h1:cy7o3Ge8ZWZ5E90mpEY81sJZZFs2pkuYcLvfngYy1l0= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/etcd-io/gofail v0.0.0-20190801230047-ad7f989257ca/go.mod h1:49H/RkXP8pKaZy4h0d+NW16rSLhyVBt4o6VLJbmOqDE= github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/evanphx/json-patch v4.0.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.1.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQobrkAqrL+WFZwQses= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= +github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -344,8 +398,9 @@ github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= @@ -353,6 +408,10 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsouza/fake-gcs-server v0.0.0-20180612165233-e85be23bdaa8/go.mod h1:1/HufuJ+eaDf4KTnYdS6HJMGvMRU8d4cYTuu/1QaBbI= +github.com/fsouza/go-dockerclient v0.0.0-20171004212419-da3951ba2e9e/go.mod h1:KpcjM623fQYE9MZiTGzKhjfxXAV9wbyX2C1cyRHfhl0= +github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= +github.com/getsentry/raven-go v0.0.0-20190513200303-c977f96e1095/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -371,6 +430,7 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= @@ -379,13 +439,15 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.2.1-0.20200730175230-ee2de8da5be6/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.3.0 h1:q4c+kbcR0d5rSurhBR8dIgieOaYpXtsdTYfx22Cu6rs= github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-logr/zapr v0.1.1/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= -github.com/go-logr/zapr v0.3.0 h1:iyiCRZ29uPmbO7mWIjOEiYMXrTxZWTyK4tCatLyGpUY= github.com/go-logr/zapr v0.3.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= +github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= +github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.17.2/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -405,7 +467,7 @@ github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpX github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.0.0-20180322222829-3a0015ad55fa/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= @@ -413,7 +475,7 @@ github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDB github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.0.0-20180322222742-3fb327e6747d/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= @@ -440,7 +502,7 @@ github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2g github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98= github.com/go-openapi/runtime v0.19.20/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.0.0-20180415031709-bcff419492ee/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.17.2/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= @@ -459,7 +521,7 @@ github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6 github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.0.0-20180405201759-811b1089cde9/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= @@ -468,8 +530,9 @@ github.com/go-openapi/swag v0.19.4/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= -github.com/go-openapi/swag v0.19.12 h1:Bc0bnY2c3AoF7Gc+IMIAQQsD8fLHjHpc19wXvYuayQI= github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M= +github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/validate v0.17.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= @@ -493,6 +556,7 @@ github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-swagger/go-swagger v0.25.0/go.mod h1:9639ioXrPX9E6BbnbaDklGXjNz7upAXoNBwL4Ok11Vk= github.com/go-swagger/scan-repo-boundary v0.0.0-20180623220736-973b3573c013/go.mod h1:b65mBPzqzZWxOZGxSWrqs4GInLIn+u99Q9q7p+GKni0= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= @@ -510,6 +574,7 @@ github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= github.com/gobuffalo/flect v0.1.5/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= github.com/gobuffalo/flect v0.2.0/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= +github.com/gobuffalo/flect v0.2.2/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= @@ -530,7 +595,9 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kETFqjFoFlOc= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godror/godror v0.13.3/go.mod h1:2ouUT4kdhUBk7TAkHWD4SN0CdI0pgEQbo8FVHhbSKWg= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -541,8 +608,9 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -552,8 +620,9 @@ github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -562,7 +631,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v0.0.0-20181025225059-d3de96c4c28e/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -578,17 +646,28 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/gomodule/redigo v1.7.0/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac/go.mod h1:P32wAyui1PQ58Oce/KYkOqQv8cVw1zAapXOl+dRFGbc= +github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82/go.mod h1:PxC8OnwL11+aosOB5+iEPoV3picfs8tUpkVd0pDo+Kg= +github.com/gonum/graph v0.0.0-20170401004347-50b27dea7ebb/go.mod h1:ye018NnX1zrbOLqwBvs2HqyyTouQgnL8C+qzYk1snPY= +github.com/gonum/internal v0.0.0-20181124074243-f884aa714029/go.mod h1:Pu4dmpkhSyOzRwuXkOgAvijx4o+4YMUJJo9OvPYMkks= +github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9/go.mod h1:XA3DeT6rxh2EAE789SSiSJNqxPaC0aE9J8NTOI0Jo/A= +github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9/go.mod h1:0EXg4mc1CNP0HCqCz+K4ts155PXIlUywf0wqN+GfPZw= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -597,14 +676,14 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-containerregistry v0.0.0-20200115214256-379933c9c22b/go.mod h1:Wtl/v6YdQxv397EREtzwgd9+Ud7Q5D8XMbi3Zazgkrs= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-licenses v0.0.0-20191112164736-212ea350c932/go.mod h1:16wa6pRqNDUIhOtwF0GcROVqMeXHZJ7H6eGDFUh5Pfk= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -639,15 +718,15 @@ github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.2.2/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= -github.com/googleapis/gnostic v0.5.1 h1:A8Yhf6EtqTv9RMsU6MQTyrtV1TjWlR6xU9BsZIwuTCM= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gophercloud/gophercloud v0.14.0 h1:c2Byo+YMxhHlTJ3TPptjQ4dOQ1YknTHDJ/9zClDH+84= github.com/gophercloud/gophercloud v0.14.0/go.mod h1:VX0Ibx85B60B5XOrZr6kaNwrmPUzcmMpwxvQ1WQIIWM= @@ -658,8 +737,10 @@ github.com/gorilla/csrf v1.6.2/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAk github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v0.0.0-20181024020800-521ea7b17d02/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v0.0.0-20191024121256-f395758b854c/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= @@ -672,15 +753,19 @@ github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:Fecb github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170330212424-2500245aa611/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.6.3/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.15.2 h1:HC+hWRWf+v5zTMPyoaYTKIJih+4sd4XRWmj0qlG87Co= github.com/grpc-ecosystem/grpc-gateway v1.15.2/go.mod h1:vO11I9oWA+KsxmfFQPhLnnIb1VDE24M+pdxZFiuZcA8= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/grpc-ecosystem/grpc-health-probe v0.2.0/go.mod h1:4GVx/bTCtZaSzhjbGueDY5YgBdsmKeVx+LErv/n0L6s= github.com/h2non/gock v1.0.9/go.mod h1:CZMcB0Lg5IWnr9bF79pPMg9WeV6WumxQiUJ1UvdO1iE= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -710,7 +795,6 @@ github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -738,11 +822,13 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.0.0-20171009183408-7fe0c75c13ab/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/improbable-eng/thanos v0.3.2/go.mod h1:GZewVGILKuJVPNRn7L4Zw+7X96qzFOwj63b22xYGXBE= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v0.0.0-20161215172503-049f9b42e9a5/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= @@ -757,6 +843,7 @@ github.com/jetstack/cert-manager v1.1.0/go.mod h1:GULIHTGjSc2LjlgBCLhQ8u5WmQ95hk github.com/jinzhu/gorm v0.0.0-20170316141641-572d0a0ab1eb/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= github.com/jinzhu/inflection v0.0.0-20190603042836-f5c5f50e6090/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -766,6 +853,7 @@ github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52Cu github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.0.0-20141017032234-72f9bd7c4e0c/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -776,8 +864,9 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jsonnet-bundler/jsonnet-bundler v0.1.0/go.mod h1:YKsSFc9VFhhLITkJS3X2PrRqWG9u2Jq99udTdDjQLfM= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= @@ -790,14 +879,19 @@ github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-2019 github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/knative/build v0.1.2/go.mod h1:/sU74ZQkwlYA5FwYDJhYTy61i/Kn+5eWfln2jDbw3Qo= github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -817,6 +911,9 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubermatic/machine-controller v1.23.1/go.mod h1:mXWbT7SjqpgFhzCFT3yMEHKdIlT+KkGy4KQCkNRM9Fc= github.com/kubermatic/machine-controller v1.26.0/go.mod h1:dcJ+GdDSCxCwM0poxwOK8hVO7epiOORDmNMmb2veyw4= +github.com/kubernetes-csi/csi-lib-utils v0.7.0/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0= +github.com/kubernetes-csi/csi-test v2.0.0+incompatible/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= +github.com/kubernetes-csi/external-snapshotter/v2 v2.1.1/go.mod h1:dV5oB3U62KBdlf9ADWkMmjGd3USauqQtwIm2OZb5mqI= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -831,7 +928,7 @@ github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20180323154445-8b799c424f57/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -840,6 +937,7 @@ github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7 github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/markbates/inflect v1.0.4/go.mod h1:1fR9+pO2KHEO9ZRtto13gDwwZaAKstQzferVeWqbgNs= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= @@ -847,6 +945,7 @@ github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qo github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.6/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -859,17 +958,23 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.5/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v0.0.0-20160514122348-38ee283dabf1/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/maxbrunsfeld/counterfeiter v0.0.0-20181017030959-1aadac120687/go.mod h1:aoVsckWnsNzazwF2kmD+bzgdr4GBlbK91zsdivQJ2eU= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= +github.com/mhenriks/library-go v0.0.0-20210511195009-51ba86622560/go.mod h1:udseDnqxn5ON8i+NBjDp00fBTK0JRu1/6Y6tf6EivDE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= +github.com/mistifyio/go-zfs v2.1.1+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= @@ -891,17 +996,20 @@ github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mrnold/go-libnbd v1.4.1-cdi/go.mod h1:t/zovtHFkgtIy65eJ+Ay1mNBFz+yO6ESu6r6CluGzdI= +github.com/mtrmac/gpgme v0.1.2/go.mod h1:GYYHnGSuS7HK3zVS2n3y73y0okK/BeKzwnn5jgiVFNI= github.com/munnerz/crd-schema-fuzz v1.0.0/go.mod h1:4z/rcm37JxUkSsExFcLL6ZIT1SgDRdLiu7qq1evdVS0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -925,8 +1033,9 @@ github.com/nelsam/hel v0.0.0-20200611165952-2d829bae0c66/go.mod h1:Rl/hm4V2s75Sc github.com/nelsam/hel/v2 v2.3.2/go.mod h1:1ZTGfU2PFTOd5mx22i5O0Lc2GY933lQ2wb/ggy+rL3w= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= @@ -941,14 +1050,18 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M= github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.2-0.20180831124310-ae19f1b56d53/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -957,8 +1070,9 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/open-policy-agent/cert-controller v0.0.0-20200921224206-24b87bbc4b6e/go.mod h1:/y33mmiq3Cc0N+6cickevrLI/iBbWcUwcEVjSKHA0z0= github.com/open-policy-agent/frameworks/constraint v0.0.0-20200929072634-d96896eff389/go.mod h1:Dr3QxvH+NTQcPPZWSt1ueNOsxW4VwgUltaLL7Ttnrac= @@ -967,14 +1081,24 @@ github.com/open-policy-agent/gatekeeper v0.0.0-20201111000257-4450f08fa95e/go.mo github.com/open-policy-agent/opa v0.19.1/go.mod h1:rrwxoT/b011T0cyj+gg2VvxqTtn6N3gp/jzmr3fjW44= github.com/open-policy-agent/opa v0.21.0/go.mod h1:cZaTfhxsj7QdIiUI0U9aBtOLLTqVNe+XE60+9kZKLHw= github.com/open-policy-agent/opa v0.24.0/go.mod h1:qEyD/i8j+RQettHGp4f86yjrjvv+ZYia+JHCMv2G7wA= -github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= +github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v0.0.0-20191031171055-b133feaeeb2e/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/openshift/api v0.0.0-20191219222812-2987a591a72c/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= -github.com/openshift/client-go v0.0.0-20191125132246-f6563a70e19a/go.mod h1:6rzn+JTr7+WYS2E1TExP4gByoABxMznR6y2SnUIkmxk= -github.com/openshift/custom-resource-status v0.0.0-20190822192428-e62f2f3b79f3 h1:XuAys09+XqT5/FjdR23G/UtbBLII89dFe9XIi73EKIQ= +github.com/opencontainers/runc v1.0.0-rc90/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.5.2/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g= +github.com/openshift/api v0.0.0-20210428205234-a8389931bee7/go.mod h1:aqU5Cq+kqKKPbDMqxo9FojgDeSpNJI7iuskjXjtojDg= +github.com/openshift/build-machinery-go v0.0.0-20200917070002-f171684f77ab/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE= +github.com/openshift/client-go v0.0.0-20210112165513-ebc401615f47/go.mod h1:u7NRAjtYVAKokiI9LouzTv4mhds8P4S1TwdVAfbjKSk= github.com/openshift/custom-resource-status v0.0.0-20190822192428-e62f2f3b79f3/go.mod h1:GDjWl0tX6FNIj82vIxeudWeSx2Ff6nDZ8uJn0ohUFvo= +github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca h1:F1MEnOMwSrTA0YAkO0he9ip9w0JhYzI/iCB2mXmaSPg= +github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca/go.mod h1:GDjWl0tX6FNIj82vIxeudWeSx2Ff6nDZ8uJn0ohUFvo= github.com/openshift/prom-label-proxy v0.1.1-0.20191016113035-b8153a7f39f1/go.mod h1:p5MuxzsYP1JPsNGwtjtcgRHHlGziCJJfztff91nNixw= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= @@ -986,9 +1110,13 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ github.com/openzipkin/zipkin-go v0.2.0/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190128024246-5eb7ae5bdb7a/go.mod h1:vq6TTFvg6ti1Bn6ACsZneZTmjTsURgDD6tQtVDbEgsU= +github.com/operator-framework/operator-registry v1.0.4/go.mod h1:hve6YwcjM2nGVlscLtNsp9sIIBkNZo6jlJgzWw7vP9s= +github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc= github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= +github.com/ovirt/go-ovirt v4.3.4+incompatible/go.mod h1:r33ZGjVKCPMiI6hw791/Zx8tNKk0Gn+4VFWbOfyIvZQ= github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888 h1:ARzSqjQJcSR9IXGMEkXN1IQEZChwd2MqXWm6YNGvO5o= github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888/go.mod h1:RQHg5xR1F614BwJyepfMqrKN+32IH0i7yX+ey43rEeQ= github.com/packethost/pkg v0.0.0-20200903155310-0433e0605550/go.mod h1:GSv7cTtIjns4yc0pyajaM1RE/KE4djJONoblFIRDrxA= @@ -1020,8 +1148,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -1029,6 +1157,8 @@ github.com/poy/onpar v0.0.0-20200406201722-06f95a1c68e8/go.mod h1:nSbFQvMj97ZyhF github.com/poy/onpar v1.0.1/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= +github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/prometheus/client_golang v0.0.0-20181025174421-f30f42803563/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -1040,8 +1170,9 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.8.0 h1:zvJNkoCFAnYFNC24FV8nW4JdRJ3GIFcLbg65lL/JDcw= github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0/7TSWwj+ITvv0TnM= +github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= +github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -1055,18 +1186,22 @@ github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7q github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20190104105734-b1c43a6df3ae/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.14.0 h1:RHRyE8UocrbjU+6UvRzwi6HjiDfxrrBU91TtbKzkGp4= github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190104112138-b1a0a9a36d74/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -1075,8 +1210,9 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/prometheus v2.3.2+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.8.0/go.mod h1:fSI0j+IUQrDd7+ZtR9WKIGtoYAYAJUKcKhYLG25tN4g= @@ -1085,14 +1221,17 @@ github.com/rcrowley/go-metrics v0.0.0-20190706150252-9beb055b7962/go.mod h1:bCqn github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= +github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg= github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -1104,6 +1243,7 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7 h1:Do8ksLD4Nr3pA0x0hnLOLftZgkiTDvwPDShRTUxtXpE= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8= +github.com/sclevine/spec v1.0.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -1127,6 +1267,8 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= @@ -1135,6 +1277,7 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -1151,6 +1294,7 @@ github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHN github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.0.1-0.20200713175500-884edc58ad08/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -1172,15 +1316,17 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/tchap/go-patricia v2.3.0+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/tektoncd/pipeline v0.10.1/go.mod h1:D2X0exT46zYx95BU7ByM8+erpjoN7thmUBvlKThOszU= github.com/tektoncd/plumbing v0.0.0-20191216083742-847dcf196de9/go.mod h1:QZHgU07PRBTRF6N57w4+ApRu8OgfYLFNqCDlfEZaD9Y= github.com/tektoncd/plumbing/pipelinerun-logs v0.0.0-20191206114338-712d544c2c21/go.mod h1:S62EUWtqmejjJgUMOGB1CCCHRp6C706laH06BoALkzU= @@ -1190,23 +1336,32 @@ github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda h1:uAHwUH+06gowZMV github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda/go.mod h1:s4k7CORR0OMWd4cYwBqNBFPSJZhnSQxeKdDtMa/aspk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJMir8RTqb4ayM= github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.18.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g= +github.com/vbauerster/mpb/v5 v5.2.2/go.mod h1:W5Fvgw4dm3/0NhqzV8j6EacfuTe5SvnzBRwiXxDR9ww= github.com/vdemeester/k8s-pkg-credentialprovider v0.0.0-20200107171650-7c61ffa44238/go.mod h1:JwQJCMWpUDqjZrB5jpw0f5VbN7U95zxFy1ZDpoEarGo= github.com/vdemeester/k8s-pkg-credentialprovider v1.13.12-1/go.mod h1:Fko0rTxEtDW2kju5Ky7yFJNS3IcNvW8IPsp4/e9oev0= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb h1:lyL3z7vYwTWXf4/bI+A01+cCSnfhKIBhy+SQ46Z/ml8= github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= +github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/vmware/govmomi v0.23.1 h1:vU09hxnNR/I7e+4zCJvW+5vHu5dO64Aoe2Lw7Yi/KRg= github.com/vmware/govmomi v0.23.1/go.mod h1:Y+Wq4lst78L85Ge/F8+ORXIWiKYqaro1vhAulACy9Lc= @@ -1215,6 +1370,10 @@ github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0B github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= @@ -1224,14 +1383,25 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.etcd.io/bbolt v1.3.1-etcd.7/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.0.0-20181031231232-83304cfc808c/go.mod h1:weASp41xM3dk0YHg1s/W8ecdGP5G4teSTMBPpYAaUgA= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= +go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= +go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= +go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= go.etcd.io/etcd/v3 v3.3.0-rc.0.0.20200728214110-6c81b20ec8de/go.mod h1:UENlOa05tkNvLx9VnNziSerG4Ro74upGK6Apd4v6M/Y= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -1251,6 +1421,18 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= +go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -1274,8 +1456,10 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go4.org v0.0.0-20200104003542-c7e774b10ea0 h1:M6XsnQeLwG+rHQ+/rrGh3puBI3WZEy9TBWmf2H+enQA= go4.org v0.0.0-20200104003542-c7e774b10ea0/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1306,12 +1490,14 @@ golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392 h1:xYJJ3S178yv++9zXV/hnr29plCAGO9vAFG9dorqaFQc= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1339,8 +1525,9 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mobile v0.0.0-20190806162312-597adff16ade/go.mod h1:AlhUtkH4DA4asiFC5RgK7ZKmauvtkAVcy9L0epCzlWo= @@ -1350,13 +1537,15 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1407,8 +1596,12 @@ golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201026091529-146b70c837a4/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE= +golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1428,9 +1621,10 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1457,6 +1651,7 @@ golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1471,6 +1666,7 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1478,16 +1674,19 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1503,17 +1702,31 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo= +golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= -golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= +golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1521,20 +1734,24 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180810170437-e96c4e24768d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011152555-a398e557df60/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1602,6 +1819,7 @@ golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200616195046-dc31b401abb5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -1613,16 +1831,20 @@ golang.org/x/tools v0.0.0-20201105220310-78b158585360/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= -gomodules.xyz/jsonpatch/v2 v2.1.0 h1:Phva6wqu+xR//Njw6iorylFFgn/z547tw5Ne3HZPQ+k= gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= +gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= +gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= @@ -1682,6 +1904,7 @@ google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191220175831-5c49e3ecc1c1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -1692,6 +1915,7 @@ google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -1702,16 +1926,20 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201026171402-d4b8fe4fd877/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201030142918-24207fddd1c3/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201106154455-f9bfe239b0ba/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201203001206-6486ece9c497 h1:jDYzwXmX9tLnuG4sL85HPmE1ruErXOopALp2i/0AHnI= google.golang.org/genproto v0.0.0-20201203001206-6486ece9c497/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/grpc v1.13.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1735,8 +1963,11 @@ google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= -google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1746,8 +1977,10 @@ google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= -google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= @@ -1781,6 +2014,7 @@ gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eR gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/ldap.v2 v2.5.1/go.mod h1:oI0cpe/D7HRtBQl8aTg+ZmzFUAvu4lsv3eLXMLGFxWk= gopkg.in/natefinch/lumberjack.v2 v2.0.0-20150622162204-20b71e5b60d7/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -1814,11 +2048,13 @@ gopkg.in/yaml.v3 v3.0.0-20190709130402-674ba3eaed22/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20190905181640-827449938966/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v0.0.0-20181223230014-1083505acf35/go.mod h1:R//lfYlUuTOTfblYI3lGoAAAebUdzjvbmQsuB7Ykd90= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1827,19 +2063,22 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.2/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8c.io/kubermatic/v2 v2.16.2/go.mod h1:NdW+2mq4ynRtfZs9yPnvcnFWQpzmM7ngntW6GeuQicU= k8c.io/operating-system-manager v0.1.0 h1:2/vmpWHOLm1j3YZ0qrlaW+ucydXXS83FF7pISDoWlKs= k8c.io/operating-system-manager v0.1.0/go.mod h1:ULyZQO1irKjsQTNjIdrHld7SZ+joHjmPnOEs5Db8G8M= k8s.io/api v0.0.0-20181018013834-843ad2d9b9ae/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20181115043458-b799cb063522/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/api v0.0.0-20181203235848-2dd39edadc55/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/api v0.0.0-20190118113203-912cbe2bfef3/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20190725062911-6607c48751ae/go.mod h1:1O0xzX/RAtnm7l+5VEUxZ1ysO2ghatfq/OZED4zM9kA= k8s.io/api v0.0.0-20190918155943-95b840bb6a1f/go.mod h1:uWuOHnjmNrtQomJrvEBg0c0HRNyQ+8KTEERVsK0PW48= k8s.io/api v0.0.0-20190918195907-bd6ac527cfd2/go.mod h1:AOxZTnaXR/xiarlQL0JUfwQPxjmKDvVYoRp58cA7lUo= k8s.io/api v0.16.4/go.mod h1:AtzMnsR45tccQss5q8RnF+W8L81DH6XwXwo/joEx9u0= k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= +k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= k8s.io/api v0.17.2/go.mod h1:BS9fjjLc4CMuqfSO8vgbHPKMt5+SF0ET6u/RVDihTo4= +k8s.io/api v0.18.0-beta.2/go.mod h1:2oeNnWEqcSmaM/ibSh3t7xcIqbkGXhzZdn4ezV9T4m0= k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= k8s.io/api v0.18.4/go.mod h1:lOIQAKYgai1+vz9J7YcDZwC26Z0zQewYOGWdyIPUUQ4= @@ -1848,22 +2087,35 @@ k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw= k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= k8s.io/api v0.19.4/go.mod h1:SbtJ2aHCItirzdJ36YslycFNzWADYH3tgOhvBEFtZAk= -k8s.io/api v0.20.2 h1:y/HR22XDZY3pniu9hIFDLpUCPq2w5eQ6aV/VFQ7uJMw= +k8s.io/api v0.20.0/go.mod h1:HyLC5l5eoS/ygQYl1BXBgFzWNlkHiAuyNAbevIn+FKg= +k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= +k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= +k8s.io/api v0.22.2 h1:M8ZzAD0V6725Fjg53fKeTJxGsJvRbk4TEm/fexHMtfw= +k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= +k8s.io/apiextensions-apiserver v0.0.0-20181204003618-e419c5771cdc/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE= k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY= k8s.io/apiextensions-apiserver v0.0.0-20190918201827-3de75813f604/go.mod h1:7H8sjDlWQu89yWB3FhZfsLyRCRLuoXoCoY5qtwW1q6I= k8s.io/apiextensions-apiserver v0.16.4/go.mod h1:HYQwjujEkXmQNhap2C9YDdIVOSskGZ3et0Mvjcyjbto= +k8s.io/apiextensions-apiserver v0.17.0/go.mod h1:XiIFUakZywkUl54fVXa7QTEHcqQz9HG55nHd1DCoHj8= k8s.io/apiextensions-apiserver v0.17.2/go.mod h1:4KdMpjkEjjDI2pPfBA15OscyNldHWdBCfsWMDWAmSTs= +k8s.io/apiextensions-apiserver v0.18.0-beta.2/go.mod h1:Hnrg5jx8/PbxRbUoqDGxtQkULjwx8FDW4WYJaKNK+fk= k8s.io/apiextensions-apiserver v0.18.0/go.mod h1:18Cwn1Xws4xnWQNC00FLq1E350b9lUF+aOdIWDOZxgo= k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY= k8s.io/apiextensions-apiserver v0.18.4/go.mod h1:NYeyeYq4SIpFlPxSAB6jHPIdvu3hL0pc36wuRChybio= k8s.io/apiextensions-apiserver v0.18.6/go.mod h1:lv89S7fUysXjLZO7ke783xOwVTm6lKizADfvUM/SS/M= k8s.io/apiextensions-apiserver v0.19.0/go.mod h1:znfQxNpjqz/ZehvbfMg5N6fvBJW5Lqu5HVLTJQdP4Fs= k8s.io/apiextensions-apiserver v0.19.2/go.mod h1:EYNjpqIAvNZe+svXVx9j4uBaVhTB4C94HkY3w058qcg= -k8s.io/apiextensions-apiserver v0.19.4 h1:D9ak9T012tb3vcGFWYmbQuj9SCC8YM4zhA4XZqsAQC4= k8s.io/apiextensions-apiserver v0.19.4/go.mod h1:B9rpH/nu4JBCtuUp3zTTk8DEjZUupZTBEec7/2zNRYw= +k8s.io/apiextensions-apiserver v0.20.0/go.mod h1:ZH+C33L2Bh1LY1+HphoRmN1IQVLTShVcTojivK3N9xg= +k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= +k8s.io/apiextensions-apiserver v0.20.2/go.mod h1:F6TXp389Xntt+LUq3vw6HFOLttPa0V8821ogLGwb6Zs= +k8s.io/apiextensions-apiserver v0.22.2 h1:zK7qI8Ery7j2CaN23UCFaC1hj7dMiI87n01+nKuewd4= +k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA= k8s.io/apimachinery v0.0.0-20181015213631-60666be32c5d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/apimachinery v0.0.0-20181110190943-2a7c93004028/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/apimachinery v0.0.0-20181203235515-3d8ee2261517/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/apimachinery v0.0.0-20190118094746-1525e4dadd2d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/apimachinery v0.0.0-20190703205208-4cfb76a8bf76/go.mod h1:M2fZgZL9DbLfeJaPBCDqSqNsdsmLN+V29knYJnIXlMA= k8s.io/apimachinery v0.0.0-20190719140911-bfcf53abc9f8/go.mod h1:sBJWIJZfxLhp7mRsRyuAE/NfKTr3kXGR1iaqg8O0gJo= k8s.io/apimachinery v0.0.0-20190816221834-a9f1d8a9c101/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= @@ -1871,8 +2123,11 @@ k8s.io/apimachinery v0.0.0-20190817020851-f2f3a405f61d/go.mod h1:3jediapYqJ2w1BF k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4= k8s.io/apimachinery v0.16.4/go.mod h1:llRdnznGEAqC3DcNm6yEj472xaFVfLM7hnYofMb12tQ= k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/apimachinery v0.17.1-beta.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apimachinery v0.17.1/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apimachinery v0.17.2/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/apimachinery v0.18.0-beta.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= k8s.io/apimachinery v0.18.4/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= @@ -1881,13 +2136,20 @@ k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCk k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apimachinery v0.19.4/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= -k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= +k8s.io/apimachinery v0.20.0/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.22.2 h1:ejz6y/zNma8clPVfNDLnPbleBo6MpoFy/HBiBqCouVk= +k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= +k8s.io/apiserver v0.0.0-20181026151315-13cfe3978170/go.mod h1:6bqaTSOSJavUIXUtfaR9Os9JtTCm8ZqH2SUl2S60C4w= k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg= k8s.io/apiserver v0.0.0-20190918200908-1e17798da8c1/go.mod h1:4FuDU+iKPjdsdQSN3GsEKZLB/feQsj1y9dhhBDVV2Ns= k8s.io/apiserver v0.16.4/go.mod h1:kbLJOak655g6W7C+muqu1F76u9wnEycfKMqbVaXIdAc= k8s.io/apiserver v0.17.0/go.mod h1:ABM+9x/prjINN6iiffRVNCBR2Wk7uY4z+EtEGZD48cg= +k8s.io/apiserver v0.17.0/go.mod h1:ABM+9x/prjINN6iiffRVNCBR2Wk7uY4z+EtEGZD48cg= k8s.io/apiserver v0.17.2/go.mod h1:lBmw/TtQdtxvrTk0e2cgtOxHizXI+d0mmGQURIHQZlo= +k8s.io/apiserver v0.18.0-beta.2/go.mod h1:bnblMkMoCFnIfVnVftd0SXJPzyvrk3RtaqSbblphF/A= k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw= k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw= k8s.io/apiserver v0.18.4/go.mod h1:q+zoFct5ABNnYkGIaGQ3bcbUNdmPyOCoEBcg51LChY8= @@ -1895,19 +2157,28 @@ k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg= k8s.io/apiserver v0.19.0/go.mod h1:XvzqavYj73931x7FLtyagh8WibHpePJ1QwWrSJs2CLk= k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= k8s.io/apiserver v0.19.4/go.mod h1:X8WRHCR1UGZDd7HpV0QDc1h/6VbbpAeAGyxSh8yzZXw= +k8s.io/apiserver v0.20.0/go.mod h1:6gRIWiOkvGvQt12WTYmsiYoUyYW0FXSiMdNl4m+sxY8= +k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= +k8s.io/apiserver v0.20.2/go.mod h1:2nKd93WyMhZx4Hp3RfgH2K5PhwyTrprrkWYnI7id7jA= +k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= k8s.io/autoscaler v0.0.0-20190218140445-7f77136aeea4/go.mod h1:QEXezc9uKPT91dwqhSJq3GNI3B1HxFRQHiku9kmrsSA= k8s.io/cli-runtime v0.19.0/go.mod h1:tun9l0eUklT8IHIM0jors17KmUjcrAxn0myoBYwuNuo= k8s.io/cli-runtime v0.19.4/go.mod h1:m8G32dVbKOeaX1foGhleLEvNd6REvU7YnZyWn5//9rw= -k8s.io/client-go v0.20.2 h1:uuf+iIAbfnCSw8IGAv/Rg0giM+2bOzHLOsbbrwrdhNQ= -k8s.io/client-go v0.20.2/go.mod h1:kH5brqWqp7HDxUFKoEgiI4v8G1xzbe9giaCenUWJzgE= +k8s.io/client-go v0.22.2 h1:DaSQgs02aCC1QcwUdkKZWOeaVsQjYvWv8ZazcZ6JcHc= +k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= k8s.io/cloud-provider v0.17.0/go.mod h1:Ze4c3w2C0bRsjkBUoHpFi+qWe3ob1wI2/7cUn+YQIDE= +k8s.io/cluster-bootstrap v0.22.2/go.mod h1:ZkmQKprEqvrUccMnbRHISsMscA1dsQ8SffM9nHq6CgE= k8s.io/code-generator v0.0.0-20181114232248-ae218e241252/go.mod h1:IPqxl/YHk05nodzupwjke6ctMjyNRdV2zZ5/j3/F204= +k8s.io/code-generator v0.0.0-20181203235156-f8cba74510f3/go.mod h1:MYiN+ZJZ9HkETbgVZdWw2AsuAi9PZ4V80cwfuf2axe8= k8s.io/code-generator v0.0.0-20190612205613-18da4a14b22b/go.mod h1:G8bQwmHm2eafm5bgtX67XDZQ8CWKSGu9DekI+yN4Y5I= k8s.io/code-generator v0.0.0-20190717022600-77f3a1fe56bb/go.mod h1:cDx5jQmWH25Ff74daM7NVYty9JWw9dvIS9zT9eIubCY= k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE= +k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= k8s.io/code-generator v0.16.4/go.mod h1:mJUgkl06XV4kstAnLHAIzJPVCOzVR+ZcfPIv4fUsFCY= +k8s.io/code-generator v0.17.0/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= k8s.io/code-generator v0.17.1/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= k8s.io/code-generator v0.17.2/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= +k8s.io/code-generator v0.18.0-beta.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= k8s.io/code-generator v0.18.4/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= @@ -1915,22 +2186,33 @@ k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8 k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= k8s.io/code-generator v0.19.4/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= +k8s.io/code-generator v0.20.0/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= +k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= +k8s.io/code-generator v0.20.2/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.4/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= +k8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA= k8s.io/component-base v0.0.0-20190918200425-ed2f0867c778/go.mod h1:DFWQCXgXVLiWtzFaS17KxHdlUeUymP7FLxZSkmL9/jU= k8s.io/component-base v0.16.4/go.mod h1:GYQ+4hlkEwdlpAp59Ztc4gYuFhdoZqiAJD1unYDJ3FM= k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= +k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= k8s.io/component-base v0.17.2/go.mod h1:zMPW3g5aH7cHJpKYQ/ZsGMcgbsA/VyhEugF3QT1awLs= +k8s.io/component-base v0.18.0-beta.2/go.mod h1:HVk5FpRnyzQ/MjBr9//e/yEBjTVa2qjGXCTuUzcD7ks= k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c= k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM= k8s.io/component-base v0.18.4/go.mod h1:7jr/Ef5PGmKwQhyAz/pjByxJbC58mhKAhiaDu0vXfPk= k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14= k8s.io/component-base v0.19.0/go.mod h1:dKsY8BxkA+9dZIAh2aWJLL/UdASFDNtGYTCItL4LM7Y= k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo= -k8s.io/component-base v0.19.4 h1:HobPRToQ8KJ9ubRju6PUAk9I5V1GNMJZ4PyWbiWA0uI= k8s.io/component-base v0.19.4/go.mod h1:ZzuSLlsWhajIDEkKF73j64Gz/5o0AgON08FgRbEPI70= +k8s.io/component-base v0.20.0/go.mod h1:wKPj+RHnAr8LW2EIBIK7AxOHPde4gme2lzXwVSoRXeA= +k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= +k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= +k8s.io/component-base v0.22.2 h1:vNIvE0AIrLhjX8drH0BgCNJcR4QZxMXcJzBsDplDx9M= +k8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug= k8s.io/csi-translation-lib v0.17.0/go.mod h1:HEF7MEz7pOLJCnxabi45IPkhSsE/KmxPQksuCrHKWls= k8s.io/gengo v0.0.0-20181106084056-51747d6e00da/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20181113154421-fd15ee9cc2f7/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -1940,8 +2222,9 @@ k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v0.0.0-20190306015804-8e90cee79f82/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.1.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= @@ -1951,24 +2234,26 @@ k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.3.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= +k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/kube-aggregator v0.0.0-20181204002017-122bac39d429/go.mod h1:8sbzT4QQKDEmSCIbfqjV0sd97GpUT7A4W626sBiYJmU= +k8s.io/kube-aggregator v0.18.0-beta.2/go.mod h1:O3Td9mheraINbLHH4pzoFP2gRzG0Wk1COqzdSL4rBPk= k8s.io/kube-aggregator v0.19.0/go.mod h1:1Ln45PQggFAG8xOqWPIYMxUq8WNtpPnYsbUJ39DpF/A= k8s.io/kube-aggregator v0.19.4/go.mod h1:cTkvun110194d797AuThyydBBlgm+cKIFUeS2uzGJfU= -k8s.io/kube-openapi v0.0.0-20181114233023-0317810137be/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= -k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= -k8s.io/kube-openapi v0.0.0-20190709113604-33be087ad058/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4= -k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= -k8s.io/kube-openapi v0.0.0-20200121204235-bf4fb3bd569c/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= -k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= -k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/kube-aggregator v0.20.0/go.mod h1:3Is/gzzWmhhG/rA3CpA1+eVye87lreBQDFGcAGT7gzo= +k8s.io/kube-aggregator v0.20.2/go.mod h1:j7ks4pWm6cjXzlVZB9tewvUdg2njjbiFuHp575ZKnqc= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kubectl v0.19.0/go.mod h1:gPCjjsmE6unJzgaUNXIFGZGafiUp5jh0If3F/x7/rRg= k8s.io/kubectl v0.19.4/go.mod h1:XPmlu4DJEYgD83pvZFeKF8+MSvGnYGqunbFSrJsqHv0= -k8s.io/kubelet v0.19.4 h1:X5xd2BAJYz7i+arNgMlQSJl7r2xzpfducf4BYqn/Loo= k8s.io/kubelet v0.19.4/go.mod h1:zJnPeb7nJCRvtAwxJhe9fFCtMLXL3cXbQiczPmpDrLU= +k8s.io/kubelet v0.22.2 h1:7ol5AXXxcW97dUE8W/QiPjkXu1ZuGshG5VmgDmviZsc= +k8s.io/kubelet v0.22.2/go.mod h1:ORIRua2/wTcx5UnEvxWosu650/8fatmzbMRC7m6WjAM= +k8s.io/kubernetes v1.11.8-beta.0.0.20190124204751-3a10094374f2/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +k8s.io/kubernetes v1.14.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/legacy-cloud-providers v0.17.0/go.mod h1:DdzaepJ3RtRy+e5YhNtrCYwlgyK87j/5+Yfp0L9Syp8= k8s.io/metrics v0.19.0/go.mod h1:WykpW8B60OeAJx1imdwUgyOID2kDljr/Q+1zrPJ98Wo= k8s.io/metrics v0.19.4/go.mod h1:a0gvAzrxQPw2ouBqnXI7X9qlggpPkKAFgWU/Py+KZiU= @@ -1979,19 +2264,27 @@ k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3 k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= +k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= knative.dev/caching v0.0.0-20190719140829-2032732871ff/go.mod h1:dHXFU6CGlLlbzaWc32g80cR92iuBSpsslDNBWI8C7eg= knative.dev/eventing-contrib v0.6.1-0.20190723221543-5ce18048c08b/go.mod h1:SnXZgSGgMSMLNFTwTnpaOH7hXDzTFtw0J8OmHflNx3g= knative.dev/pkg v0.0.0-20191111150521-6d806b998379/go.mod h1:pgODObA1dTyhNoFxPZTTjNWfx6F0aKsKzn+vaT9XO/Q= kubevirt.io/client-go v0.30.0 h1:0jUvTa/Ev03lCN+Dr4mH22ipoJ9otAOkpFh6wA66b5M= kubevirt.io/client-go v0.30.0/go.mod h1:JY7hQq+SUT0aLvleXrW/+28fDfZ6BPe4E6f8FyC8jkY= -kubevirt.io/containerized-data-importer v1.10.6 h1:xkqLb48pkbdoY8gB2VDP2o+KXpO18tgQuLjcXNn0qAI= kubevirt.io/containerized-data-importer v1.10.6/go.mod h1:qF594BtRRkruyrqLwt3zbLCWdPIQNs1qWh4LR1cOzy0= +kubevirt.io/containerized-data-importer v1.40.0 h1:EjrlOxWKe/gktOC4elC1JCfbynS8erdf9SfmfXJpJ3c= +kubevirt.io/containerized-data-importer v1.40.0/go.mod h1:IlhJj5CBgyzXSfgrPBDmHqCmadEl19Vs04YE3fgtsf8= +kubevirt.io/controller-lifecycle-operator-sdk v0.2.1-0.20210723143736-64585ea1d1bd h1:QCXqLkTzaBTpPw4Onzh31I5L72orz6ItavRyc6TEe2c= +kubevirt.io/controller-lifecycle-operator-sdk v0.2.1-0.20210723143736-64585ea1d1bd/go.mod h1:ZJhLceiY2Gl5CXFGSp5eMGt/sksOiJP0289nAZFCQf0= +kubevirt.io/qe-tools v0.1.6/go.mod h1:PJyH/YXC4W0AmxfheDmXWMbLNsMSboVGXKpMAwfKzVE= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= @@ -2004,28 +2297,35 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-runtime v0.3.0/go.mod h1:Cw6PkEg0Sa7dAYovGT4R0tRkGhHXpYijwNxYhAnAZZk= sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns= sigs.k8s.io/controller-runtime v0.6.1/go.mod h1:XRYBPdbf5XJu9kpS84VJiZ7h/u1hF3gEORz0efEja7A= sigs.k8s.io/controller-runtime v0.6.2/go.mod h1:vhcq/rlnENJ09SIRp3EveTaZ0yqH526hjf9iJdbUJ/E= sigs.k8s.io/controller-runtime v0.6.3/go.mod h1:WlZNXcM0++oyaQt4B7C2lEE5JYRs8vJUzRP4N4JpdAY= -sigs.k8s.io/controller-runtime v0.7.0 h1:bU20IBBEPccWz5+zXpLnpVsgBYxqclaHu1pVDl/gEt8= sigs.k8s.io/controller-runtime v0.7.0/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= +sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= +sigs.k8s.io/controller-runtime v0.10.2 h1:jW8qiY+yMnnPx6O9hu63tgcwaKzd1yLYui+mpvClOOc= +sigs.k8s.io/controller-runtime v0.10.2/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= sigs.k8s.io/controller-tools v0.2.4/go.mod h1:m/ztfQNocGYBgTTCmFdnK94uVvgxeZeE3LtJvd/jIzA= +sigs.k8s.io/controller-tools v0.2.8/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= sigs.k8s.io/controller-tools v0.2.9-0.20200414181213-645d44dca7c0/go.mod h1:YKE/iHvcKITCljdnlqHYe+kAt7ZldvtAwUzQff0k1T0= sigs.k8s.io/controller-tools v0.4.1/go.mod h1:G9rHdZMVlBDocIxGkK3jHLWqcTMNvveypYJwrvYKjWU= +sigs.k8s.io/controller-tools v0.5.0/go.mod h1:JTsstrMpxs+9BUj6eGuAaEb6SDSPTeVtUyp0jmnAM/I= +sigs.k8s.io/kube-storage-version-migrator v0.0.3/go.mod h1:mXfSLkx9xbJHQsgNDDUZK/iQTs2tMbx/hsJlWe6Fthw= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= sigs.k8s.io/structured-merge-diff v1.0.1 h1:LOs1LZWMsz1xs77Phr/pkB4LFaavH7IVq/3+WTN9XTA= sigs.k8s.io/structured-merge-diff v1.0.1/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/hack/ci-e2e-test.sh b/hack/ci-e2e-test.sh index 3e68be68d..fff145bdd 100755 --- a/hack/ci-e2e-test.sh +++ b/hack/ci-e2e-test.sh @@ -17,6 +17,9 @@ set -euo pipefail set -o monitor +export TF_IN_AUTOMATION=true +export TF_CLI_ARGS="-no-color" + function cleanup { set +e diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml index c18b064f9..157d535d4 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml +++ b/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml @@ -11,31 +11,31 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "openstack" cloudProviderSpec: - # If empty, ca be set via OS_AUTH_URL env var + # If empty, ca be set via OS_AUTH_URL env var identityEndpoint: secretKeyRef: namespace: kube-system name: machine-controller-openstack key: identityEndpoint - # If empty, ca be set via OS_USER_NAME env var + # If empty, ca be set via OS_USER_NAME env var username: secretKeyRef: namespace: kube-system name: machine-controller-openstack key: username - # If empty, ca be set via OS_PASSWORD env var + # If empty, ca be set via OS_PASSWORD env var password: secretKeyRef: namespace: kube-system name: machine-controller-openstack key: password - # If empty, ca be set via OS_DOMAIN_NAME env var + # If empty, ca be set via OS_DOMAIN_NAME env var domainName: secretKeyRef: namespace: kube-system name: machine-controller-openstack key: domainName - # If empty, ca be set via OS_TENANT_NAME env var + # If empty, ca be set via OS_TENANT_NAME env var tenantName: secretKeyRef: namespace: kube-system diff --git a/pkg/containerruntime/containerruntime.go b/pkg/containerruntime/containerruntime.go index f15d9ee21..8cc5e291a 100644 --- a/pkg/containerruntime/containerruntime.go +++ b/pkg/containerruntime/containerruntime.go @@ -17,7 +17,7 @@ limitations under the License. package containerruntime import ( - "github.com/Masterminds/semver" + "github.com/Masterminds/semver/v3" "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) @@ -102,10 +102,10 @@ func (cfg Config) Engine(kubeletVersion *semver.Version) Engine { } ) - moreThen122, _ := semver.NewConstraint(">= 1.22") + moreThan122, _ := semver.NewConstraint(">= 1.22") switch { - case moreThen122.Check(kubeletVersion): + case moreThan122.Check(kubeletVersion): return containerd case cfg.Docker != nil: return docker diff --git a/pkg/containerruntime/docker.go b/pkg/containerruntime/docker.go index fc9d8b5b2..5f6113ffd 100644 --- a/pkg/containerruntime/docker.go +++ b/pkg/containerruntime/docker.go @@ -21,7 +21,7 @@ import ( "strings" "text/template" - "github.com/Masterminds/semver" + "github.com/Masterminds/semver/v3" "github.com/kubermatic/machine-controller/pkg/providerconfig/types" "github.com/kubermatic/machine-controller/pkg/userdata/helper" @@ -64,12 +64,6 @@ func (eng *Docker) ScriptFor(os types.OperatingSystem) (string, error) { ContainerdVersion: DefaultContainerdVersion, } - lessThen117, _ := semver.NewConstraint("< 1.17") - if lessThen117.Check(eng.kubeletVersion) { - args.DockerVersion = LegacyDockerVersion - args.ContainerdVersion = "" - } - switch os { case types.OperatingSystemAmazonLinux2: err := dockerAmazonTemplate.Execute(&buf, args) diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 314055aac..9d2c29a71 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -1217,7 +1217,7 @@ ExecStart=/opt/bin/bootstrap hostname: {{ .MachineSpec.Name }} {{- /* Never set the hostname on AWS nodes. Kubernetes(kube-proxy) requires the hostname to be the private dns name */}} {{ end }} -ssh_pwauth: no +ssh_pwauth: false {{- if .ProviderSpec.SSHPublicKeys }} ssh_authorized_keys: diff --git a/pkg/controller/nodecsrapprover/node_csr_approver.go b/pkg/controller/nodecsrapprover/node_csr_approver.go index 1f3f1701d..9f0d60128 100644 --- a/pkg/controller/nodecsrapprover/node_csr_approver.go +++ b/pkg/controller/nodecsrapprover/node_csr_approver.go @@ -26,14 +26,11 @@ import ( "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" certificatesv1 "k8s.io/api/certificates/v1" - certificatesv1beta1 "k8s.io/api/certificates/v1beta1" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/client-go/discovery" certificatesv1client "k8s.io/client-go/kubernetes/typed/certificates/v1" - certificatesv1beta1client "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" "k8s.io/klog" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -54,6 +51,14 @@ const ( authenticatedGroup = "system:authenticated" ) +var ( + allowedUsages = []certificatesv1.KeyUsage{ + certificatesv1.UsageDigitalSignature, + certificatesv1.UsageKeyEncipherment, + certificatesv1.UsageServerAuth, + } +) + type reconciler struct { client.Client // Have to use the typed client because csr approval is a subresource @@ -62,56 +67,13 @@ type reconciler struct { } func Add(mgr manager.Manager) error { - // TODO: delete whole file node_csr_approver_v1beta1.go and dynamic API groups discovery - // after we drop kubernetes 1.18 support - discoveryClient, err := discovery.NewDiscoveryClientForConfig(mgr.GetConfig()) + certClient, err := certificatesv1client.NewForConfig(mgr.GetConfig()) if err != nil { - return fmt.Errorf("failed to init discovery client: %w", err) - } - - srvGroups, err := discoveryClient.ServerGroups() - if err != nil { - return fmt.Errorf("failed to get server API groups: %w", err) - } - - certificatesVersionFound := "" - for _, group := range srvGroups.Groups { - if group.Name != "certificates.k8s.io" { - continue - } - - for _, groupVersion := range group.Versions { - if groupVersion.Version == "v1" { - certificatesVersionFound = "v1" - } - - if certificatesVersionFound == "" { - certificatesVersionFound = "v1beta1" - } - } + return fmt.Errorf("failed to create certificate client: %v", err) } - var ( - rec reconcile.Reconciler - watchType client.Object - ) - - switch certificatesVersionFound { - case "v1": - certClient, err := certificatesv1client.NewForConfig(mgr.GetConfig()) - if err != nil { - return fmt.Errorf("failed to create certificate client: %v", err) - } - rec = &reconciler{Client: mgr.GetClient(), certClient: certClient.CertificateSigningRequests()} - watchType = &certificatesv1.CertificateSigningRequest{} - case "v1beta1": - certClient, err := certificatesv1beta1client.NewForConfig(mgr.GetConfig()) - if err != nil { - return fmt.Errorf("failed to create certificate client: %v", err) - } - rec = &reconcilerv1beta1{Client: mgr.GetClient(), certClient: certClient.CertificateSigningRequests()} - watchType = &certificatesv1beta1.CertificateSigningRequest{} - } + rec := &reconciler{Client: mgr.GetClient(), certClient: certClient.CertificateSigningRequests()} + watchType := &certificatesv1.CertificateSigningRequest{} cntrl, err := controller.New(ControllerName, mgr, controller.Options{Reconciler: rec}) if err != nil { @@ -121,14 +83,6 @@ func Add(mgr manager.Manager) error { return cntrl.Watch(&source.Kind{Type: watchType}, &handler.EnqueueRequestForObject{}) } -var ( - allowedUsages = []certificatesv1.KeyUsage{ - certificatesv1.UsageDigitalSignature, - certificatesv1.UsageKeyEncipherment, - certificatesv1.UsageServerAuth, - } -) - func (r *reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { err := r.reconcile(ctx, request) if err != nil { diff --git a/pkg/controller/nodecsrapprover/node_csr_approver_v1beta1.go b/pkg/controller/nodecsrapprover/node_csr_approver_v1beta1.go deleted file mode 100644 index dda9063c6..000000000 --- a/pkg/controller/nodecsrapprover/node_csr_approver_v1beta1.go +++ /dev/null @@ -1,232 +0,0 @@ -/* -Copyright 2020 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package nodecsrapprover - -import ( - "context" - "crypto/x509" - "encoding/pem" - "fmt" - "strings" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - - certificatesv1beta1 "k8s.io/api/certificates/v1beta1" - corev1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/sets" - certificatesv1beta1client "k8s.io/client-go/kubernetes/typed/certificates/v1beta1" - "k8s.io/klog" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/reconcile" -) - -var ( - allowedUsagesV1beta1 = []certificatesv1beta1.KeyUsage{ - certificatesv1beta1.UsageDigitalSignature, - certificatesv1beta1.UsageKeyEncipherment, - certificatesv1beta1.UsageServerAuth, - } -) - -type reconcilerv1beta1 struct { - client.Client - // Have to use the typed client because csr approval is a subresource - // the dynamic client does not approve - certClient certificatesv1beta1client.CertificateSigningRequestInterface -} - -func (r *reconcilerv1beta1) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - err := r.reconcile(ctx, request) - if err != nil { - klog.Errorf("Reconciliation of request %s failed: %v", request.NamespacedName.String(), err) - } - return reconcile.Result{}, err -} - -func (r *reconcilerv1beta1) reconcile(ctx context.Context, request reconcile.Request) error { - // Get the CSR object - csr := &certificatesv1beta1.CertificateSigningRequest{} - if err := r.Get(ctx, request.NamespacedName, csr); err != nil { - if kerrors.IsNotFound(err) { - return nil - } - return err - } - klog.V(4).Infof("Reconciling CSR %s", csr.ObjectMeta.Name) - - // If CSR is approved, skip it - for _, condition := range csr.Status.Conditions { - if condition.Type == certificatesv1beta1.CertificateApproved { - klog.V(4).Infof("CSR %s already approved, skipping reconciling", csr.ObjectMeta.Name) - return nil - } - } - - // Validate the CSR object and get the node name - nodeName, err := r.validateCSRObject(csr) - if err != nil { - klog.V(4).Infof("Skipping reconciling CSR '%s' because CSR object is not valid: %v", csr.ObjectMeta.Name, err) - return nil - } - - // Get machine name for the appropriate node - machine, found, err := r.getMachineForNode(ctx, nodeName) - if err != nil { - return fmt.Errorf("failed to get machine for node '%s': %v", nodeName, err) - } - if !found { - return fmt.Errorf("no machine found for given node '%s'", nodeName) - } - - // Parse the certificate request - csrBlock, rest := pem.Decode(csr.Spec.Request) - if csrBlock == nil { - return fmt.Errorf("no certificate request found for the given CSR") - } - if len(rest) != 0 { - return fmt.Errorf("found more than one PEM encoded block in the result") - } - certRequest, err := x509.ParseCertificateRequest(csrBlock.Bytes) - if err != nil { - return err - } - - // Validate the certificate request - if err := r.validateX509CSR(csr, certRequest, machine); err != nil { - return fmt.Errorf("error validating the x509 certificate request: %v", err) - } - - // Approve CSR - klog.V(4).Infof("Approving CSR %s", csr.ObjectMeta.Name) - approvalCondition := certificatesv1beta1.CertificateSigningRequestCondition{ - Type: certificatesv1beta1.CertificateApproved, - Reason: "machine-controller NodeCSRApprover controller approved node serving cert", - Status: corev1.ConditionTrue, - } - csr.Status.Conditions = append(csr.Status.Conditions, approvalCondition) - - if _, err := r.certClient.UpdateApproval(ctx, csr, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("failed to approve CSR %q: %v", csr.Name, err) - } - - klog.Infof("Successfully approved CSR %s", csr.ObjectMeta.Name) - return nil -} - -// validateCSRObject valides the CSR object and returns name of the node that requested the certificate -func (r *reconcilerv1beta1) validateCSRObject(csr *certificatesv1beta1.CertificateSigningRequest) (string, error) { - // Get and validate the node name - if !strings.HasPrefix(csr.Spec.Username, nodeUserPrefix) { - return "", fmt.Errorf("username must have the '%s' prefix", nodeUserPrefix) - } - nodeName := strings.TrimPrefix(csr.Spec.Username, nodeUserPrefix) - if len(nodeName) == 0 { - return "", fmt.Errorf("node name is empty") - } - - // Ensure system:nodes and system:authenticated are in groups - if len(csr.Spec.Groups) < 2 { - return "", fmt.Errorf("there are less than 2 groups") - } - if !sets.NewString(csr.Spec.Groups...).HasAll(nodeGroup, authenticatedGroup) { - return "", fmt.Errorf("'%s' and/or '%s' are not in its groups", nodeGroup, authenticatedGroup) - } - - // Check are present usages matching allowed usages - if len(csr.Spec.Usages) != 3 { - return "", fmt.Errorf("there are no exactly three usages defined") - } - for _, usage := range csr. - Spec.Usages { - if !isUsageInUsageListV1beta1(usage, allowedUsagesV1beta1) { - return "", fmt.Errorf("usage %v is not in the list of allowed usages (%v)", usage, allowedUsages) - } - } - - return nodeName, nil -} - -// validateX509CSR validates the certificate request by comparing CN with username, -// and organization with groups. -func (r *reconcilerv1beta1) validateX509CSR(csr *certificatesv1beta1.CertificateSigningRequest, certReq *x509.CertificateRequest, machine v1alpha1.Machine) error { - // Validate Subject CommonName - if certReq.Subject.CommonName != csr.Spec.Username { - return fmt.Errorf("commonName '%s' is different then CSR username '%s'", certReq.Subject.CommonName, csr.Spec.Username) - } - - // Validate Subject Organization - if len(certReq.Subject.Organization) != 1 { - return fmt.Errorf("expected only one organization but got %d instead", len(certReq.Subject.Organization)) - } - if certReq.Subject.Organization[0] != nodeGroup { - return fmt.Errorf("organization '%s' doesn't match node group '%s'", certReq.Subject.Organization[0], nodeGroup) - } - - machineAddressSet := sets.NewString(machine.Status.NodeRef.Name) - for _, addr := range machine.Status.Addresses { - machineAddressSet.Insert(addr.Address) - } - - // Validate SAN DNS names - for _, dns := range certReq.DNSNames { - if len(dns) == 0 { - continue - } - if !machineAddressSet.Has(dns) { - return fmt.Errorf("dns name '%s' cannot be associated with node '%s'", dns, machine.Status.NodeRef.Name) - } - } - - // Validate SAN IP addresses - for _, ip := range certReq.IPAddresses { - if len(ip) == 0 { - continue - } - if !machineAddressSet.Has(ip.String()) { - return fmt.Errorf("ip address '%v' cannot be associated with node '%s'", ip, machine.Status.NodeRef.Name) - } - } - - return nil -} - -func (r *reconcilerv1beta1) getMachineForNode(ctx context.Context, nodeName string) (v1alpha1.Machine, bool, error) { - // List all Machines in all namespaces - machines := &v1alpha1.MachineList{} - if err := r.Client.List(ctx, machines); err != nil { - return v1alpha1.Machine{}, false, fmt.Errorf("failed to list all machine objects: %v", err) - } - - for _, machine := range machines.Items { - if machine.Status.NodeRef != nil && machine.Status.NodeRef.Name == nodeName { - return machine, true, nil - } - } - - return v1alpha1.Machine{}, false, fmt.Errorf("failed to get machine for given node name '%s'", nodeName) -} - -func isUsageInUsageListV1beta1(usage certificatesv1beta1.KeyUsage, usageList []certificatesv1beta1.KeyUsage) bool { - for _, usageListItem := range usageList { - if usage == usageListItem { - return true - } - } - return false -} diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index 1adbf8127..554dbd415 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -26,7 +26,7 @@ import ( "strings" "text/template" - "github.com/Masterminds/semver" + "github.com/Masterminds/semver/v3" "github.com/kubermatic/machine-controller/pkg/apis/plugin" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -142,7 +142,7 @@ package_upgrade: true package_reboot_if_required: true {{- end }} -ssh_pwauth: no +ssh_pwauth: false {{- if ne (len .ProviderSpec.SSHPublicKeys) 0 }} ssh_authorized_keys: diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index e78e4ae27..e67cccec0 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -100,107 +100,98 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.16-aws", + name: "kubelet-v1.19-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.16.16", + Kubelet: "1.19.15", }, }, }, { - name: "kubelet-v1.17-aws", + name: "kubelet-v1.20-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.17.16", + Kubelet: "1.20.11", }, }, }, { - name: "kubelet-v1.17-aws-external", + name: "containerd-kubelet-v1.20-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.17.16", + Kubelet: "1.20.11", }, }, - externalCloudProvider: true, + containerruntime: "containerd", }, { - name: "kubelet-v1.17-vsphere", + name: "kubelet-v1.21-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.17.16", + Kubelet: "1.21.5", }, }, - cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.17-vsphere-proxy", + name: "kubelet-v1.21-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.17.16", + Kubelet: "1.21.5", }, }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128/", - noProxy: "192.168.1.0", - insecureRegistries: []string{"192.168.100.100:5000", "10.0.0.1:5000"}, - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", + externalCloudProvider: true, }, { - name: "kubelet-v1.17-vsphere-mirrors", + name: "kubelet-v1.21-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.17.16", + Kubelet: "1.21.5", }, }, cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128/", - noProxy: "192.168.1.0", - registryMirrors: []string{"https://registry.docker-cn.com/"}, - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.18-aws", + name: "kubelet-v1.21-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.18.14", - }, - }, - }, - { - name: "kubelet-v1.19-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.19.4", + Kubelet: "1.21.5", }, }, + cloudProviderName: stringPtr("vsphere"), + httpProxy: "http://192.168.100.100:3128/", + noProxy: "192.168.1.0", + insecureRegistries: []string{"192.168.100.100:5000", "10.0.0.1:5000"}, + pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.20-aws", + name: "kubelet-v1.21-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.20.1", + Kubelet: "1.21.5", }, }, + cloudProviderName: stringPtr("vsphere"), + httpProxy: "http://192.168.100.100:3128/", + noProxy: "192.168.1.0", + registryMirrors: []string{"https://registry.docker-cn.com/"}, + pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "containerd-kubelet-v1.20-aws", + name: "kubelet-v1.22-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.20.1", + Kubelet: "1.22.2", }, }, - containerruntime: "containerd", }, } diff --git a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml index c942b97ac..bd5a7d25f 100644 --- a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml @@ -1,7 +1,7 @@ #cloud-config -ssh_pwauth: no +ssh_pwauth: false write_files: @@ -143,7 +143,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" + KUBE_VERSION="${KUBE_VERSION:-v1.20.11}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -312,12 +312,15 @@ write_files: ephemeral-storage: 1Gi memory: 200Mi logging: {} + memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s protectKernelDefaults: true rotateCertificates: true runtimeRequestTimeout: 0s serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s staticPodPath: /etc/kubernetes/manifests streamingConnectionIdleTimeout: 0s syncFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml deleted file mode 100644 index 231b4f456..000000000 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.16-aws.yaml +++ /dev/null @@ -1,408 +0,0 @@ -#cloud-config - - -ssh_pwauth: no - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - cpuManagerReconcilePeriod: 0s - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml index 9b8931534..76d1bc1f9 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml @@ -1,7 +1,7 @@ #cloud-config -ssh_pwauth: no +ssh_pwauth: false write_files: @@ -140,7 +140,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.19.4}" + KUBE_VERSION="${KUBE_VERSION:-v1.19.15}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -309,12 +309,15 @@ write_files: ephemeral-storage: 1Gi memory: 200Mi logging: {} + memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s protectKernelDefaults: true rotateCertificates: true runtimeRequestTimeout: 0s serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s staticPodPath: /etc/kubernetes/manifests streamingConnectionIdleTimeout: 0s syncFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml index 379798b1f..6a4e85c10 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml @@ -1,7 +1,7 @@ #cloud-config -ssh_pwauth: no +ssh_pwauth: false write_files: @@ -140,7 +140,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" + KUBE_VERSION="${KUBE_VERSION:-v1.20.11}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -309,12 +309,15 @@ write_files: ephemeral-storage: 1Gi memory: 200Mi logging: {} + memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s protectKernelDefaults: true rotateCertificates: true runtimeRequestTimeout: 0s serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s staticPodPath: /etc/kubernetes/manifests streamingConnectionIdleTimeout: 0s syncFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml similarity index 98% rename from pkg/userdata/amzn2/testdata/kubelet-v1.17-aws-external.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml index ee92a26bc..5876eaf44 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml @@ -1,7 +1,7 @@ #cloud-config -ssh_pwauth: no +ssh_pwauth: false write_files: @@ -140,7 +140,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -309,12 +309,15 @@ write_files: ephemeral-storage: 1Gi memory: 200Mi logging: {} + memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s protectKernelDefaults: true rotateCertificates: true runtimeRequestTimeout: 0s serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s staticPodPath: /etc/kubernetes/manifests streamingConnectionIdleTimeout: 0s syncFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml similarity index 98% rename from pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml index 7ccb62d5f..36972a6bf 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.18-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml @@ -1,7 +1,7 @@ #cloud-config -ssh_pwauth: no +ssh_pwauth: false write_files: @@ -140,7 +140,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.18.14}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -309,12 +309,15 @@ write_files: ephemeral-storage: 1Gi memory: 200Mi logging: {} + memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s protectKernelDefaults: true rotateCertificates: true runtimeRequestTimeout: 0s serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s staticPodPath: /etc/kubernetes/manifests streamingConnectionIdleTimeout: 0s syncFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml index e036edb8d..0d8ea0dac 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -3,7 +3,7 @@ hostname: node1 -ssh_pwauth: no +ssh_pwauth: false write_files: - path: "/etc/environment" @@ -153,7 +153,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -326,12 +326,15 @@ write_files: ephemeral-storage: 1Gi memory: 200Mi logging: {} + memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s protectKernelDefaults: true rotateCertificates: true runtimeRequestTimeout: 0s serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s staticPodPath: /etc/kubernetes/manifests streamingConnectionIdleTimeout: 0s syncFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml index bce80e1c9..8b9e3525e 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -3,7 +3,7 @@ hostname: node1 -ssh_pwauth: no +ssh_pwauth: false write_files: - path: "/etc/environment" @@ -153,7 +153,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -326,12 +326,15 @@ write_files: ephemeral-storage: 1Gi memory: 200Mi logging: {} + memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s protectKernelDefaults: true rotateCertificates: true runtimeRequestTimeout: 0s serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s staticPodPath: /etc/kubernetes/manifests streamingConnectionIdleTimeout: 0s syncFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml similarity index 98% rename from pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml index 026b0874c..9bc2df93d 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml @@ -3,7 +3,7 @@ hostname: node1 -ssh_pwauth: no +ssh_pwauth: false write_files: @@ -145,7 +145,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.17.16}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -317,12 +317,15 @@ write_files: ephemeral-storage: 1Gi memory: 200Mi logging: {} + memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s protectKernelDefaults: true rotateCertificates: true runtimeRequestTimeout: 0s serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s staticPodPath: /etc/kubernetes/manifests streamingConnectionIdleTimeout: 0s syncFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml similarity index 91% rename from pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml index 30c2bc6d3..7804790e6 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.17-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml @@ -1,7 +1,7 @@ #cloud-config -ssh_pwauth: no +ssh_pwauth: false write_files: @@ -81,22 +81,25 @@ write_files: ipvsadm - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + mkdir -p /etc/systemd/system/containerd.service.d - cat < Date: Wed, 10 Nov 2021 18:34:52 +0100 Subject: [PATCH 019/489] change flatcar image (#1098) Signed-off-by: Moath Qasim --- test/e2e/provisioning/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 03584c354..d8bdb5493 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -52,7 +52,7 @@ var ( string(providerconfigtypes.OperatingSystemUbuntu): "machine-controller-e2e-ubuntu-20-04", string(providerconfigtypes.OperatingSystemCentOS): "machine-controller-e2e-centos", string(providerconfigtypes.OperatingSystemRHEL): "machine-controller-e2e-rhel", - string(providerconfigtypes.OperatingSystemFlatcar): "Flatcar Stable (2021-10-27)", + string(providerconfigtypes.OperatingSystemFlatcar): "machine-controller-e2e-flatcar-stable-2983", } ) From a147ec057350befad2aa2f7b8d7d1a98e4ce0a4a Mon Sep 17 00:00:00 2001 From: Vincent Gramer Date: Fri, 12 Nov 2021 12:31:04 +0100 Subject: [PATCH 020/489] Openstack support auth with projectName/ projectID (#1091) * refactoring: store tenantName and tenantID under ProjectName and ProjectID Signed-off-by: Vincent Gramer * openstack has deprecated OS_TENANT_NAME/OS_TENANT_ID in favor or OS_PROJECT_NAME/OS_PROJECT_ID, but these variables contain the same value. The cloudProviderSpec has been updated to add projectName and projectID. We first try to read project variables. If they are not defined, we fall back to tenant variables. Internally (ie Config struct), the values are stored under Project variables, even if the user has specified it in tenant vars under cloudProviderSpec Signed-off-by: Vincent Gramer * apply gimps Signed-off-by: Vincent Gramer --- .prow.yaml | 23 +++++++ cmd/machine-controller/main.go | 2 +- docs/cloud-provider.md | 10 ++- examples/openstack-machinedeployment.yaml | 25 ++++++++ .../provider/openstack/provider.go | 56 +++++++++++----- .../provider/openstack/provider_test.go | 64 ++++++++++++++++++- .../provider/openstack/types/cloudconfig.go | 19 ++++-- .../openstack/types/cloudconfig_test.go | 48 +++++++------- .../provider/openstack/types/types.go | 2 + pkg/providerconfig/types_test.go | 4 +- test/e2e/provisioning/all_e2e_test.go | 32 ++++++++++ ...hinedeployment-openstack-project-auth.yaml | 49 ++++++++++++++ 12 files changed, 281 insertions(+), 53 deletions(-) create mode 100644 test/e2e/provisioning/testdata/machinedeployment-openstack-project-auth.yaml diff --git a/.prow.yaml b/.prow.yaml index 5eb765939..6dc74529c 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -293,6 +293,29 @@ presubmits: memory: 1Gi cpu: 500m + - name: pull-machine-controller-e2e-openstack-project-auth + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-openstack: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-rhel: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.17.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestOpenstackProjectAuthProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + - name: pull-machine-controller-e2e-aws always_run: true decorate: true diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 33590d281..45660de34 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -28,6 +28,7 @@ import ( "github.com/docker/distribution/reference" "github.com/prometheus/client_golang/prometheus" + osmv1alpha1 "k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1/migrations" @@ -43,7 +44,6 @@ import ( machinesv1alpha1 "github.com/kubermatic/machine-controller/pkg/machines/v1alpha1" "github.com/kubermatic/machine-controller/pkg/node" "github.com/kubermatic/machine-controller/pkg/signals" - osmv1alpha1 "k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/types" diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index d79f1c88d..b26359073 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -101,8 +101,14 @@ username: "" password: "" # the openstack domain domainName: "default" -# tenant name -tenantName: "" +# project name +projectName: "" +# project id +projectID: "" +# tenant name (deprecated, should use projectName) +tenantName: "" +# tenant Id (deprecated, should use projectID) +tenantID: "" # image to use (currently only ubuntu is supported) image: "Ubuntu 18.04 amd64" # instance flavor diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index de2552703..cf8ea8267 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -12,6 +12,9 @@ stringData: password: << OS_PASSWORD >> domainName: << OS_DOMAIN_NAME >> tenantName: << OS_TENANT_NAME >> + tenantID: << OS_TENANT_ID >> + projectName: << OS_PROJECT_NAME >> + projectID: << OS_PROJECT_ID >> region: << OS_REGION_NAME >> instanceReadyCheckPeriod: << INSTANCE_READY_CHECK_PERIOD >> instanceReadyCheckTimeout: << INSTANCE_READY_TIMEOUT >> @@ -88,12 +91,34 @@ spec: namespace: kube-system name: machine-controller-openstack key: domainName + + # --- WARN: You should define either projectName or tenantName. if both are defined, tenantName is ignored --- + # If empty, can be set via OS_PROJECT_NAME env var + projectName: + secretKeyRef: + namespace: kube-system + name: machine-controller-openstack + key: projectName # If empty, can be set via OS_TENANT_NAME env var tenantName: secretKeyRef: namespace: kube-system name: machine-controller-openstack key: tenantName + # --- WARN: You should define either projectID or tenantID. if both are defined, tenantID is ignored --- + # If empty, can be set via OS_PROJECT_ID env var + projectID: + secretKeyRef: + namespace: kube-system + name: machine-controller-openstack + key: projectID + # If empty, can be set via OS_TENANT_ID env var + tenantID: + secretKeyRef: + namespace: kube-system + name: machine-controller-openstack + key: tenantID + # Only required if there is more than one region to choose from region: secretKeyRef: diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index d96225818..8f60f6d69 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -85,8 +85,8 @@ type Config struct { Username string Password string DomainName string - TenantName string - TenantID string + ProjectName string + ProjectID string TokenID string Region string ComputeAPIVersion string @@ -118,6 +118,28 @@ const ( // Protects floating ip assignment var floatingIPAssignLock = &sync.Mutex{} +// Get the Project name from config or env var. If not defined fallback to tenant name +func (p *provider) getProjectNameOrTenantName(rawConfig *openstacktypes.RawConfig) (string, error) { + projectName, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProjectName, "OS_PROJECT_NAME") + if err == nil && len(projectName) > 0 { + return projectName, nil + } + + //fallback to tenantName + return p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.TenantName, "OS_TENANT_NAME") +} + +// Get the Project id from config or env var. If not defined fallback to tenant id +func (p *provider) getProjectIDOrTenantID(rawConfig *openstacktypes.RawConfig) (string, error) { + projectID, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProjectID, "OS_PROJECT_ID") + if err == nil && len(projectID) > 0 { + return projectID, nil + } + + //fallback to tenantName + return p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.TenantID, "OS_TENANT_ID") +} + func (p *provider) getConfigAuth(c *Config, rawConfig *openstacktypes.RawConfig) error { var err error c.ApplicationCredentialID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ApplicationCredentialID, "OS_APPLICATION_CREDENTIAL_ID") @@ -140,13 +162,13 @@ func (p *provider) getConfigAuth(c *Config, rawConfig *openstacktypes.RawConfig) if err != nil { return fmt.Errorf("failed to get the value of \"password\" field, error = %v", err) } - c.TenantName, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.TenantName, "OS_TENANT_NAME") + c.ProjectName, err = p.getProjectNameOrTenantName(rawConfig) if err != nil { - return fmt.Errorf("failed to get the value of \"tenantName\" field, error = %v", err) + return fmt.Errorf("failed to get the value of \"projectName\" field or fallback to \"tenantName\" field, error = %v", err) } - c.TenantID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.TenantID, "OS_TENANT_ID") + c.ProjectID, err = p.getProjectIDOrTenantID(rawConfig) if err != nil { - return fmt.Errorf("failed to get the value of \"tenantID\" field, error = %v", err) + return fmt.Errorf("failed to get the value of \"projectID\" or fallback to\"tenantID\" field, error = %v", err) } return nil } @@ -279,12 +301,14 @@ func setProviderSpec(rawConfig openstacktypes.RawConfig, s v1alpha1.ProviderSpec func getClient(c *Config) (*gophercloud.ProviderClient, error) { opts := gophercloud.AuthOptions{ - IdentityEndpoint: c.IdentityEndpoint, - Username: c.Username, - Password: c.Password, - DomainName: c.DomainName, - TenantName: c.TenantName, - TenantID: c.TenantID, + IdentityEndpoint: c.IdentityEndpoint, + Username: c.Username, + Password: c.Password, + DomainName: c.DomainName, + // gophercloud internally store projectName/projectID under tenantName/TenantID. We store it under projectName + // to be coherent with KPP code + TenantName: c.ProjectName, + TenantID: c.ProjectID, TokenID: c.TokenID, ApplicationCredentialID: c.ApplicationCredentialID, ApplicationCredentialSecret: c.ApplicationCredentialSecret, @@ -408,8 +432,8 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { return errors.New("password must be configured") } - if c.TenantID == "" && c.TenantName == "" { - return errors.New("either tenantID or tenantName must be configured") + if c.ProjectID == "" && c.ProjectName == "" { + return errors.New("either projectID / tenantID or projectName / tenantName must be configured") } } else { if c.ApplicationCredentialSecret == "" { @@ -814,8 +838,8 @@ func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, nam Username: c.Username, Password: c.Password, DomainName: c.DomainName, - TenantName: c.TenantName, - TenantID: c.TenantID, + ProjectName: c.ProjectName, + ProjectID: c.ProjectID, Region: c.Region, ApplicationCredentialSecret: c.ApplicationCredentialSecret, ApplicationCredentialID: c.ApplicationCredentialID, diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index 72e3c8559..8bebddd7e 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -31,10 +31,12 @@ import ( th "github.com/gophercloud/gophercloud/testhelper" "github.com/gophercloud/gophercloud/testhelper/client" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidertesting "github.com/kubermatic/machine-controller/pkg/cloudprovider/testing" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/providerconfig" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/pointer" fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -145,6 +147,10 @@ type openstackProviderSpecConf struct { RootDiskVolumeType string ApplicationCredentialID string ApplicationCredentialSecret string + ProjectName string + ProjectID string + TenantID string + TenantName string ComputeAPIVersion string } @@ -185,8 +191,14 @@ func (o openstackProviderSpecConf) rawProviderSpec(t *testing.T) []byte { "applicationCredentialID": "{{ .ApplicationCredentialID }}", "applicationCredentialSecret": "{{ .ApplicationCredentialSecret }}", {{- else }} - "tenantID": "", - "tenantName": "eu-de", + {{ if .ProjectID }} + "projectID": "{{ .ProjectID }}", + "projectName": "{{ .ProjectName }}", + {{- end }} + {{- if .TenantID }} + "tenantID": "{{ .TenantID }}", + "tenantName": "{{ .TenantName }}", + {{- end }} "username": "dummy", "password": "this_is_a_password", {{- end }} @@ -292,6 +304,54 @@ func TestCreateServer(t *testing.T) { } } +func TestProjectAuthVarsAreCorrectlyLoaded(t *testing.T) { + tests := []struct { + name string + expectedName string + expectedID string + specConf openstackProviderSpecConf + }{ + { + name: "Project auth vars should be when tenant vars are not defined", + expectedID: "the_project_id", + expectedName: "the_project_name", + specConf: openstackProviderSpecConf{ProjectID: "the_project_id", ProjectName: "the_project_name"}, + }, + { + name: "Project auth vars should be used even if tenant vars are defined", + expectedID: "the_project_id", + expectedName: "the_project_name", + specConf: openstackProviderSpecConf{ProjectID: "the_project_id", ProjectName: "the_project_name", TenantID: "the_tenant_id", TenantName: "the_tenant_name"}, + }, + { + name: "Tenant auth vars should be used when project vars are not defined", + expectedID: "the_tenant_id", + expectedName: "the_tenant_name", + specConf: openstackProviderSpecConf{TenantID: "the_tenant_id", TenantName: "the_tenant_name"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &provider{ + // Note that configVarResolver is not used in this test as the getConfigFunc is mocked. + configVarResolver: providerconfig.NewConfigVarResolver(context.Background(), fakeclient.NewFakeClient()), + } + conf, _, _, _ := p.getConfig(v1alpha1.ProviderSpec{ + Value: &runtime.RawExtension{ + Raw: tt.specConf.rawProviderSpec(t), + }, + }) + + if conf.ProjectID != tt.expectedID { + t.Errorf("ProjectID = %v, wanted %v", conf.ProjectID, tt.expectedID) + } + if conf.ProjectName != tt.expectedName { + t.Errorf("ProjectName = %v, wanted %v", conf.ProjectName, tt.expectedName) + } + }) + } +} + type ServerResponse struct { Server servers.Server `json:"server"` } diff --git a/pkg/cloudprovider/provider/openstack/types/cloudconfig.go b/pkg/cloudprovider/provider/openstack/types/cloudconfig.go index 13d1123ad..917665d9f 100644 --- a/pkg/cloudprovider/provider/openstack/types/cloudconfig.go +++ b/pkg/cloudprovider/provider/openstack/types/cloudconfig.go @@ -39,8 +39,8 @@ application-credential-secret = {{ .Global.ApplicationCredentialSecret | iniEsca {{- else }} username = {{ .Global.Username | iniEscape }} password = {{ .Global.Password | iniEscape }} -tenant-name = {{ .Global.TenantName | iniEscape }} -tenant-id = {{ .Global.TenantID | iniEscape }} +tenant-name = {{ .Global.ProjectName | iniEscape }} +tenant-id = {{ .Global.ProjectID | iniEscape }} {{- end }} domain-name = {{ .Global.DomainName | iniEscape }} region = {{ .Global.Region | iniEscape }} @@ -104,10 +104,17 @@ type GlobalOpts struct { Password string ApplicationCredentialID string `gcfg:"application-credential-id"` ApplicationCredentialSecret string `gcfg:"application-credential-secret"` - TenantName string `gcfg:"tenant-name"` - TenantID string `gcfg:"tenant-id"` - DomainName string `gcfg:"domain-name"` - Region string + + // project name formerly known as tenant name. + // it serialized as tenant-name because openstack CCM reads only tenant-name. In CCM, internally project and tenant + // are stored into tenant-name. + ProjectName string `gcfg:"tenant-name"` + + // project id formerly known as tenant id. + // serialized as tenant-id for same reason as ProjectName + ProjectID string `gcfg:"tenant-id"` + DomainName string `gcfg:"domain-name"` + Region string } // CloudConfig is used to read and store information from the cloud configuration file diff --git a/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go b/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go index c7af4cedf..8414bacc2 100644 --- a/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go +++ b/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go @@ -40,12 +40,12 @@ func TestCloudConfigToString(t *testing.T) { name: "simple-config", config: &CloudConfig{ Global: GlobalOpts{ - AuthURL: "https://127.0.0.1:8443/", - Username: "admin", - Password: "password", - DomainName: "Default", - TenantName: "Test", - Region: "eu-central1", + AuthURL: "https://127.0.0.1:8443/", + Username: "admin", + Password: "password", + DomainName: "Default", + ProjectName: "Test", + Region: "eu-central1", }, BlockStorage: BlockStorageOpts{ BSVersion: "v2", @@ -63,12 +63,12 @@ func TestCloudConfigToString(t *testing.T) { name: "use-octavia-explicitly-enabled", config: &CloudConfig{ Global: GlobalOpts{ - AuthURL: "https://127.0.0.1:8443/", - Username: "admin", - Password: "password", - DomainName: "Default", - TenantName: "Test", - Region: "eu-central1", + AuthURL: "https://127.0.0.1:8443/", + Username: "admin", + Password: "password", + DomainName: "Default", + ProjectName: "Test", + Region: "eu-central1", }, BlockStorage: BlockStorageOpts{ BSVersion: "v2", @@ -87,12 +87,12 @@ func TestCloudConfigToString(t *testing.T) { name: "use-octavia-explicitly-disabled", config: &CloudConfig{ Global: GlobalOpts{ - AuthURL: "https://127.0.0.1:8443/", - Username: "admin", - Password: "password", - DomainName: "Default", - TenantName: "Test", - Region: "eu-central1", + AuthURL: "https://127.0.0.1:8443/", + Username: "admin", + Password: "password", + DomainName: "Default", + ProjectName: "Test", + Region: "eu-central1", }, BlockStorage: BlockStorageOpts{ BSVersion: "v2", @@ -111,12 +111,12 @@ func TestCloudConfigToString(t *testing.T) { name: "config-with-special-chars", config: &CloudConfig{ Global: GlobalOpts{ - AuthURL: "https://127.0.0.1:8443/", - Username: "admin", - Password: `.)\^x[tt0L@};p>=%s", osAuthURL), + fmt.Sprintf("<< USERNAME >>=%s", osUsername), + fmt.Sprintf("<< PASSWORD >>=%s", osPassword), + fmt.Sprintf("<< DOMAIN_NAME >>=%s", osDomain), + fmt.Sprintf("<< REGION >>=%s", osRegion), + fmt.Sprintf("<< PROJECT_NAME >>=%s", osProject), + fmt.Sprintf("<< NETWORK_NAME >>=%s", osNetwork), + } + + selector := OsSelector("ubuntu") + runScenarios(t, selector, params, OSManifestProjectAuth, fmt.Sprintf("os-%s", *testRunIdentifier)) +} + // TestDigitalOceanProvisioning - a test suite that exercises digital ocean provider // by requesting nodes with different combination of container runtime type, container runtime version and the OS flavour. // diff --git a/test/e2e/provisioning/testdata/machinedeployment-openstack-project-auth.yaml b/test/e2e/provisioning/testdata/machinedeployment-openstack-project-auth.yaml new file mode 100644 index 000000000..a97d47e6b --- /dev/null +++ b/test/e2e/provisioning/testdata/machinedeployment-openstack-project-auth.yaml @@ -0,0 +1,49 @@ +apiVersion: "cluster.k8s.io/v1alpha1" +kind: MachineDeployment +metadata: + name: << MACHINE_NAME >> + namespace: kube-system +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + name: << MACHINE_NAME >> + template: + metadata: + labels: + name: << MACHINE_NAME >> + spec: + providerSpec: + value: + sshPublicKeys: + - "<< YOUR_PUBLIC_KEY >>" + cloudProvider: "openstack" + cloudProviderSpec: + identityEndpoint: "<< IDENTITY_ENDPOINT >>" + username: "<< USERNAME >>" + password: "<< PASSWORD >>" + projectName: "<< PROJECT_NAME >>" + image: "<< OS_IMAGE >>" + flavor: "m1.tiny" + floatingIpPool: "" + domainName: "<< DOMAIN_NAME >>" + region: "<< REGION >>" + network: "<< NETWORK_NAME >>" + instanceReadyCheckPeriod: 5s + instanceReadyCheckTimeout: 2m + operatingSystem: "<< OS_NAME >>" + operatingSystemSpec: + distUpgradeOnBoot: false + disableAutoUpdate: true + # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` + rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" + # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` + rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" + rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" + versions: + kubelet: "<< KUBERNETES_VERSION >>" From e6aef335cbee5e9fe9e5966da7ed4c93fc580d25 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Mon, 15 Nov 2021 05:42:59 +0100 Subject: [PATCH 021/489] Add Marvin to OWNERS_ALIASES (#1097) Signed-off-by: Marvin Beckers --- OWNERS_ALIASES | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 7980289a8..e3ba2e465 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -2,6 +2,7 @@ aliases: machine-controller-maintainers: + - embik - kron4eg - mfranczy - moadqassem From e73150c2b207e0b367de1a4adbf783ad77afa21a Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 15 Nov 2021 20:32:05 +0500 Subject: [PATCH 022/489] Add Waleed as project owner (#1101) --- OWNERS_ALIASES | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index e3ba2e465..9c55374f6 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -2,6 +2,7 @@ aliases: machine-controller-maintainers: + - ahmedwaleedmalik - embik - kron4eg - mfranczy From 5b7732906c35e4b8fd023f24aa99be44a357ea6b Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Mon, 15 Nov 2021 17:23:01 +0100 Subject: [PATCH 023/489] Include optional subscription attachment in RHEL MachineDeployment (#1100) * add support for auto attach in rhel Signed-off-by: Moath Qasim * update fixtures Signed-off-by: Moath Qasim * address PR reviews Signed-off-by: Moath Qasim --- pkg/userdata/rhel/provider.go | 2 +- pkg/userdata/rhel/rhel.go | 1 + pkg/userdata/rhel/testdata/kubelet-containerd-v1.19-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.22-aws-external.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-mirrors.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-proxy.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere.yaml | 2 +- .../testdata/machinedeployment-vsphere-datastore-cluster.yaml | 1 + test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml | 1 + 13 files changed, 13 insertions(+), 10 deletions(-) diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index e80cda1c2..46828984d 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -363,7 +363,7 @@ rh_subscription: {{- else }} username: "{{.OSConfig.RHELSubscriptionManagerUser}}" password: "{{.OSConfig.RHELSubscriptionManagerPassword}}" - auto-attach: true + auto-attach: {{.OSConfig.AttachSubscription}} {{- end }} runcmd: diff --git a/pkg/userdata/rhel/rhel.go b/pkg/userdata/rhel/rhel.go index 285b5bb50..be6431a58 100644 --- a/pkg/userdata/rhel/rhel.go +++ b/pkg/userdata/rhel/rhel.go @@ -28,6 +28,7 @@ type Config struct { RHELSubscriptionManagerUser string `json:"rhelSubscriptionManagerUser,omitempty"` RHELSubscriptionManagerPassword string `json:"rhelSubscriptionManagerPassword,omitempty"` RHSMOfflineToken string `json:"rhsmOfflineToken,omitempty"` + AttachSubscription bool `json:"attachSubscription"` RHELUseSatelliteServer bool `json:"rhelUseSatelliteServer"` RHELSatelliteServer string `json:"rhelSatelliteServer"` RHELOrganizationName string `json:"rhelOrganizationName"` diff --git a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.19-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.19-aws.yaml index f4e220e38..791db14cf 100644 --- a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.19-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.19-aws.yaml @@ -460,7 +460,7 @@ write_files: rh_subscription: username: "" password: "" - auto-attach: true + auto-attach: false runcmd: - systemctl start setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml index fe39ae429..86e5e7413 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml @@ -444,7 +444,7 @@ write_files: rh_subscription: username: "" password: "" - auto-attach: true + auto-attach: false runcmd: - systemctl start setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml index e5970572c..163e3b9a3 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml @@ -444,7 +444,7 @@ write_files: rh_subscription: username: "" password: "" - auto-attach: true + auto-attach: false runcmd: - systemctl start setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml index ba9eac9f7..c3fb9ed80 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml @@ -444,7 +444,7 @@ write_files: rh_subscription: username: "" password: "" - auto-attach: true + auto-attach: false runcmd: - systemctl start setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws-external.yaml index 3aa2038a2..f1bf1bed4 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws-external.yaml @@ -460,7 +460,7 @@ write_files: rh_subscription: username: "" password: "" - auto-attach: true + auto-attach: false runcmd: - systemctl start setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index 8da895c40..9650e5ade 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -460,7 +460,7 @@ write_files: rh_subscription: username: "" password: "" - auto-attach: true + auto-attach: false runcmd: - systemctl start setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-mirrors.yaml index 56e2a09c5..2d48e9e45 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-mirrors.yaml @@ -478,7 +478,7 @@ write_files: rh_subscription: username: "" password: "" - auto-attach: true + auto-attach: false runcmd: - systemctl start setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-proxy.yaml index 71c2808f0..6b1771de8 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-proxy.yaml @@ -482,7 +482,7 @@ write_files: rh_subscription: username: "" password: "" - auto-attach: true + auto-attach: false runcmd: - systemctl start setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere.yaml index 0583b2fab..6831995a1 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere.yaml @@ -469,7 +469,7 @@ write_files: rh_subscription: username: "" password: "" - auto-attach: true + auto-attach: false runcmd: - systemctl start setup.service diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml index c47004af4..a92caf029 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml @@ -41,6 +41,7 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false disableAutoUpdate: true + attachSubscription: false # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml index c19942254..f45f4ff0d 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml @@ -40,6 +40,7 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false disableAutoUpdate: true + attachSubscription: false # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` From 3c24b572d8c0853ccc037c3d3bd32a05e0c51154 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 15 Nov 2021 22:47:07 +0500 Subject: [PATCH 024/489] Add bootstrap config for ignition (#1096) * Generate bootstrap config for ignition * Add bootstrapping logic for ignition * refactor code; fix ignition config errors * refactored code; indentation fixes in bootstrap.go * refactored caode * refactored code * fix lint errors * update ignition bootstrap script --- .gitignore | 1 + pkg/controller/machine/bootstrap.go | 261 +++++++++++++++++++ pkg/controller/machine/machine_controller.go | 138 +--------- 3 files changed, 266 insertions(+), 134 deletions(-) create mode 100644 pkg/controller/machine/bootstrap.go diff --git a/.gitignore b/.gitignore index 257869384..b782ea31f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ examples/*.csr examples/*.srl /webhook /vendor +.vscode \ No newline at end of file diff --git a/pkg/controller/machine/bootstrap.go b/pkg/controller/machine/bootstrap.go new file mode 100644 index 000000000..3f9cabe6e --- /dev/null +++ b/pkg/controller/machine/bootstrap.go @@ -0,0 +1,261 @@ +/* +Copyright 2021 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "regexp" + "text/template" + + "github.com/Masterminds/sprig/v3" + + "github.com/kubermatic/machine-controller/pkg/apis/plugin" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "github.com/kubermatic/machine-controller/pkg/userdata/convert" + "github.com/kubermatic/machine-controller/pkg/userdata/flatcar" + + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func getOSMBootstrapUserdata(ctx context.Context, client ctrlruntimeclient.Client, req plugin.UserDataRequest, secretName string) (string, error) { + + var clusterName string + for key := range req.Kubeconfig.Clusters { + clusterName = key + } + + token, err := util.ExtractAPIServerToken(ctx, client) + if err != nil { + return "", fmt.Errorf("failed to fetch api-server token: %v", err) + } + + // Retrieve provider config from machine + pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) + if err != nil { + return "", fmt.Errorf("failed to get providerSpec: %v", err) + } + + // Ignition configuration is used for flatcar + if useIgnition(pconfig) { + return getOSMBootstrapUserDataForIgnition(ctx, req, pconfig.SSHPublicKeys, token, secretName, clusterName) + } + // cloud-init is used for all other operating systems + return getOSMBootstrapUserDataForCloudInit(ctx, req, pconfig, token, secretName, clusterName) +} + +// getOSMBootstrapUserDataForIgnition returns the userdata for the ignition bootstrap config +func getOSMBootstrapUserDataForIgnition(ctx context.Context, req plugin.UserDataRequest, sshPublicKeys []string, token, secretName, clusterName string) (string, error) { + data := struct { + Token string + SecretName string + ServerURL string + }{ + Token: token, + SecretName: secretName, + ServerURL: req.Kubeconfig.Clusters[clusterName].Server, + } + bsScript, err := template.New("bootstrap-script").Parse(ignitionBootstrapBinContentTemplate) + if err != nil { + return "", fmt.Errorf("failed to parse bootstrapBinContentTemplate template for ignition: %v", err) + } + script := &bytes.Buffer{} + err = bsScript.Execute(script, data) + if err != nil { + return "", fmt.Errorf("failed to execute bootstrapBinContentTemplate template for ignition: %v", err) + } + bsIgnitionConfig, err := template.New("bootstrap-ignition-config").Funcs(sprig.TxtFuncMap()).Parse(ignitionTemplate) + if err != nil { + return "", fmt.Errorf("failed to parse bootstrap-ignition-config template: %v", err) + } + + ignitionConfig := &bytes.Buffer{} + err = bsIgnitionConfig.Execute(ignitionConfig, struct { + Script string + Service string + SSHPublicKeys []string + }{ + Script: script.String(), + Service: bootstrapServiceContentTemplate, + SSHPublicKeys: sshPublicKeys, + }) + if err != nil { + return "", fmt.Errorf("failed to execute ignitionTemplate template: %v", err) + } + + return convert.ToIgnition(ignitionConfig.String()) +} + +// getOSMBootstrapUserDataForCloudInit returns the userdata for the cloud-init bootstrap script +func getOSMBootstrapUserDataForCloudInit(ctx context.Context, req plugin.UserDataRequest, pconfig *providerconfigtypes.Config, token, secretName, clusterName string) (string, error) { + data := struct { + Token string + SecretName string + ServerURL string + MachineName string + }{ + Token: token, + SecretName: secretName, + ServerURL: req.Kubeconfig.Clusters[clusterName].Server, + MachineName: req.MachineSpec.Name, + } + bsScript, err := template.New("bootstrap-cloud-init").Parse(bootstrapBinContentTemplate) + if err != nil { + return "", fmt.Errorf("failed to parse bootstrapBinContentTemplate template: %v", err) + } + script := &bytes.Buffer{} + err = bsScript.Execute(script, data) + if err != nil { + return "", fmt.Errorf("failed to execute bootstrapBinContentTemplate template: %v", err) + } + bsCloudInit, err := template.New("bootstrap-cloud-init").Parse(cloudInitTemplate) + if err != nil { + return "", fmt.Errorf("failed to parse download-binaries template: %v", err) + } + + cloudInit := &bytes.Buffer{} + err = bsCloudInit.Execute(cloudInit, struct { + Script string + Service string + plugin.UserDataRequest + ProviderSpec *providerconfigtypes.Config + }{ + Script: base64.StdEncoding.EncodeToString(script.Bytes()), + Service: base64.StdEncoding.EncodeToString([]byte(bootstrapServiceContentTemplate)), + UserDataRequest: req, + ProviderSpec: pconfig, + }) + if err != nil { + return "", fmt.Errorf("failed to execute cloudInitTemplate template: %v", err) + } + return cloudInit.String(), nil +} + +// cleanupTemplateOutput postprocesses the output of the template processing. Those +// may exist due to the working of template functions like those of the sprig package +// or template condition. +func cleanupTemplateOutput(output string) (string, error) { + // Valid YAML files are not allowed to have empty lines containing spaces or tabs. + // So far only cleanup. + woBlankLines := regexp.MustCompile(`(?m)^[ \t]+$`).ReplaceAllString(output, "") + return woBlankLines, nil +} + +func useIgnition(p *providerconfigtypes.Config) bool { + if p.OperatingSystem == providerconfigtypes.OperatingSystemFlatcar { + config, err := flatcar.LoadConfig(p.OperatingSystemSpec) + if err != nil { + return false + } + return config.ProvisioningUtility == flatcar.Ignition + } + return false +} + +const ( + bootstrapBinContentTemplate = `#!/bin/bash +set -xeuo pipefail +apt update && apt install -y curl jq +curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-config"]' -r| base64 -d > /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg +cloud-init clean +cloud-init --file /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg init +systemctl daemon-reload +systemctl restart setup.service +systemctl restart kubelet.service +systemctl restart kubelet-healthcheck.service + ` + + bootstrapServiceContentTemplate = `[Install] +WantedBy=multi-user.target + +[Unit] +Requires=network-online.target +After=network-online.target +[Service] +Type=oneshot +RemainAfterExit=true +ExecStart=/opt/bin/bootstrap + ` + + cloudInitTemplate = `#cloud-config +{{ if ne .CloudProviderName "aws" }} +hostname: {{ .MachineSpec.Name }} +{{- /* Never set the hostname on AWS nodes. Kubernetes(kube-proxy) requires the hostname to be the private dns name */}} +{{ end }} +ssh_pwauth: no + +{{- if .ProviderSpec.SSHPublicKeys }} +ssh_authorized_keys: +{{- range .ProviderSpec.SSHPublicKeys }} +- "{{ . }}" +{{- end }} +{{- end }} + +write_files: +- path: /opt/bin/bootstrap + permissions: '0755' + encoding: b64 + content: | + {{ .Script }} +- path: /etc/systemd/system/bootstrap.service + permissions: '0644' + encoding: b64 + content: | + {{ .Service }} +runcmd: +- systemctl restart bootstrap.service +- systemctl daemon-reload +` + + ignitionBootstrapBinContentTemplate = `#!/bin/bash +set -xeuo pipefail +apt update && apt install -y curl jq +curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-config"]' -r| base64 -d > /usr/share/oem/config.ign +touch /boot/flatcar/first_boot +systemctl disable bootstrap.service +rm /etc/systemd/system/bootstrap.service +rm /etc/machine-id +reboot +` + + ignitionTemplate = `passwd: +{{- if ne (len .SSHPublicKeys) 0 }} + users: + - name: core + ssh_authorized_keys: + {{range .SSHPublicKeys }}- {{.}} + {{end}} +{{- end }} +storage: + files: + - path: /opt/bin/bootstrap + mode: 0755 + filesystem: root + contents: + inline: | +{{ .Script | indent 10}} +systemd: + units: + - name: bootstrap.service + enabled: true + contents: | +{{ .Service | indent 10 }} +` +) diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 9d2c29a71..1385bf3b2 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -17,16 +17,12 @@ limitations under the License. package controller import ( - "bytes" "context" - "encoding/base64" "errors" "fmt" "net" - "regexp" "strconv" "strings" - "text/template" "time" "github.com/heptiolabs/healthcheck" @@ -39,7 +35,6 @@ import ( cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/containerruntime" kuberneteshelper "github.com/kubermatic/machine-controller/pkg/kubernetes" "github.com/kubermatic/machine-controller/pkg/node/eviction" @@ -754,19 +749,19 @@ func (r *Reconciler) ensureInstanceExistsForMachine( return nil, fmt.Errorf("failed to find machine's MachineDployment: %v", err) } - cloudInitConfigSecretName := fmt.Sprintf("%s-%s", + cloudConfigSecretName := fmt.Sprintf("%s-%s", referencedMachineDeployment, provisioningSuffix) - // It is important to check if the secret which holds the cloud init configurations + // It is important to check if the secret holding cloud-config exists if err := r.client.Get(ctx, - types.NamespacedName{Name: cloudInitConfigSecretName, Namespace: "kube-system"}, + types.NamespacedName{Name: cloudConfigSecretName, Namespace: "kube-system"}, &corev1.Secret{}); err != nil { klog.Errorf("Cloud init configurations for machine: %v is not ready yet", machine.Name) return nil, err } - userdata, err = getOSMBootstrapUserdata(ctx, r.client, req, cloudInitConfigSecretName) + userdata, err = getOSMBootstrapUserdata(ctx, r.client, req, cloudConfigSecretName) if err != nil { return nil, fmt.Errorf("failed get OSM userdata: %v", err) } @@ -1117,128 +1112,3 @@ func (r *Reconciler) getMachineDeploymentNameForMachine(ctx context.Context, mac return "", fmt.Errorf("failed to find machine deployment reference for the machine %s", machine.Name) } - -func getOSMBootstrapUserdata(ctx context.Context, client ctrlruntimeclient.Client, req plugin.UserDataRequest, secretName string) (string, error) { - - var clusterName string - for key := range req.Kubeconfig.Clusters { - clusterName = key - } - - token, err := util.ExtractAPIServerToken(ctx, client) - if err != nil { - return "", fmt.Errorf("failed to fetch api-server token: %v", err) - } - - data := struct { - Token string - SecretName string - ServerURL string - MachineName string - }{ - Token: token, - SecretName: secretName, - ServerURL: req.Kubeconfig.Clusters[clusterName].Server, - MachineName: req.MachineSpec.Name, - } - bsScript, err := template.New("bootstrap-cloud-init").Parse(bootstrapBinContentTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse bootstrapBinContentTemplate template: %v", err) - } - script := &bytes.Buffer{} - err = bsScript.Execute(script, data) - if err != nil { - return "", fmt.Errorf("failed to execute bootstrapBinContentTemplate template: %v", err) - } - bsCloudInit, err := template.New("bootstrap-cloud-init").Parse(cloudInitTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse download-binaries template: %v", err) - } - pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) - if err != nil { - return "", fmt.Errorf("failed to get providerSpec: %v", err) - } - - cloudInit := &bytes.Buffer{} - err = bsCloudInit.Execute(cloudInit, struct { - Script string - Service string - plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - }{ - Script: base64.StdEncoding.EncodeToString(script.Bytes()), - Service: base64.StdEncoding.EncodeToString([]byte(bootstrapServiceContentTemplate)), - UserDataRequest: req, - ProviderSpec: pconfig, - }) - if err != nil { - return "", fmt.Errorf("failed to execute cloudInitTemplate template: %v", err) - } - return cloudInit.String(), nil -} - -// cleanupTemplateOutput postprocesses the output of the template processing. Those -// may exist due to the working of template functions like those of the sprig package -// or template condition. -func cleanupTemplateOutput(output string) (string, error) { - // Valid YAML files are not allowed to have empty lines containing spaces or tabs. - // So far only cleanup. - woBlankLines := regexp.MustCompile(`(?m)^[ \t]+$`).ReplaceAllString(output, "") - return woBlankLines, nil -} - -const ( - bootstrapBinContentTemplate = `#!/bin/bash -set -xeuo pipefail -apt update && apt install -y curl jq -curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-init"]' -r| base64 -d > /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg -cloud-init clean -cloud-init --file /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg init -systemctl daemon-reload -systemctl restart setup.service -systemctl restart kubelet.service -systemctl restart kubelet-healthcheck.service - ` - - bootstrapServiceContentTemplate = `[Install] -WantedBy=multi-user.target - -[Unit] -Requires=network-online.target -After=network-online.target -[Service] -Type=oneshot -RemainAfterExit=true -ExecStart=/opt/bin/bootstrap - ` - - cloudInitTemplate = `#cloud-config -{{ if ne .CloudProviderName "aws" }} -hostname: {{ .MachineSpec.Name }} -{{- /* Never set the hostname on AWS nodes. Kubernetes(kube-proxy) requires the hostname to be the private dns name */}} -{{ end }} -ssh_pwauth: false - -{{- if .ProviderSpec.SSHPublicKeys }} -ssh_authorized_keys: -{{- range .ProviderSpec.SSHPublicKeys }} -- "{{ . }}" -{{- end }} -{{- end }} - -write_files: -- path: /opt/bin/bootstrap - permissions: '0755' - encoding: b64 - content: | - {{ .Script }} -- path: /etc/systemd/system/bootstrap.service - permissions: '0644' - encoding: b64 - content: | - {{ .Service }} -runcmd: -- systemctl restart bootstrap.service -- systemctl daemon-reload -` -) From a44de322619b30889bf27341ea72367c7c17af2a Mon Sep 17 00:00:00 2001 From: Vincent Gramer Date: Mon, 15 Nov 2021 22:44:06 +0100 Subject: [PATCH 025/489] e2e test Openstack Project Auth Provisioning only with one version of k8s (#1102) Signed-off-by: Vincent Gramer --- test/e2e/provisioning/all_e2e_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 7ba495a68..927bd2362 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -376,8 +376,14 @@ func TestOpenstackProjectAuthProvisioningE2E(t *testing.T) { fmt.Sprintf("<< NETWORK_NAME >>=%s", osNetwork), } - selector := OsSelector("ubuntu") - runScenarios(t, selector, params, OSManifestProjectAuth, fmt.Sprintf("os-%s", *testRunIdentifier)) + scenario := scenario{ + name: "MachineDeploy with project auth vars", + osName: "ubuntu", + containerRuntime: "containerd", + kubernetesVersion: "1.21.2", + executor: verifyCreateAndDelete, + } + testScenario(t, scenario, *testRunIdentifier, params, OSManifestProjectAuth, false) } // TestDigitalOceanProvisioning - a test suite that exercises digital ocean provider From fe856b902a6800fcca91cf1d09dba26932cbe1bf Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Thu, 18 Nov 2021 05:10:05 +0100 Subject: [PATCH 026/489] support centos bootstrapping (#1105) Signed-off-by: Moath Qasim Signed-off-by: Moath Qasim --- pkg/controller/machine/bootstrap.go | 38 +++++++++++++++++--- pkg/controller/machine/machine_controller.go | 3 +- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/pkg/controller/machine/bootstrap.go b/pkg/controller/machine/bootstrap.go index 3f9cabe6e..b3a46ca93 100644 --- a/pkg/controller/machine/bootstrap.go +++ b/pkg/controller/machine/bootstrap.go @@ -116,14 +116,29 @@ func getOSMBootstrapUserDataForCloudInit(ctx context.Context, req plugin.UserDat ServerURL: req.Kubeconfig.Clusters[clusterName].Server, MachineName: req.MachineSpec.Name, } - bsScript, err := template.New("bootstrap-cloud-init").Parse(bootstrapBinContentTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse bootstrapBinContentTemplate template: %v", err) + + var ( + bsScript *template.Template + err error + ) + + switch pconfig.OperatingSystem { + case providerconfigtypes.OperatingSystemUbuntu: + bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapAptBinContentTemplate) + if err != nil { + return "", fmt.Errorf("failed to parse bootstrapAptBinContentTemplate template: %v", err) + } + case providerconfigtypes.OperatingSystemCentOS: + bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapYumBinContentTemplate) + if err != nil { + return "", fmt.Errorf("failed to parse bootstrapYumBinContentTemplate template: %v", err) + } } + script := &bytes.Buffer{} err = bsScript.Execute(script, data) if err != nil { - return "", fmt.Errorf("failed to execute bootstrapBinContentTemplate template: %v", err) + return "", fmt.Errorf("failed to execute bootstrap script template: %v", err) } bsCloudInit, err := template.New("bootstrap-cloud-init").Parse(cloudInitTemplate) if err != nil { @@ -170,7 +185,7 @@ func useIgnition(p *providerconfigtypes.Config) bool { } const ( - bootstrapBinContentTemplate = `#!/bin/bash + bootstrapAptBinContentTemplate = `#!/bin/bash set -xeuo pipefail apt update && apt install -y curl jq curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-config"]' -r| base64 -d > /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg @@ -182,6 +197,19 @@ systemctl restart kubelet.service systemctl restart kubelet-healthcheck.service ` + bootstrapYumBinContentTemplate = `#!/bin/bash +set -xeuo pipefail +yum install epel-release -y +yum install -y curl jq +curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-config"]' -r| base64 -d > /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg +cloud-init clean +cloud-init --file /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg init +systemctl daemon-reload +systemctl restart setup.service +systemctl restart kubelet.service +systemctl restart kubelet-healthcheck.service + ` + bootstrapServiceContentTemplate = `[Install] WantedBy=multi-user.target diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 1385bf3b2..693974d53 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -35,6 +35,7 @@ import ( cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/containerruntime" kuberneteshelper "github.com/kubermatic/machine-controller/pkg/kubernetes" "github.com/kubermatic/machine-controller/pkg/node/eviction" @@ -755,7 +756,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( // It is important to check if the secret holding cloud-config exists if err := r.client.Get(ctx, - types.NamespacedName{Name: cloudConfigSecretName, Namespace: "kube-system"}, + types.NamespacedName{Name: cloudConfigSecretName, Namespace: util.CloudInitNamespace}, &corev1.Secret{}); err != nil { klog.Errorf("Cloud init configurations for machine: %v is not ready yet", machine.Name) return nil, err From f9528ee5868b5e624c0fe18eb5fda10b138d1a08 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Fri, 19 Nov 2021 20:15:18 +0100 Subject: [PATCH 027/489] disable ssh pwauth (#1107) Signed-off-by: Moath Qasim --- pkg/controller/machine/bootstrap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/machine/bootstrap.go b/pkg/controller/machine/bootstrap.go index b3a46ca93..6ed7ccfee 100644 --- a/pkg/controller/machine/bootstrap.go +++ b/pkg/controller/machine/bootstrap.go @@ -227,7 +227,7 @@ ExecStart=/opt/bin/bootstrap hostname: {{ .MachineSpec.Name }} {{- /* Never set the hostname on AWS nodes. Kubernetes(kube-proxy) requires the hostname to be the private dns name */}} {{ end }} -ssh_pwauth: no +ssh_pwauth: false {{- if .ProviderSpec.SSHPublicKeys }} ssh_authorized_keys: From 26bda9b43b87698163b537aa5920189e72c31eee Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 23 Nov 2021 06:03:16 +0100 Subject: [PATCH 028/489] adjust yum bootstrapping script (#1108) Signed-off-by: Moath Qasim --- pkg/controller/machine/bootstrap.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pkg/controller/machine/bootstrap.go b/pkg/controller/machine/bootstrap.go index 6ed7ccfee..cc5529c58 100644 --- a/pkg/controller/machine/bootstrap.go +++ b/pkg/controller/machine/bootstrap.go @@ -106,10 +106,11 @@ func getOSMBootstrapUserDataForIgnition(ctx context.Context, req plugin.UserData // getOSMBootstrapUserDataForCloudInit returns the userdata for the cloud-init bootstrap script func getOSMBootstrapUserDataForCloudInit(ctx context.Context, req plugin.UserDataRequest, pconfig *providerconfigtypes.Config, token, secretName, clusterName string) (string, error) { data := struct { - Token string - SecretName string - ServerURL string - MachineName string + Token string + SecretName string + ServerURL string + MachineName string + EnterpriseLinux bool }{ Token: token, SecretName: secretName, @@ -129,6 +130,12 @@ func getOSMBootstrapUserDataForCloudInit(ctx context.Context, req plugin.UserDat return "", fmt.Errorf("failed to parse bootstrapAptBinContentTemplate template: %v", err) } case providerconfigtypes.OperatingSystemCentOS: + data.EnterpriseLinux = true + bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapYumBinContentTemplate) + if err != nil { + return "", fmt.Errorf("failed to parse bootstrapYumBinContentTemplate template: %v", err) + } + case providerconfigtypes.OperatingSystemAmazonLinux2: bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapYumBinContentTemplate) if err != nil { return "", fmt.Errorf("failed to parse bootstrapYumBinContentTemplate template: %v", err) @@ -199,7 +206,9 @@ systemctl restart kubelet-healthcheck.service bootstrapYumBinContentTemplate = `#!/bin/bash set -xeuo pipefail +{{- if .EnterpriseLinux }} yum install epel-release -y +{{- end }} yum install -y curl jq curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-config"]' -r| base64 -d > /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg cloud-init clean From 3cd8a6344519e96487cbc843aa1c2687737687bf Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 23 Nov 2021 15:06:17 +0500 Subject: [PATCH 029/489] Populate /etc/hostname during bootstrap for ignition (#1106) * Set hostname for ignition * refactored code * Fix linting errors --- pkg/controller/machine/bootstrap.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/pkg/controller/machine/bootstrap.go b/pkg/controller/machine/bootstrap.go index cc5529c58..b55dfd29f 100644 --- a/pkg/controller/machine/bootstrap.go +++ b/pkg/controller/machine/bootstrap.go @@ -91,10 +91,12 @@ func getOSMBootstrapUserDataForIgnition(ctx context.Context, req plugin.UserData Script string Service string SSHPublicKeys []string + plugin.UserDataRequest }{ - Script: script.String(), - Service: bootstrapServiceContentTemplate, - SSHPublicKeys: sshPublicKeys, + Script: script.String(), + Service: bootstrapServiceContentTemplate, + SSHPublicKeys: sshPublicKeys, + UserDataRequest: req, }) if err != nil { return "", fmt.Errorf("failed to execute ignitionTemplate template: %v", err) @@ -288,6 +290,14 @@ storage: contents: inline: | {{ .Script | indent 10}} +{{ if ne .CloudProviderName "aws" }} +{{- /* Never set the hostname on AWS nodes. Kubernetes(kube-proxy) requires the hostname to be the private dns name */}} + - path: /etc/hostname + mode: 0600 + filesystem: root + contents: + inline: '{{ .MachineSpec.Name }}' +{{ end }} systemd: units: - name: bootstrap.service From 291aaf251a2a1bdcac5c24b58de992e333768edf Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Thu, 25 Nov 2021 17:21:34 +0200 Subject: [PATCH 030/489] Flatcar improvements (#1109) * Flatcar improvements: - remove containerized kubelet - removed docker service dependency - use the same containerd config and kubelet.service as other providers - allow to use the pause image as the sandbox image in containerd to gain more control over the used image Signed-off-by: Florin Peter * flatcar: use correct containerruntime scripts Signed-off-by: Artiom Diomin * Fix variable escaping Signed-off-by: Artiom Diomin * update fixtures Signed-off-by: Artiom Diomin Co-authored-by: Florin Peter --- cmd/machine-controller/main.go | 1 + pkg/containerruntime/containerd.go | 21 +- pkg/containerruntime/containerruntime.go | 8 + pkg/containerruntime/docker.go | 8 +- pkg/userdata/flatcar/provider.go | 268 ++++++------------ .../flatcar/testdata/cloud-init_v1.19.15.yaml | 149 ++++------ .../flatcar/testdata/cloud-init_v1.20.11.yaml | 149 ++++------ .../flatcar/testdata/cloud-init_v1.21.5.yaml | 149 ++++------ .../flatcar/testdata/cloud-init_v1.22.2.yaml | 178 ++++++------ pkg/userdata/flatcar/testdata/containerd.yaml | 174 ++++++------ .../flatcar/testdata/ignition_v1.19.15.json | 2 +- .../flatcar/testdata/ignition_v1.20.11.json | 2 +- .../flatcar/testdata/ignition_v1.21.5.json | 2 +- .../flatcar/testdata/ignition_v1.22.2.json | 2 +- pkg/userdata/helper/helper.go | 8 +- 15 files changed, 470 insertions(+), 651 deletions(-) diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 45660de34..49d1ca052 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -276,6 +276,7 @@ func main() { nodeContainerRuntime, containerruntime.WithInsecureRegistries(insecureRegistries), containerruntime.WithRegistryMirrors(registryMirrors), + containerruntime.WithSandboxImage(nodePauseImage), ), }, useOSM: useOSM, diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index e42bff608..28e4e4203 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -32,10 +32,11 @@ const ( type Containerd struct { insecureRegistries []string registryMirrors []string + sandboxImage string } func (eng *Containerd) Config() (string, error) { - return helper.ContainerdConfig(eng.insecureRegistries, eng.registryMirrors) + return helper.ContainerdConfig(eng.insecureRegistries, eng.registryMirrors, eng.sandboxImage) } func (eng *Containerd) ConfigFileName() string { @@ -69,7 +70,8 @@ func (eng *Containerd) ScriptFor(os types.OperatingSystem) (string, error) { err := containerdAptTemplate.Execute(&buf, args) return buf.String(), err case types.OperatingSystemFlatcar: - return "", nil + err := containedFlatcarTemplate.Execute(&buf, args) + return buf.String(), err case types.OperatingSystemSLES: return "", nil } @@ -78,6 +80,21 @@ func (eng *Containerd) ScriptFor(os types.OperatingSystem) (string, error) { } var ( + containedFlatcarTemplate = template.Must(template.New("containerd-flatcar").Parse(` +mkdir -p /etc/systemd/system/containerd.service.d + +cat < /tmp/environment' - ExecStart=/usr/bin/docker run --name %n \ - --rm --tty --restart no \ - --network host \ - --pid host \ - --env-file /tmp/environment \ - --privileged \ - --cgroup-parent system.slice \ - --entrypoint kubelet \ - -v /dev:/dev \ - -v /etc/cni/net.d:/etc/cni/net.d \ - -v /etc/kubernetes:/etc/kubernetes \ - -v /etc/machine-id:/etc/machine-id:ro \ - -v /etc/os-release:/etc/os-release:ro \ - -v /etc/resolv.conf:/etc/resolv.conf:ro \ - -v /lib/modules:/lib/modules \ - -v /mnt:/mnt:rshared \ - -v /opt/cni/bin:/opt/cni/bin:ro \ - -v /run:/run \ - -v /sys:/sys \ - -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \ - -v /var/lib/calico:/var/lib/calico:ro \ - -v /var/lib/cni:/var/lib/cni \ - -v /var/lib/docker:/var/lib/docker \ - -v /var/lib/kubelet:/var/lib/kubelet:rshared \ - -v /var/log/pods:/var/log/pods \ - {{ .KubeletImage }} \ -{{ kubeletFlags .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 10 }} - ExecStop=-/usr/bin/docker stop %n - Restart=always - RestartSec=10 - [Install] - WantedBy=multi-user.target - - - name: docker.service enabled: true dropins: - - name: 10-environment.conf + - name: 10-nodeip.conf contents: | [Service] - EnvironmentFile=-/etc/environment + EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: 40-download.conf + contents: | + [Unit] + Requires=download-script.service + After=download-script.service + contents: | +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 8 }} storage: files: @@ -447,13 +403,6 @@ storage: }); {{- end }} - - path: /etc/docker/daemon.json - filesystem: root - mode: 0644 - contents: - inline: | -{{ dockerConfig .InsecureRegistries .RegistryMirrors | indent 10 }} - - path: /opt/bin/download.sh filesystem: root mode: 0755 @@ -461,8 +410,30 @@ storage: inline: | #!/bin/bash set -xeuo pipefail + {{ safeDownloadBinariesScript .KubeletVersion | indent 10 }} + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat < /tmp/environment' - ExecStart=/usr/bin/docker run --name %n \ - --rm --tty --restart no \ - --network host \ - --pid host \ - --env-file /tmp/environment \ - --privileged \ - --cgroup-parent system.slice \ - --entrypoint kubelet \ - -v /dev:/dev \ - -v /etc/cni/net.d:/etc/cni/net.d \ - -v /etc/kubernetes:/etc/kubernetes \ - -v /etc/machine-id:/etc/machine-id:ro \ - -v /etc/os-release:/etc/os-release:ro \ - -v /etc/resolv.conf:/etc/resolv.conf:ro \ - -v /lib/modules:/lib/modules \ - -v /mnt:/mnt:rshared \ - -v /opt/cni/bin:/opt/cni/bin:ro \ - -v /run:/run \ - -v /sys:/sys \ - -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \ - -v /var/lib/calico:/var/lib/calico:ro \ - -v /var/lib/cni:/var/lib/cni \ - -v /var/lib/docker:/var/lib/docker \ - -v /var/lib/kubelet:/var/lib/kubelet:rshared \ - -v /var/log/pods:/var/log/pods \ - {{ .KubeletImage }} \ -{{ kubeletFlags .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 10 }} - ExecStop=-/usr/bin/docker stop %n - Restart=always - RestartSec=10 - [Install] - WantedBy=multi-user.target - - - name: docker.service enable: true command: start drop-ins: - - name: 10-environment.conf + - name: 10-nodeip.conf content: | [Service] - EnvironmentFile=-/etc/environment + EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service + content: | +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 6 }} - name: apply-sysctl-settings.service enable: true @@ -728,17 +636,13 @@ write_files: }); {{- end }} -- path: /etc/docker/daemon.json - permissions: "0644" - content: | -{{ dockerConfig .InsecureRegistries .RegistryMirrors | indent 4 }} - - path: /opt/bin/download.sh permissions: "0755" content: | #!/bin/bash set -xeuo pipefail {{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} +{{ .ContainerRuntimeScript | indent 4 }} systemctl disable download-script.service - path: /opt/bin/apply_sysctl_settings.sh @@ -761,4 +665,16 @@ write_files: {{ sshConfigAddendum | indent 4 }} append: true {{- end }} + +- path: {{ .ContainerRuntimeConfigFileName }} + permissions: "0644" + user: root + content: | +{{ .ContainerRuntimeConfig | indent 4 }} + +- path: /etc/crictl.yaml + permissions: "0644" + user: root + content: | + runtime-endpoint: unix:///run/containerd/containerd.sock ` diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.19.15.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.19.15.yaml index 0ca8bc9e9..4148f486e 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.19.15.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.19.15.yaml @@ -27,9 +27,6 @@ coreos: mask: true - name: locksmithd.service mask: true - - name: docker.service - enable: true - command: start - name: download-script.service enable: true command: start @@ -44,31 +41,11 @@ coreos: [Install] WantedBy=multi-user.target - - name: docker-healthcheck.service - enable: true - command: start - drop-ins: - - name: 40-docker.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - - name: kubelet-healthcheck.service enable: true command: start drop-ins: - - name: 40-docker.conf + - name: 40-download.conf content: | [Unit] Requires=download-script.service @@ -104,81 +81,56 @@ coreos: - name: kubelet.service enable: true command: start + drop-ins: + - name: 10-nodeip.conf + content: | + [Service] + EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service content: | [Unit] - Description=Kubernetes Kubelet - Requires=docker.service After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + [Service] - TimeoutStartSec=5min + Restart=always + StartLimitInterval=0 + RestartSec=10 CPUAccounting=true MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" EnvironmentFile=-/etc/environment - EnvironmentFile=/etc/kubernetes/nodeip.conf - Environment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStartPre=/bin/mkdir -p /var/lib/calico - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests - ExecStartPre=/bin/mkdir -p /etc/cni/net.d - ExecStartPre=/bin/mkdir -p /opt/cni/bin + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - ExecStartPre=/bin/sh -c '/usr/bin/env > /tmp/environment' - ExecStart=/usr/bin/docker run --name %n \ - --rm --tty --restart no \ - --network host \ - --pid host \ - --env-file /tmp/environment \ - --privileged \ - --cgroup-parent system.slice \ - --entrypoint kubelet \ - -v /dev:/dev \ - -v /etc/cni/net.d:/etc/cni/net.d \ - -v /etc/kubernetes:/etc/kubernetes \ - -v /etc/machine-id:/etc/machine-id:ro \ - -v /etc/os-release:/etc/os-release:ro \ - -v /etc/resolv.conf:/etc/resolv.conf:ro \ - -v /lib/modules:/lib/modules \ - -v /mnt:/mnt:rshared \ - -v /opt/cni/bin:/opt/cni/bin:ro \ - -v /run:/run \ - -v /sys:/sys \ - -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \ - -v /var/lib/calico:/var/lib/calico:ro \ - -v /var/lib/cni:/var/lib/cni \ - -v /var/lib/docker:/var/lib/docker \ - -v /var/lib/kubelet:/var/lib/kubelet:rshared \ - -v /var/log/pods:/var/log/pods \ - for-kubernetes-more-then-1.19/kubeletImage:v1.19.15 \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=anexia \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --node-ip ${KUBELET_NODE_IP} - ExecStop=-/usr/bin/docker stop %n - Restart=always - RestartSec=10 + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=anexia \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + [Install] WantedBy=multi-user.target - - name: docker.service - enable: true - command: start - drop-ins: - - name: 10-environment.conf - content: | - [Service] - EnvironmentFile=-/etc/environment - - name: apply-sysctl-settings.service enable: true command: start @@ -402,11 +354,6 @@ write_files: PasswordAuthentication no ChallengeResponseAuthentication no -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} - - path: /opt/bin/download.sh permissions: "0755" content: | @@ -477,6 +424,10 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + + systemctl daemon-reload + systemctl enable --now docker + systemctl disable download-script.service - path: /opt/bin/apply_sysctl_settings.sh @@ -497,3 +448,15 @@ write_files: TrustedUserCAKeys /etc/ssh/trusted-user-ca-keys.pem CASignatureAlgorithms ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,ssh-ed25519,rsa-sha2-512,rsa-sha2-256,ssh-rsa append: true + +- path: /etc/docker/daemon.json + permissions: "0644" + user: root + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + +- path: /etc/crictl.yaml + permissions: "0644" + user: root + content: | + runtime-endpoint: unix:///run/containerd/containerd.sock diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.11.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.11.yaml index 9502f765e..355e70c4d 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.11.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.11.yaml @@ -27,9 +27,6 @@ coreos: mask: true - name: locksmithd.service mask: true - - name: docker.service - enable: true - command: start - name: download-script.service enable: true command: start @@ -44,31 +41,11 @@ coreos: [Install] WantedBy=multi-user.target - - name: docker-healthcheck.service - enable: true - command: start - drop-ins: - - name: 40-docker.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - - name: kubelet-healthcheck.service enable: true command: start drop-ins: - - name: 40-docker.conf + - name: 40-download.conf content: | [Unit] Requires=download-script.service @@ -104,81 +81,56 @@ coreos: - name: kubelet.service enable: true command: start + drop-ins: + - name: 10-nodeip.conf + content: | + [Service] + EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service content: | [Unit] - Description=Kubernetes Kubelet - Requires=docker.service After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + [Service] - TimeoutStartSec=5min + Restart=always + StartLimitInterval=0 + RestartSec=10 CPUAccounting=true MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" EnvironmentFile=-/etc/environment - EnvironmentFile=/etc/kubernetes/nodeip.conf - Environment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStartPre=/bin/mkdir -p /var/lib/calico - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests - ExecStartPre=/bin/mkdir -p /etc/cni/net.d - ExecStartPre=/bin/mkdir -p /opt/cni/bin + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - ExecStartPre=/bin/sh -c '/usr/bin/env > /tmp/environment' - ExecStart=/usr/bin/docker run --name %n \ - --rm --tty --restart no \ - --network host \ - --pid host \ - --env-file /tmp/environment \ - --privileged \ - --cgroup-parent system.slice \ - --entrypoint kubelet \ - -v /dev:/dev \ - -v /etc/cni/net.d:/etc/cni/net.d \ - -v /etc/kubernetes:/etc/kubernetes \ - -v /etc/machine-id:/etc/machine-id:ro \ - -v /etc/os-release:/etc/os-release:ro \ - -v /etc/resolv.conf:/etc/resolv.conf:ro \ - -v /lib/modules:/lib/modules \ - -v /mnt:/mnt:rshared \ - -v /opt/cni/bin:/opt/cni/bin:ro \ - -v /run:/run \ - -v /sys:/sys \ - -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \ - -v /var/lib/calico:/var/lib/calico:ro \ - -v /var/lib/cni:/var/lib/cni \ - -v /var/lib/docker:/var/lib/docker \ - -v /var/lib/kubelet:/var/lib/kubelet:rshared \ - -v /var/log/pods:/var/log/pods \ - for-kubernetes-more-then-1.19/kubeletImage:v1.20.11 \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=anexia \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --node-ip ${KUBELET_NODE_IP} - ExecStop=-/usr/bin/docker stop %n - Restart=always - RestartSec=10 + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=anexia \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + [Install] WantedBy=multi-user.target - - name: docker.service - enable: true - command: start - drop-ins: - - name: 10-environment.conf - content: | - [Service] - EnvironmentFile=-/etc/environment - - name: apply-sysctl-settings.service enable: true command: start @@ -402,11 +354,6 @@ write_files: PasswordAuthentication no ChallengeResponseAuthentication no -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} - - path: /opt/bin/download.sh permissions: "0755" content: | @@ -477,6 +424,10 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + + systemctl daemon-reload + systemctl enable --now docker + systemctl disable download-script.service - path: /opt/bin/apply_sysctl_settings.sh @@ -497,3 +448,15 @@ write_files: TrustedUserCAKeys /etc/ssh/trusted-user-ca-keys.pem CASignatureAlgorithms ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,ssh-ed25519,rsa-sha2-512,rsa-sha2-256,ssh-rsa append: true + +- path: /etc/docker/daemon.json + permissions: "0644" + user: root + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + +- path: /etc/crictl.yaml + permissions: "0644" + user: root + content: | + runtime-endpoint: unix:///run/containerd/containerd.sock diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.5.yaml index 1978505d3..3017ec388 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.5.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.5.yaml @@ -27,9 +27,6 @@ coreos: mask: true - name: locksmithd.service mask: true - - name: docker.service - enable: true - command: start - name: download-script.service enable: true command: start @@ -44,31 +41,11 @@ coreos: [Install] WantedBy=multi-user.target - - name: docker-healthcheck.service - enable: true - command: start - drop-ins: - - name: 40-docker.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - - name: kubelet-healthcheck.service enable: true command: start drop-ins: - - name: 40-docker.conf + - name: 40-download.conf content: | [Unit] Requires=download-script.service @@ -104,81 +81,56 @@ coreos: - name: kubelet.service enable: true command: start + drop-ins: + - name: 10-nodeip.conf + content: | + [Service] + EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service content: | [Unit] - Description=Kubernetes Kubelet - Requires=docker.service After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + [Service] - TimeoutStartSec=5min + Restart=always + StartLimitInterval=0 + RestartSec=10 CPUAccounting=true MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" EnvironmentFile=-/etc/environment - EnvironmentFile=/etc/kubernetes/nodeip.conf - Environment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStartPre=/bin/mkdir -p /var/lib/calico - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests - ExecStartPre=/bin/mkdir -p /etc/cni/net.d - ExecStartPre=/bin/mkdir -p /opt/cni/bin + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - ExecStartPre=/bin/sh -c '/usr/bin/env > /tmp/environment' - ExecStart=/usr/bin/docker run --name %n \ - --rm --tty --restart no \ - --network host \ - --pid host \ - --env-file /tmp/environment \ - --privileged \ - --cgroup-parent system.slice \ - --entrypoint kubelet \ - -v /dev:/dev \ - -v /etc/cni/net.d:/etc/cni/net.d \ - -v /etc/kubernetes:/etc/kubernetes \ - -v /etc/machine-id:/etc/machine-id:ro \ - -v /etc/os-release:/etc/os-release:ro \ - -v /etc/resolv.conf:/etc/resolv.conf:ro \ - -v /lib/modules:/lib/modules \ - -v /mnt:/mnt:rshared \ - -v /opt/cni/bin:/opt/cni/bin:ro \ - -v /run:/run \ - -v /sys:/sys \ - -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \ - -v /var/lib/calico:/var/lib/calico:ro \ - -v /var/lib/cni:/var/lib/cni \ - -v /var/lib/docker:/var/lib/docker \ - -v /var/lib/kubelet:/var/lib/kubelet:rshared \ - -v /var/log/pods:/var/log/pods \ - for-kubernetes-more-then-1.19/kubeletImage:v1.21.5 \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=anexia \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --node-ip ${KUBELET_NODE_IP} - ExecStop=-/usr/bin/docker stop %n - Restart=always - RestartSec=10 + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=anexia \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + [Install] WantedBy=multi-user.target - - name: docker.service - enable: true - command: start - drop-ins: - - name: 10-environment.conf - content: | - [Service] - EnvironmentFile=-/etc/environment - - name: apply-sysctl-settings.service enable: true command: start @@ -402,11 +354,6 @@ write_files: PasswordAuthentication no ChallengeResponseAuthentication no -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} - - path: /opt/bin/download.sh permissions: "0755" content: | @@ -477,6 +424,10 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + + systemctl daemon-reload + systemctl enable --now docker + systemctl disable download-script.service - path: /opt/bin/apply_sysctl_settings.sh @@ -497,3 +448,15 @@ write_files: TrustedUserCAKeys /etc/ssh/trusted-user-ca-keys.pem CASignatureAlgorithms ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,ssh-ed25519,rsa-sha2-512,rsa-sha2-256,ssh-rsa append: true + +- path: /etc/docker/daemon.json + permissions: "0644" + user: root + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + +- path: /etc/crictl.yaml + permissions: "0644" + user: root + content: | + runtime-endpoint: unix:///run/containerd/containerd.sock diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.2.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.2.yaml index aabc8e7d5..f42f4375f 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.2.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.2.yaml @@ -27,9 +27,6 @@ coreos: mask: true - name: locksmithd.service mask: true - - name: docker.service - enable: true - command: start - name: download-script.service enable: true command: start @@ -44,31 +41,11 @@ coreos: [Install] WantedBy=multi-user.target - - name: docker-healthcheck.service - enable: true - command: start - drop-ins: - - name: 40-docker.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - - name: kubelet-healthcheck.service enable: true command: start drop-ins: - - name: 40-docker.conf + - name: 40-download.conf content: | [Unit] Requires=download-script.service @@ -104,81 +81,56 @@ coreos: - name: kubelet.service enable: true command: start + drop-ins: + - name: 10-nodeip.conf + content: | + [Service] + EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service content: | [Unit] - Description=Kubernetes Kubelet - Requires=docker.service - After=docker.service + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + [Service] - TimeoutStartSec=5min + Restart=always + StartLimitInterval=0 + RestartSec=10 CPUAccounting=true MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" EnvironmentFile=-/etc/environment - EnvironmentFile=/etc/kubernetes/nodeip.conf - Environment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStartPre=/bin/mkdir -p /var/lib/calico - ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests - ExecStartPre=/bin/mkdir -p /etc/cni/net.d - ExecStartPre=/bin/mkdir -p /opt/cni/bin + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - ExecStartPre=/bin/sh -c '/usr/bin/env > /tmp/environment' - ExecStart=/usr/bin/docker run --name %n \ - --rm --tty --restart no \ - --network host \ - --pid host \ - --env-file /tmp/environment \ - --privileged \ - --cgroup-parent system.slice \ - --entrypoint kubelet \ - -v /dev:/dev \ - -v /etc/cni/net.d:/etc/cni/net.d \ - -v /etc/kubernetes:/etc/kubernetes \ - -v /etc/machine-id:/etc/machine-id:ro \ - -v /etc/os-release:/etc/os-release:ro \ - -v /etc/resolv.conf:/etc/resolv.conf:ro \ - -v /lib/modules:/lib/modules \ - -v /mnt:/mnt:rshared \ - -v /opt/cni/bin:/opt/cni/bin:ro \ - -v /run:/run \ - -v /sys:/sys \ - -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \ - -v /var/lib/calico:/var/lib/calico:ro \ - -v /var/lib/cni:/var/lib/cni \ - -v /var/lib/docker:/var/lib/docker \ - -v /var/lib/kubelet:/var/lib/kubelet:rshared \ - -v /var/log/pods:/var/log/pods \ - for-kubernetes-more-then-1.19/kubeletImage:v1.22.2 \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=anexia \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --node-ip ${KUBELET_NODE_IP} - ExecStop=-/usr/bin/docker stop %n - Restart=always - RestartSec=10 + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=anexia \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + [Install] WantedBy=multi-user.target - - name: docker.service - enable: true - command: start - drop-ins: - - name: 10-environment.conf - content: | - [Service] - EnvironmentFile=-/etc/environment - - name: apply-sysctl-settings.service enable: true command: start @@ -402,11 +354,6 @@ write_files: PasswordAuthentication no ChallengeResponseAuthentication no -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} - - path: /opt/bin/download.sh permissions: "0755" content: | @@ -477,6 +424,20 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + + mkdir -p /etc/systemd/system/containerd.service.d + + cat < /tmp/environment' - ExecStart=/usr/bin/docker run --name %n \ - --rm --tty --restart no \ - --network host \ - --pid host \ - --env-file /tmp/environment \ - --privileged \ - --cgroup-parent system.slice \ - --entrypoint kubelet \ - -v /dev:/dev \ - -v /etc/cni/net.d:/etc/cni/net.d \ - -v /etc/kubernetes:/etc/kubernetes \ - -v /etc/machine-id:/etc/machine-id:ro \ - -v /etc/os-release:/etc/os-release:ro \ - -v /etc/resolv.conf:/etc/resolv.conf:ro \ - -v /lib/modules:/lib/modules \ - -v /mnt:/mnt:rshared \ - -v /opt/cni/bin:/opt/cni/bin:ro \ - -v /run:/run \ - -v /sys:/sys \ - -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \ - -v /var/lib/calico:/var/lib/calico:ro \ - -v /var/lib/cni:/var/lib/cni \ - -v /var/lib/docker:/var/lib/docker \ - -v /var/lib/kubelet:/var/lib/kubelet:rshared \ - -v /var/log/pods:/var/log/pods \ - for-kubernetes-more-then-1.19/kubeletImage:v1.20.11 \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --node-ip ${KUBELET_NODE_IP} - ExecStop=-/usr/bin/docker stop %n - Restart=always - RestartSec=10 + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + [Install] WantedBy=multi-user.target - - name: docker.service - enable: true - command: start - drop-ins: - - name: 10-environment.conf - content: | - [Service] - EnvironmentFile=-/etc/environment - - name: apply-sysctl-settings.service enable: true command: start @@ -385,11 +337,6 @@ write_files: PasswordAuthentication no ChallengeResponseAuthentication no -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} - - path: /opt/bin/download.sh permissions: "0755" content: | @@ -460,6 +407,20 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + + mkdir -p /etc/systemd/system/containerd.service.d + + cat < Date: Mon, 29 Nov 2021 07:26:32 +0100 Subject: [PATCH 031/489] adjust rhel bootstrapping script (#1111) Signed-off-by: Moath Qasim --- pkg/controller/machine/bootstrap.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/controller/machine/bootstrap.go b/pkg/controller/machine/bootstrap.go index b55dfd29f..313ddedaf 100644 --- a/pkg/controller/machine/bootstrap.go +++ b/pkg/controller/machine/bootstrap.go @@ -142,6 +142,11 @@ func getOSMBootstrapUserDataForCloudInit(ctx context.Context, req plugin.UserDat if err != nil { return "", fmt.Errorf("failed to parse bootstrapYumBinContentTemplate template: %v", err) } + case providerconfigtypes.OperatingSystemRHEL: + bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapYumBinContentTemplate) + if err != nil { + return "", fmt.Errorf("failed to parse bootstrapYumBinContentTemplate template: %v", err) + } } script := &bytes.Buffer{} From 6c0e9389239857024adc587860bd4a330ea2ced1 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Wed, 1 Dec 2021 12:07:02 +0200 Subject: [PATCH 032/489] Default flatcar provisionUtility on AWS to cloud-init (#1119) Signed-off-by: Artiom Diomin --- pkg/admission/machines.go | 6 +++++- pkg/providerconfig/types.go | 9 +++++++-- pkg/providerconfig/types_test.go | 2 +- pkg/userdata/flatcar/flatcar.go | 13 ++++++++++++- 4 files changed, 25 insertions(+), 5 deletions(-) diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index 750fa7f3b..a955938b2 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -124,7 +124,11 @@ func (ad *admissionData) defaultAndValidateMachineSpec(spec *clusterv1alpha1.Mac return fmt.Errorf("Invalid public keys specified: %v", err) } - defaultedOperatingSystemSpec, err := providerconfig.DefaultOperatingSystemSpec(providerConfig.OperatingSystem, providerConfig.OperatingSystemSpec) + defaultedOperatingSystemSpec, err := providerconfig.DefaultOperatingSystemSpec( + providerConfig.OperatingSystem, + providerConfig.CloudProvider, + providerConfig.OperatingSystemSpec, + ) if err != nil { return err } diff --git a/pkg/providerconfig/types.go b/pkg/providerconfig/types.go index 8c1e37d91..a1380b850 100644 --- a/pkg/providerconfig/types.go +++ b/pkg/providerconfig/types.go @@ -156,14 +156,19 @@ func NewConfigVarResolver(ctx context.Context, client ctrlruntimeclient.Client) } } -func DefaultOperatingSystemSpec(osys providerconfigtypes.OperatingSystem, operatingSystemSpec runtime.RawExtension) (runtime.RawExtension, error) { +func DefaultOperatingSystemSpec( + osys providerconfigtypes.OperatingSystem, + cloudProvider providerconfigtypes.CloudProvider, + operatingSystemSpec runtime.RawExtension, +) (runtime.RawExtension, error) { + switch osys { case providerconfigtypes.OperatingSystemAmazonLinux2: return amzn2.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemCentOS: return centos.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemFlatcar: - return flatcar.DefaultConfig(operatingSystemSpec), nil + return flatcar.DefaultConfigForCloud(operatingSystemSpec, cloudProvider), nil case providerconfigtypes.OperatingSystemRHEL: return rhel.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemSLES: diff --git a/pkg/providerconfig/types_test.go b/pkg/providerconfig/types_test.go index c7387856e..896593020 100644 --- a/pkg/providerconfig/types_test.go +++ b/pkg/providerconfig/types_test.go @@ -30,7 +30,7 @@ func TestDefaultOperatingSystemSpec(t *testing.T) { for _, osys := range providerconfigtypes.AllOperatingSystems { osys := osys t.Run(string(osys), func(t *testing.T) { - operatingSystemSpec, err := DefaultOperatingSystemSpec(osys, runtime.RawExtension{}) + operatingSystemSpec, err := DefaultOperatingSystemSpec(osys, "", runtime.RawExtension{}) if err != nil { t.Error("no error expected") diff --git a/pkg/userdata/flatcar/flatcar.go b/pkg/userdata/flatcar/flatcar.go index 938c94b9d..ac07f36d1 100644 --- a/pkg/userdata/flatcar/flatcar.go +++ b/pkg/userdata/flatcar/flatcar.go @@ -19,6 +19,8 @@ package flatcar import ( "encoding/json" + "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8s.io/apimachinery/pkg/runtime" ) @@ -42,8 +44,17 @@ type Config struct { } func DefaultConfig(operatingSystemSpec runtime.RawExtension) runtime.RawExtension { + return DefaultConfigForCloud(operatingSystemSpec, "") +} + +func DefaultConfigForCloud(operatingSystemSpec runtime.RawExtension, cloudProvider types.CloudProvider) runtime.RawExtension { + osSpec := Config{} + if cloudProvider == types.CloudProviderAWS { + osSpec.ProvisioningUtility = CloudInit + } + if operatingSystemSpec.Raw == nil { - operatingSystemSpec.Raw, _ = json.Marshal(Config{}) + operatingSystemSpec.Raw, _ = json.Marshal(osSpec) } return operatingSystemSpec From 9d7ba63ef804f24d904d8d831aa4419af5e32b6f Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 1 Dec 2021 15:59:02 +0500 Subject: [PATCH 033/489] Fix bootstrapping on digital ocean for ubuntu (#1116) * Fix bootstrapping on digital ocean for ubuntu * Refactored code; fix indentation --- pkg/controller/machine/bootstrap.go | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/pkg/controller/machine/bootstrap.go b/pkg/controller/machine/bootstrap.go index 313ddedaf..471c5e8b5 100644 --- a/pkg/controller/machine/bootstrap.go +++ b/pkg/controller/machine/bootstrap.go @@ -113,11 +113,13 @@ func getOSMBootstrapUserDataForCloudInit(ctx context.Context, req plugin.UserDat ServerURL string MachineName string EnterpriseLinux bool + ProviderSpec *providerconfigtypes.Config }{ - Token: token, - SecretName: secretName, - ServerURL: req.Kubeconfig.Clusters[clusterName].Server, - MachineName: req.MachineSpec.Name, + Token: token, + SecretName: secretName, + ServerURL: req.Kubeconfig.Clusters[clusterName].Server, + MachineName: req.MachineSpec.Name, + ProviderSpec: pconfig, } var ( @@ -206,6 +208,14 @@ curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api cloud-init clean cloud-init --file /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg init systemctl daemon-reload + +{{- /* The default cloud-init configurations files have some bug on Digital Ocean which causes the machine to be in-accessible on 2nd cloud-init. We have to manually run the module */}} +{{- if and (eq .ProviderSpec.CloudProvider "digitalocean") (eq .ProviderSpec.OperatingSystem "ubuntu") }} +rm /etc/netplan/50-cloud-init.yaml +netplan generate +netplan apply +{{- end }} + systemctl restart setup.service systemctl restart kubelet.service systemctl restart kubelet-healthcheck.service @@ -263,6 +273,13 @@ write_files: encoding: b64 content: | {{ .Service }} +{{- /* The default cloud-init configurations files have some bug on Digital Ocean which causes the machine to be in-accessible on 2nd cloud-init. Hence we disable network configuration */}} +{{- if and (eq .ProviderSpec.CloudProvider "digitalocean") (eq .ProviderSpec.OperatingSystem "ubuntu") }} +- path: /etc/cloud/cloud.cfg.d/99-custom-networking.cfg + permissions: '0644' + content: | + network: {config: disabled} +{{- end }} runcmd: - systemctl restart bootstrap.service - systemctl daemon-reload From e6b1f0014291e143c6f2ad2047ab86a43831a64e Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 2 Dec 2021 14:17:56 +0500 Subject: [PATCH 034/489] Add bootsrapping support for SLES (#1117) --- pkg/controller/machine/bootstrap.go | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/pkg/controller/machine/bootstrap.go b/pkg/controller/machine/bootstrap.go index 471c5e8b5..f1a031e6a 100644 --- a/pkg/controller/machine/bootstrap.go +++ b/pkg/controller/machine/bootstrap.go @@ -144,6 +144,11 @@ func getOSMBootstrapUserDataForCloudInit(ctx context.Context, req plugin.UserDat if err != nil { return "", fmt.Errorf("failed to parse bootstrapYumBinContentTemplate template: %v", err) } + case providerconfigtypes.OperatingSystemSLES: + bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapZypperBinContentTemplate) + if err != nil { + return "", fmt.Errorf("failed to parse bootstrapZypperBinContentTemplate template: %v", err) + } case providerconfigtypes.OperatingSystemRHEL: bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapYumBinContentTemplate) if err != nil { @@ -227,7 +232,27 @@ set -xeuo pipefail yum install epel-release -y {{- end }} yum install -y curl jq -curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-config"]' -r| base64 -d > /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg +curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-config"]' -r| base64 -d > /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg +cloud-init clean +cloud-init --file /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg init +systemctl daemon-reload +systemctl restart setup.service +systemctl restart kubelet.service +systemctl restart kubelet-healthcheck.service + ` + + bootstrapZypperBinContentTemplate = `#!/bin/bash +set -xeuo pipefail + +# Install JQ +zypper -n --quiet addrepo -C https://download.opensuse.org/repositories/utilities/openSUSE_Leap_15.3/utilities.repo +zypper -n --no-gpg-checks refresh +zypper -n install jq + +# Install CURL +zypper -n install curl + +curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-config"]' -r| base64 -d > /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg cloud-init clean cloud-init --file /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg init systemctl daemon-reload From 745c1434a7c0e0edf8c69ce936055d403934c310 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Thu, 2 Dec 2021 12:39:59 +0100 Subject: [PATCH 035/489] merge KubeVirt tests (#1121) Signed-off-by: Moath Qasim --- .prow.yaml | 24 --------- test/e2e/provisioning/all_e2e_test.go | 25 --------- ...machinedeployment-kubevirt-dns-config.yaml | 51 ------------------- .../testdata/machinedeployment-kubevirt.yaml | 4 ++ 4 files changed, 4 insertions(+), 100 deletions(-) delete mode 100644 test/e2e/provisioning/testdata/machinedeployment-kubevirt-dns-config.yaml diff --git a/.prow.yaml b/.prow.yaml index 6dc74529c..4b58cf1c3 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -199,30 +199,6 @@ presubmits: memory: 1Gi cpu: 500m - - name: pull-machine-controller-e2e-kubevirt-dns-config - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - max_concurrency: 1 - labels: - preset-kubevirt: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.17.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestKubevirtDNSConfigProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - name: pull-machine-controller-e2e-alibaba optional: true always_run: false diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 927bd2362..a760be5db 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -73,7 +73,6 @@ const ( OSUpgradeManifest = "./testdata/machinedeployment-openstack-upgrade.yml" invalidMachineManifest = "./testdata/machine-invalid.yaml" kubevirtManifest = "./testdata/machinedeployment-kubevirt.yaml" - kubevirtManifestDNSConfig = "./testdata/machinedeployment-kubevirt-dns-config.yaml" alibabaManifest = "./testdata/machinedeployment-alibaba.yaml" anexiaManifest = "./testdata/machinedeployment-anexia.yaml" ) @@ -296,30 +295,6 @@ func TestKubevirtProvisioningE2E(t *testing.T) { runScenarios(t, selector, params, kubevirtManifest, fmt.Sprintf("kubevirt-%s", *testRunIdentifier)) } -func TestKubevirtDNSConfigProvisioningE2E(t *testing.T) { - t.Parallel() - - kubevirtKubeconfig := os.Getenv("KUBEVIRT_E2E_TESTS_KUBECONFIG") - - if kubevirtKubeconfig == "" { - t.Fatalf("Unable to run kubevirt tests, KUBEVIRT_E2E_TESTS_KUBECONFIG must be set") - } - - params := []string{ - fmt.Sprintf("<< KUBECONFIG >>=%s", kubevirtKubeconfig), - } - - scenario := scenario{ - name: "Kubevirt with dns config", - osName: "ubuntu", - containerRuntime: "docker", - kubernetesVersion: "v1.22.2", - executor: verifyCreateAndDelete, - } - - testScenario(t, scenario, *testRunIdentifier, params, kubevirtManifestDNSConfig, false) -} - func TestOpenstackProvisioningE2E(t *testing.T) { t.Parallel() diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt-dns-config.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt-dns-config.yaml deleted file mode 100644 index 7a736c903..000000000 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt-dns-config.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: << MACHINE_NAME >> - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - name: << MACHINE_NAME >> - template: - metadata: - labels: - name: << MACHINE_NAME >> - spec: - providerSpec: - value: - sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "kubevirt" - cloudProviderSpec: - storageClassName: local-path - pvcSize: "10Gi" - sourceURL: http://10.107.208.71/<< OS_NAME >>.img - cpus: "1" - memory: "4096M" - dnsPolicy: "None" - dnsConfig: - nameservers: - - 8.8.8.8 - kubeconfig: - value: '<< KUBECONFIG >>' - namespace: kube-system - operatingSystem: "<< OS_NAME >>" - operatingSystemSpec: - distUpgradeOnBoot: false - disableAutoUpdate: true - # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` - rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" - # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` - rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" - versions: - kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index 1c7045f05..936e74675 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -31,6 +31,10 @@ spec: sourceURL: http://10.107.208.71/<< OS_NAME >>.img cpus: "1" memory: "4096M" + dnsPolicy: "None" + dnsConfig: + nameservers: + - 8.8.8.8 kubeconfig: value: '<< KUBECONFIG >>' namespace: kube-system From 07e3ca3aa199ca8ada73fada539ef565b45612a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Thu, 2 Dec 2021 15:05:52 +0100 Subject: [PATCH 036/489] Use cloud-init for Flatcar machines on AWS unconditionally (#1122) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- pkg/userdata/flatcar/flatcar.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/userdata/flatcar/flatcar.go b/pkg/userdata/flatcar/flatcar.go index ac07f36d1..cdb5166e9 100644 --- a/pkg/userdata/flatcar/flatcar.go +++ b/pkg/userdata/flatcar/flatcar.go @@ -39,7 +39,7 @@ type Config struct { DisableUpdateEngine bool `json:"disableUpdateEngine"` // ProvisioningUtility specifies the type of provisioning utility, allowed values are cloud-init and ignition. - // Defaults to ignition. + // Defaults to cloud-init for AWS, and ignition for other providers. ProvisioningUtility `json:"provisioningUtility,omitempty"` } @@ -49,13 +49,15 @@ func DefaultConfig(operatingSystemSpec runtime.RawExtension) runtime.RawExtensio func DefaultConfigForCloud(operatingSystemSpec runtime.RawExtension, cloudProvider types.CloudProvider) runtime.RawExtension { osSpec := Config{} + + if operatingSystemSpec.Raw != nil { + _ = json.Unmarshal(operatingSystemSpec.Raw, &osSpec) + } if cloudProvider == types.CloudProviderAWS { osSpec.ProvisioningUtility = CloudInit } - if operatingSystemSpec.Raw == nil { - operatingSystemSpec.Raw, _ = json.Marshal(osSpec) - } + operatingSystemSpec.Raw, _ = json.Marshal(osSpec) return operatingSystemSpec } From f2c109dd6799732ac0e7f2d20393ba8abee1919e Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Wed, 8 Dec 2021 15:32:20 +0100 Subject: [PATCH 037/489] Add support for gp3 disk type in AWS (#1125) * Add support for gp3 disk type Signed-off-by: Marvin Beckers * Fix error message for gp3 IOPS Signed-off-by: Marvin Beckers * Exclude provider config function from cyclomatic complexity Signed-off-by: Marvin Beckers * Escape linter config correctly Signed-off-by: Marvin Beckers * Fix validation of IOPS for gp3 disks Signed-off-by: Marvin Beckers --- .golangci.yml | 1 + pkg/cloudprovider/provider/aws/provider.go | 11 +++++++++++ 2 files changed, 12 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 666019630..2531e4c67 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -35,3 +35,4 @@ issues: - 'eviction\.go:221:4: the surrounding loop is unconditionally terminated' - 'cyclomatic complexity 31 of func `verifyMigrateUID` is high' - 'cyclomatic complexity 31 of func `main` is high' + - 'cyclomatic complexity 34 of func `\(\*provider\)\.getConfig` is high' diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index ec2778c35..ceda33dac 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -92,6 +92,7 @@ var ( ec2.VolumeTypeStandard, ec2.VolumeTypeIo1, ec2.VolumeTypeGp2, + ec2.VolumeTypeGp3, ec2.VolumeTypeSc1, ec2.VolumeTypeSt1, ) @@ -423,8 +424,18 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return nil, nil, nil, errors.New("Invalid value for `diskIops` (min: 100, max: 64000)") } + c.DiskIops = rawConfig.DiskIops + } else if c.DiskType == ec2.VolumeTypeGp3 && rawConfig.DiskIops != nil { + // gp3 disks start with 3000 IOPS by default, we _can_ pass better IOPS, but it is not a required field + iops := *rawConfig.DiskIops + + if iops < 3000 || iops > 64000 { + return nil, nil, nil, errors.New("Invalid value for `diskIops` (min: 3000, max: 64000)") + } + c.DiskIops = rawConfig.DiskIops } + c.EBSVolumeEncrypted, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.EBSVolumeEncrypted) if err != nil { return nil, nil, nil, fmt.Errorf("failed to get ebsVolumeEncrypted value: %v", err) From 5b6f0f98a0af4d27fec085b8d2c8f76f2385cfc5 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Wed, 15 Dec 2021 14:06:17 +0200 Subject: [PATCH 038/489] New -node-containerd-registry-mirrors flag (#1133) * New -node-containerd-registry-mirrors flag Signed-off-by: Artiom Diomin * Fix empty docker.io mirrors Signed-off-by: Artiom Diomin * Move registryMirrorsFlags to dedicated file Signed-off-by: Artiom Diomin --- cmd/machine-controller/custom_flags.go | 60 +++++++++++++++++++ cmd/machine-controller/main.go | 20 ++++++- pkg/containerruntime/containerd.go | 2 +- pkg/containerruntime/containerruntime.go | 34 +++++------ pkg/containerruntime/docker.go | 3 - pkg/userdata/amzn2/provider_test.go | 4 +- pkg/userdata/centos/provider_test.go | 4 +- pkg/userdata/flatcar/provider_test.go | 2 +- pkg/userdata/helper/helper.go | 43 +++++++------ pkg/userdata/helper/template_functions.go | 1 - pkg/userdata/rhel/provider_test.go | 4 +- .../testdata/kubelet-v1.22-vsphere-proxy.yaml | 11 ++-- pkg/userdata/sles/provider.go | 2 +- pkg/userdata/sles/provider_test.go | 4 +- pkg/userdata/ubuntu/provider_test.go | 4 +- 15 files changed, 141 insertions(+), 57 deletions(-) create mode 100644 cmd/machine-controller/custom_flags.go diff --git a/cmd/machine-controller/custom_flags.go b/cmd/machine-controller/custom_flags.go new file mode 100644 index 000000000..7198766f5 --- /dev/null +++ b/cmd/machine-controller/custom_flags.go @@ -0,0 +1,60 @@ +/* +Copyright 2019 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "sort" + "strings" +) + +type registryMirrorsFlags map[string][]string + +func (fl registryMirrorsFlags) Set(val string) error { + split := strings.SplitN(val, "=", 2) + if len(split) != 2 { + return fmt.Errorf("should have exactly 1 =") + } + + key, value := split[0], split[1] + slice := fl[key] + slice = append(slice, value) + fl[key] = slice + + return nil +} + +func (fl registryMirrorsFlags) String() string { + var ( + registryNames []string + result []string + ) + + for registryName := range fl { + registryNames = append(registryNames, registryName) + } + + sort.Strings(registryNames) + + for _, registryName := range registryNames { + for _, mirror := range fl[registryName] { + result = append(result, fmt.Sprintf("%s=%s", registryName, mirror)) + } + } + + return fmt.Sprintf("%v", result) +} diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 49d1ca052..f2a821e0b 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -23,11 +23,13 @@ import ( "net" "net/http" "net/http/pprof" + "net/url" "strings" "time" "github.com/docker/distribution/reference" "github.com/prometheus/client_golang/prometheus" + osmv1alpha1 "k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -83,6 +85,8 @@ var ( nodeContainerRuntime string podCidr string nodePortRange string + + nodeContainerdRegistryMirrors = registryMirrorsFlags{} ) const ( @@ -163,6 +167,7 @@ func main() { flag.StringVar(&nodePauseImage, "node-pause-image", "", "Image for the pause container including tag. If not set, the kubelet default will be used: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/") flag.StringVar(&nodeKubeletRepository, "node-kubelet-repository", "quay.io/kubermatic/kubelet", "Repository for the kubelet container. Only has effect on Flatcar Linux.") flag.StringVar(&nodeContainerRuntime, "node-container-runtime", "docker", "container-runtime to deploy") + flag.Var(&nodeContainerdRegistryMirrors, "node-containerd-registry-mirrors", "Configure registry mirrors endpoints. Can be used multiple times to specify multiple mirrors") flag.StringVar(&caBundleFile, "ca-bundle", "", "path to a file containing all PEM-encoded CA certificates (will be used instead of the host's certificates if set)") flag.BoolVar(&nodeCSRApprover, "node-csr-approver", true, "Enable NodeCSRApprover controller to automatically approve node serving certificate requests") flag.StringVar(&podCidr, "pod-cidr", "172.25.0.0/16", "The network ranges from which POD networks are allocated") @@ -253,10 +258,23 @@ func main() { var registryMirrors []string for _, mirror := range strings.Split(nodeRegistryMirrors, ",") { if trimmedMirror := strings.TrimSpace(mirror); trimmedMirror != "" { + if !strings.HasPrefix(mirror, "http") { + trimmedMirror = "https://" + mirror + } + + _, err := url.Parse(trimmedMirror) + if err != nil { + klog.Fatalf("incorrect mirror provided: %v", err) + } + registryMirrors = append(registryMirrors, trimmedMirror) } } + if len(registryMirrors) > 0 { + nodeContainerdRegistryMirrors["docker.io"] = registryMirrors + } + runOptions := controllerRunOptions{ kubeClient: kubeClient, kubeconfigProvider: kubeconfigProvider, @@ -275,7 +293,7 @@ func main() { ContainerRuntime: containerruntime.Get( nodeContainerRuntime, containerruntime.WithInsecureRegistries(insecureRegistries), - containerruntime.WithRegistryMirrors(registryMirrors), + containerruntime.WithRegistryMirrors(nodeContainerdRegistryMirrors), containerruntime.WithSandboxImage(nodePauseImage), ), }, diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index 28e4e4203..7a84a65f4 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -31,7 +31,7 @@ const ( type Containerd struct { insecureRegistries []string - registryMirrors []string + registryMirrors map[string][]string sandboxImage string } diff --git a/pkg/containerruntime/containerruntime.go b/pkg/containerruntime/containerruntime.go index 3c09ac3c3..dccc48eec 100644 --- a/pkg/containerruntime/containerruntime.go +++ b/pkg/containerruntime/containerruntime.go @@ -43,7 +43,7 @@ func WithInsecureRegistries(registries []string) Opt { } } -func WithRegistryMirrors(mirrors []string) Opt { +func WithRegistryMirrors(mirrors map[string][]string) Opt { return func(cfg *Config) { cfg.RegistryMirrors = mirrors } @@ -78,11 +78,11 @@ func Get(containerRuntimeName string, opts ...Opt) Config { } type Config struct { - Docker *Docker `json:",omitempty"` - Containerd *Containerd `json:",omitempty"` - InsecureRegistries []string `json:",omitempty"` - RegistryMirrors []string `json:",omitempty"` - SandboxImage string `json:",omitempty"` + Docker *Docker `json:",omitempty"` + Containerd *Containerd `json:",omitempty"` + InsecureRegistries []string `json:",omitempty"` + RegistryMirrors map[string][]string `json:",omitempty"` + SandboxImage string `json:",omitempty"` } func (cfg Config) String() string { @@ -97,18 +97,16 @@ func (cfg Config) String() string { } func (cfg Config) Engine(kubeletVersion *semver.Version) Engine { - var ( - docker = &Docker{ - insecureRegistries: cfg.InsecureRegistries, - registryMirrors: cfg.RegistryMirrors, - kubeletVersion: kubeletVersion, - } - containerd = &Containerd{ - insecureRegistries: cfg.InsecureRegistries, - registryMirrors: cfg.RegistryMirrors, - sandboxImage: cfg.SandboxImage, - } - ) + docker := &Docker{ + insecureRegistries: cfg.InsecureRegistries, + registryMirrors: cfg.RegistryMirrors["docker.io"], + } + + containerd := &Containerd{ + insecureRegistries: cfg.InsecureRegistries, + registryMirrors: cfg.RegistryMirrors, + sandboxImage: cfg.SandboxImage, + } moreThan122, _ := semver.NewConstraint(">= 1.22") diff --git a/pkg/containerruntime/docker.go b/pkg/containerruntime/docker.go index c701a0f0e..0c545d94a 100644 --- a/pkg/containerruntime/docker.go +++ b/pkg/containerruntime/docker.go @@ -21,8 +21,6 @@ import ( "strings" "text/template" - "github.com/Masterminds/semver/v3" - "github.com/kubermatic/machine-controller/pkg/providerconfig/types" "github.com/kubermatic/machine-controller/pkg/userdata/helper" ) @@ -33,7 +31,6 @@ const ( ) type Docker struct { - kubeletVersion *semver.Version insecureRegistries []string registryMirrors []string } diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index e67cccec0..6d77d9149 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -88,7 +88,7 @@ type userDataTestCase struct { httpProxy string noProxy string insecureRegistries []string - registryMirrors []string + registryMirrors map[string][]string pauseImage string containerruntime string } @@ -181,7 +181,7 @@ func TestUserDataGeneration(t *testing.T) { cloudProviderName: stringPtr("vsphere"), httpProxy: "http://192.168.100.100:3128/", noProxy: "192.168.1.0", - registryMirrors: []string{"https://registry.docker-cn.com/"}, + registryMirrors: map[string][]string{"docker.io": {"https://registry.docker-cn.com/"}}, pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index fb02d17e0..55660717b 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -88,7 +88,7 @@ type userDataTestCase struct { httpProxy string noProxy string insecureRegistries []string - registryMirrors []string + registryMirrors map[string][]string pauseImage string containerruntime string } @@ -181,7 +181,7 @@ func TestUserDataGeneration(t *testing.T) { cloudProviderName: stringPtr("vsphere"), httpProxy: "http://192.168.100.100:3128/", noProxy: "192.168.1.0", - registryMirrors: []string{"https://registry.docker-cn.com/"}, + registryMirrors: map[string][]string{"docker.io": {"https://registry.docker-cn.com/"}}, pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index e46cb739b..7d6742496 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -110,7 +110,7 @@ type userDataTestCase struct { httpProxy string noProxy string insecureRegistries []string - registryMirrors []string + registryMirrors map[string][]string pauseImage string hyperkubeImage string kubeletImage string diff --git a/pkg/userdata/helper/helper.go b/pkg/userdata/helper/helper.go index 2fce30f20..ea6798fde 100644 --- a/pkg/userdata/helper/helper.go +++ b/pkg/userdata/helper/helper.go @@ -133,14 +133,23 @@ type containerdCRIRuncOptions struct { } type containerdCRIRegistry struct { - Mirrors map[string]containerdMirror `toml:"mirrors"` + Mirrors map[string]containerdRegistryMirror `toml:"mirrors"` + Configs map[string]containerdRegistryConfig `toml:"configs"` } -type containerdMirror struct { +type containerdRegistryMirror struct { Endpoint []string `toml:"endpoint"` } -func ContainerdConfig(insecureRegistries, registryMirrors []string, sandboxImage string) (string, error) { +type containerdRegistryConfig struct { + TLS *containerdRegistryTLSConfig `toml:"tls"` +} + +type containerdRegistryTLSConfig struct { + InsecureSkipVerify bool `toml:"insecure_skip_verify"` +} + +func ContainerdConfig(insecureRegistries []string, registryMirrors map[string][]string, sandboxImage string) (string, error) { criPlugin := containerdCRIPlugin{ SandboxImage: sandboxImage, Containerd: &containerdCRISettings{ @@ -154,7 +163,7 @@ func ContainerdConfig(insecureRegistries, registryMirrors []string, sandboxImage }, }, Registry: &containerdCRIRegistry{ - Mirrors: map[string]containerdMirror{ + Mirrors: map[string]containerdRegistryMirror{ "docker.io": { Endpoint: []string{"https://registry-1.docker.io/"}, }, @@ -162,15 +171,21 @@ func ContainerdConfig(insecureRegistries, registryMirrors []string, sandboxImage }, } - for _, insecureRegistry := range insecureRegistries { - criPlugin.Registry.Mirrors[insecureRegistry] = containerdMirror{ - Endpoint: []string{fmt.Sprintf("http://%s", insecureRegistry)}, - } + for registryName := range registryMirrors { + registry := criPlugin.Registry.Mirrors[registryName] + registry.Endpoint = registryMirrors[registryName] + criPlugin.Registry.Mirrors[registryName] = registry } - if len(registryMirrors) > 0 { - criPlugin.Registry.Mirrors["docker.io"] = containerdMirror{ - Endpoint: registryMirrors, + if len(insecureRegistries) > 0 { + criPlugin.Registry.Configs = map[string]containerdRegistryConfig{} + } + + for _, registry := range insecureRegistries { + criPlugin.Registry.Configs[registry] = containerdRegistryConfig{ + TLS: &containerdRegistryTLSConfig{ + InsecureSkipVerify: true, + }, } } @@ -217,12 +232,6 @@ func DockerConfig(insecureRegistries, registryMirrors []string) (string, error) InsecureRegistries: insecureRegistries, RegistryMirrors: registryMirrors, } - if insecureRegistries == nil { - cfg.InsecureRegistries = []string{} - } - if registryMirrors == nil { - cfg.RegistryMirrors = []string{} - } b, err := json.Marshal(cfg) return string(b), err diff --git a/pkg/userdata/helper/template_functions.go b/pkg/userdata/helper/template_functions.go index c8db76532..62ecbade8 100644 --- a/pkg/userdata/helper/template_functions.go +++ b/pkg/userdata/helper/template_functions.go @@ -39,7 +39,6 @@ func TxtFuncMap() template.FuncMap { funcMap["kubeletHealthCheckSystemdUnit"] = KubeletHealthCheckSystemdUnit funcMap["containerRuntimeHealthCheckSystemdUnit"] = ContainerRuntimeHealthCheckSystemdUnit funcMap["dockerConfig"] = DockerConfig - funcMap["containerdConfig"] = ContainerdConfig funcMap["proxyEnvironment"] = ProxyEnvironment funcMap["sshConfigAddendum"] = SSHConfigAddendum diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index c7d4620b7..e1170990d 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -88,7 +88,7 @@ type userDataTestCase struct { httpProxy string noProxy string insecureRegistries []string - registryMirrors []string + registryMirrors map[string][]string pauseImage string containerruntime string } @@ -190,7 +190,7 @@ func TestUserDataGeneration(t *testing.T) { cloudProviderName: stringPtr("vsphere"), httpProxy: "http://192.168.100.100:3128/", noProxy: "192.168.1.0", - registryMirrors: []string{"https://registry.docker-cn.com/"}, + registryMirrors: map[string][]string{"docker.io": {"https://registry.docker-cn.com/"}}, pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, } diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-proxy.yaml index 6b1771de8..b931b31a3 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-proxy.yaml @@ -429,12 +429,15 @@ write_files: SystemdCgroup = true [plugins."io.containerd.grpc.v1.cri".registry] [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."10.0.0.1:5000"] - endpoint = ["http://10.0.0.1:5000/"] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."192.168.100.100:5000"] - endpoint = ["http://192.168.100.100:5000/"] [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] endpoint = ["https://registry-1.docker.io/"] + [plugins."io.containerd.grpc.v1.cri".registry.configs] + [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000".tls] + insecure_skip_verify = true + [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000".tls] + insecure_skip_verify = true - path: /etc/systemd/system/kubelet-healthcheck.service diff --git a/pkg/userdata/sles/provider.go b/pkg/userdata/sles/provider.go index b5cc52f88..ba5967e55 100644 --- a/pkg/userdata/sles/provider.go +++ b/pkg/userdata/sles/provider.go @@ -104,7 +104,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { KubernetesCACert: kubernetesCACert, NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), InsecureRegistries: req.ContainerRuntime.InsecureRegistries, - RegistryMirrors: req.ContainerRuntime.RegistryMirrors, + RegistryMirrors: req.ContainerRuntime.RegistryMirrors["docker.io"], } b := &bytes.Buffer{} err = tmpl.Execute(b, data) diff --git a/pkg/userdata/sles/provider_test.go b/pkg/userdata/sles/provider_test.go index 5d52d85af..f4d64cfea 100644 --- a/pkg/userdata/sles/provider_test.go +++ b/pkg/userdata/sles/provider_test.go @@ -119,7 +119,7 @@ type userDataTestCase struct { httpProxy string noProxy string insecureRegistries []string - registryMirrors []string + registryMirrors map[string][]string pauseImage string containerruntime string } @@ -350,7 +350,7 @@ func TestUserDataGeneration(t *testing.T) { }, httpProxy: "http://192.168.100.100:3128/", noProxy: "192.168.1.0", - registryMirrors: []string{"https://registry.docker-cn.com/"}, + registryMirrors: map[string][]string{"docker.io": {"https://registry.docker-cn.com/"}}, pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 419f66b29..e5dedb836 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -118,7 +118,7 @@ type userDataTestCase struct { httpProxy string noProxy string insecureRegistries []string - registryMirrors []string + registryMirrors map[string][]string pauseImage string containerruntime string } @@ -404,7 +404,7 @@ func TestUserDataGeneration(t *testing.T) { }, httpProxy: "http://192.168.100.100:3128/", noProxy: "192.168.1.0", - registryMirrors: []string{"https://registry.docker-cn.com/"}, + registryMirrors: map[string][]string{"docker.io": {"https://registry.docker-cn.com/"}}, pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { From 791cfbcd0238d33314ed0161bc4f9cc549bb5c75 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Wed, 15 Dec 2021 19:24:12 +0200 Subject: [PATCH 039/489] Remove unused hyperkube and kubelet repository (#1134) Signed-off-by: Artiom Diomin --- cmd/machine-controller/main.go | 23 +++++--------------- go.mod | 1 - pkg/apis/plugin/plugin.go | 2 -- pkg/controller/machine/machine_controller.go | 6 ----- pkg/userdata/flatcar/provider.go | 17 --------------- pkg/userdata/flatcar/provider_test.go | 22 ------------------- 6 files changed, 5 insertions(+), 66 deletions(-) diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index f2a821e0b..455c25b03 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -27,9 +27,7 @@ import ( "strings" "time" - "github.com/docker/distribution/reference" "github.com/prometheus/client_golang/prometheus" - osmv1alpha1 "k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -81,7 +79,6 @@ var ( nodeInsecureRegistries string nodeRegistryMirrors string nodePauseImage string - nodeKubeletRepository string nodeContainerRuntime string podCidr string nodePortRange string @@ -165,7 +162,7 @@ func main() { flag.StringVar(&nodeInsecureRegistries, "node-insecure-registries", "", "Comma separated list of registries which should be configured as insecure on the container runtime") flag.StringVar(&nodeRegistryMirrors, "node-registry-mirrors", "", "Comma separated list of Docker image mirrors") flag.StringVar(&nodePauseImage, "node-pause-image", "", "Image for the pause container including tag. If not set, the kubelet default will be used: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/") - flag.StringVar(&nodeKubeletRepository, "node-kubelet-repository", "quay.io/kubermatic/kubelet", "Repository for the kubelet container. Only has effect on Flatcar Linux.") + flag.String("node-kubelet-repository", "quay.io/kubermatic/kubelet", "[NO-OP] Repository for the kubelet container. Has no effects.") flag.StringVar(&nodeContainerRuntime, "node-container-runtime", "docker", "container-runtime to deploy") flag.Var(&nodeContainerdRegistryMirrors, "node-containerd-registry-mirrors", "Configure registry mirrors endpoints. Can be used multiple times to specify multiple mirrors") flag.StringVar(&caBundleFile, "ca-bundle", "", "path to a file containing all PEM-encoded CA certificates (will be used instead of the host's certificates if set)") @@ -209,15 +206,6 @@ func main() { klog.Fatalf("failed to add osmv1alpha1 api to scheme: %v", err) } - // Check if the kubelet image has a tag set - kubeletRepoRef, err := reference.Parse(nodeKubeletRepository) - if err != nil { - klog.Fatalf("failed to parse -node-kubelet-repository %s: %v", nodeKubeletRepository, err) - } - if _, ok := kubeletRepoRef.(reference.NamedTagged); ok { - klog.Fatalf("-node-kubelet-repository must not contain a tag. The tag will be dynamically set for each Machine.") - } - cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) if err != nil { klog.Fatalf("error building kubeconfig: %v", err) @@ -285,11 +273,10 @@ func main() { skipEvictionAfter: skipEvictionAfter, nodeCSRApprover: nodeCSRApprover, node: machinecontroller.NodeSettings{ - ClusterDNSIPs: clusterDNSIPs, - HTTPProxy: nodeHTTPProxy, - KubeletRepository: nodeKubeletRepository, - NoProxy: nodeNoProxy, - PauseImage: nodePauseImage, + ClusterDNSIPs: clusterDNSIPs, + HTTPProxy: nodeHTTPProxy, + NoProxy: nodeNoProxy, + PauseImage: nodePauseImage, ContainerRuntime: containerruntime.Get( nodeContainerRuntime, containerruntime.WithInsecureRegistries(insecureRegistries), diff --git a/go.mod b/go.mod index 95ffd2bbb..5b8e3ae72 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,6 @@ require ( github.com/coreos/container-linux-config-transpiler v0.9.0 github.com/davecgh/go-spew v1.1.1 github.com/digitalocean/godo v1.54.0 - github.com/docker/distribution v2.7.1+incompatible github.com/ghodss/yaml v1.0.0 github.com/go-test/deep v1.0.7 github.com/google/uuid v1.1.2 diff --git a/pkg/apis/plugin/plugin.go b/pkg/apis/plugin/plugin.go index d4a25b71d..18fe8a09f 100644 --- a/pkg/apis/plugin/plugin.go +++ b/pkg/apis/plugin/plugin.go @@ -50,8 +50,6 @@ type UserDataRequest struct { HTTPProxy string NoProxy string PauseImage string - HyperkubeImage string - KubeletRepository string KubeletFeatureGates map[string]bool ContainerRuntime containerruntime.Config PodCIDR string diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 693974d53..c6c78f1a9 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -133,10 +133,6 @@ type NodeSettings struct { RegistryMirrors []string // Translates to --pod-infra-container-image on the kubelet. If not set, the kubelet will default it. PauseImage string - // The hyperkube image to use. Currently only Container Linux and Flatcar Linux uses it. - HyperkubeImage string - // The kubelet repository to use. Currently only Flatcar Linux uses it. - KubeletRepository string // Translates to feature gates on the kubelet. // Default: RotateKubeletServerCertificate=true KubeletFeatureGates map[string]bool @@ -732,8 +728,6 @@ func (r *Reconciler) ensureInstanceExistsForMachine( ExternalCloudProvider: externalCloudProvider, DNSIPs: r.nodeSettings.ClusterDNSIPs, PauseImage: r.nodeSettings.PauseImage, - HyperkubeImage: r.nodeSettings.HyperkubeImage, - KubeletRepository: r.nodeSettings.KubeletRepository, KubeletFeatureGates: kubeletFeatureGates, NoProxy: r.nodeSettings.NoProxy, HTTPProxy: r.nodeSettings.HTTPProxy, diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index 65b94700c..e72ad632d 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -33,10 +33,6 @@ import ( userdatahelper "github.com/kubermatic/machine-controller/pkg/userdata/helper" ) -const ( - lessThen119Check = "< 1.19" -) - // Provider is a pkg/userdata/plugin.Provider implementation. type Provider struct{} @@ -86,17 +82,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { flatcarConfig.DisableUpdateEngine = true } - kubeletImage := req.KubeletRepository - lessThen119, err := semver.NewConstraint(lessThen119Check) - if err != nil { - return "", err - } - - if lessThen119.Check(kubeletVersion) { - kubeletImage = req.HyperkubeImage - } - kubeletImage = kubeletImage + ":v" + kubeletVersion.String() - crEngine := req.ContainerRuntime.Engine(kubeletVersion) crScript, err := crEngine.ScriptFor(providerconfigtypes.OperatingSystemFlatcar) if err != nil { @@ -114,7 +99,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { FlatcarConfig *Config Kubeconfig string KubernetesCACert string - KubeletImage string KubeletVersion string NodeIPScript string ExtraKubeletFlags []string @@ -128,7 +112,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { FlatcarConfig: flatcarConfig, Kubeconfig: kubeconfigString, KubernetesCACert: kubernetesCACert, - KubeletImage: kubeletImage, KubeletVersion: kubeletVersion.String(), NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), ExtraKubeletFlags: crEngine.KubeletFlags(), diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index 7d6742496..9c9509c69 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -112,8 +112,6 @@ type userDataTestCase struct { insecureRegistries []string registryMirrors map[string][]string pauseImage string - hyperkubeImage string - kubeletImage string containerruntime string } @@ -153,8 +151,6 @@ func TestUserDataGeneration(t *testing.T) { DisableAutoUpdate: true, ProvisioningUtility: Ignition, }, - hyperkubeImage: "for-kubernetes-less-then-1.19/hyperkubeImage", - kubeletImage: "for-kubernetes-more-then-1.19/kubeletImage", }, { name: "ignition_v1.20.11", @@ -186,8 +182,6 @@ func TestUserDataGeneration(t *testing.T) { DisableAutoUpdate: true, ProvisioningUtility: Ignition, }, - hyperkubeImage: "for-kubernetes-less-then-1.19/hyperkubeImage", - kubeletImage: "for-kubernetes-more-then-1.19/kubeletImage", }, { name: "ignition_v1.21.5", @@ -219,8 +213,6 @@ func TestUserDataGeneration(t *testing.T) { DisableAutoUpdate: true, ProvisioningUtility: Ignition, }, - hyperkubeImage: "for-kubernetes-less-then-1.19/hyperkubeImage", - kubeletImage: "for-kubernetes-more-then-1.19/kubeletImage", }, { name: "ignition_v1.22.2", @@ -252,8 +244,6 @@ func TestUserDataGeneration(t *testing.T) { DisableAutoUpdate: true, ProvisioningUtility: Ignition, }, - hyperkubeImage: "for-kubernetes-less-then-1.19/hyperkubeImage", - kubeletImage: "for-kubernetes-more-then-1.19/kubeletImage", }, { name: "cloud-init_v1.19.15", @@ -285,8 +275,6 @@ func TestUserDataGeneration(t *testing.T) { DisableAutoUpdate: true, ProvisioningUtility: CloudInit, }, - hyperkubeImage: "for-kubernetes-less-then-1.19/hyperkubeImage", - kubeletImage: "for-kubernetes-more-then-1.19/kubeletImage", }, { name: "cloud-init_v1.20.11", @@ -318,8 +306,6 @@ func TestUserDataGeneration(t *testing.T) { DisableAutoUpdate: true, ProvisioningUtility: CloudInit, }, - hyperkubeImage: "for-kubernetes-less-then-1.19/hyperkubeImage", - kubeletImage: "for-kubernetes-more-then-1.19/kubeletImage", }, { name: "cloud-init_v1.21.5", @@ -351,8 +337,6 @@ func TestUserDataGeneration(t *testing.T) { DisableAutoUpdate: true, ProvisioningUtility: CloudInit, }, - hyperkubeImage: "for-kubernetes-less-then-1.19/hyperkubeImage", - kubeletImage: "for-kubernetes-more-then-1.19/kubeletImage", }, { name: "cloud-init_v1.22.2", @@ -384,8 +368,6 @@ func TestUserDataGeneration(t *testing.T) { DisableAutoUpdate: true, ProvisioningUtility: CloudInit, }, - hyperkubeImage: "for-kubernetes-less-then-1.19/hyperkubeImage", - kubeletImage: "for-kubernetes-more-then-1.19/kubeletImage", }, { name: "containerd", @@ -408,8 +390,6 @@ func TestUserDataGeneration(t *testing.T) { DisableAutoUpdate: true, ProvisioningUtility: CloudInit, }, - hyperkubeImage: "for-kubernetes-less-then-1.19/hyperkubeImage", - kubeletImage: "for-kubernetes-more-then-1.19/kubeletImage", }, } @@ -450,8 +430,6 @@ func TestUserDataGeneration(t *testing.T) { HTTPProxy: test.httpProxy, NoProxy: test.noProxy, PauseImage: test.pauseImage, - HyperkubeImage: test.hyperkubeImage, - KubeletRepository: test.kubeletImage, KubeletFeatureGates: kubeletFeatureGates, ContainerRuntime: containerruntime.Get( test.containerruntime, From 953f239aab92b8dd7c210c66437106f8cf637dbb Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Sun, 19 Dec 2021 22:48:18 +0200 Subject: [PATCH 040/489] Kubelet configuration Machine annotations (#1135) * New v1.kubelet-config.machine-controller.kubermatic.io annotation To pass some kubeletConfiguraton custom values. Possible values are: * SystemReserved * KubeReserved * EvictionHard If those found, serialized map will be used to popualate corresponding kubeletConfiguraton field. Signed-off-by: Artiom Diomin * Update fixtures Signed-off-by: Artiom Diomin --- pkg/apis/cluster/common/consts.go | 33 ++++++++++++++---- pkg/apis/plugin/plugin.go | 1 + pkg/controller/machine/machine_controller.go | 6 ++-- pkg/userdata/amzn2/provider.go | 2 +- .../containerd-kubelet-v1.20-aws.yaml | 5 +++ .../amzn2/testdata/kubelet-v1.19-aws.yaml | 5 +++ .../amzn2/testdata/kubelet-v1.20-aws.yaml | 5 +++ .../testdata/kubelet-v1.21-aws-external.yaml | 5 +++ .../amzn2/testdata/kubelet-v1.21-aws.yaml | 5 +++ .../kubelet-v1.21-vsphere-mirrors.yaml | 5 +++ .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 5 +++ .../amzn2/testdata/kubelet-v1.21-vsphere.yaml | 5 +++ .../amzn2/testdata/kubelet-v1.22-aws.yaml | 5 +++ pkg/userdata/centos/provider.go | 2 +- .../kubelet-containerd-v1.20-aws.yaml | 5 +++ .../centos/testdata/kubelet-v1.19-aws.yaml | 5 +++ .../centos/testdata/kubelet-v1.20-aws.yaml | 5 +++ .../testdata/kubelet-v1.21-aws-external.yaml | 5 +++ .../centos/testdata/kubelet-v1.21-aws.yaml | 5 +++ .../kubelet-v1.21-vsphere-mirrors.yaml | 5 +++ .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 5 +++ .../testdata/kubelet-v1.21-vsphere.yaml | 5 +++ .../centos/testdata/kubelet-v1.22-aws.yaml | 5 +++ pkg/userdata/flatcar/provider.go | 4 +-- .../flatcar/testdata/cloud-init_v1.19.15.yaml | 5 +++ .../flatcar/testdata/cloud-init_v1.20.11.yaml | 5 +++ .../flatcar/testdata/cloud-init_v1.21.5.yaml | 5 +++ .../flatcar/testdata/cloud-init_v1.22.2.yaml | 5 +++ pkg/userdata/flatcar/testdata/containerd.yaml | 5 +++ .../flatcar/testdata/ignition_v1.19.15.json | 2 +- .../flatcar/testdata/ignition_v1.20.11.json | 2 +- .../flatcar/testdata/ignition_v1.21.5.json | 2 +- .../flatcar/testdata/ignition_v1.22.2.json | 2 +- pkg/userdata/helper/kubelet.go | 34 ++++++++++++++++++- pkg/userdata/rhel/provider.go | 2 +- .../kubelet-containerd-v1.19-aws.yaml | 5 +++ .../rhel/testdata/kubelet-v1.19-aws.yaml | 5 +++ .../rhel/testdata/kubelet-v1.20-aws.yaml | 5 +++ .../rhel/testdata/kubelet-v1.21-aws.yaml | 5 +++ .../testdata/kubelet-v1.22-aws-external.yaml | 5 +++ .../rhel/testdata/kubelet-v1.22-aws.yaml | 5 +++ .../kubelet-v1.22-vsphere-mirrors.yaml | 5 +++ .../testdata/kubelet-v1.22-vsphere-proxy.yaml | 5 +++ .../rhel/testdata/kubelet-v1.22-vsphere.yaml | 5 +++ pkg/userdata/sles/provider.go | 2 +- .../sles/testdata/dist-upgrade-on-boot.yaml | 5 +++ .../kubelet-version-without-v-prefix.yaml | 5 +++ .../sles/testdata/multiple-dns-servers.yaml | 5 +++ .../sles/testdata/multiple-ssh-keys.yaml | 5 +++ .../openstack-overwrite-cloud-config.yaml | 5 +++ pkg/userdata/sles/testdata/openstack.yaml | 5 +++ .../sles/testdata/version-1.19.15.yaml | 5 +++ .../sles/testdata/version-1.20.11.yaml | 5 +++ .../sles/testdata/version-1.21.5.yaml | 5 +++ .../sles/testdata/version-1.22.2.yaml | 5 +++ .../sles/testdata/vsphere-mirrors.yaml | 5 +++ pkg/userdata/sles/testdata/vsphere-proxy.yaml | 5 +++ pkg/userdata/sles/testdata/vsphere.yaml | 5 +++ pkg/userdata/ubuntu/provider.go | 2 +- pkg/userdata/ubuntu/testdata/containerd.yaml | 5 +++ .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 5 +++ .../kubelet-version-without-v-prefix.yaml | 5 +++ .../ubuntu/testdata/multiple-dns-servers.yaml | 5 +++ .../ubuntu/testdata/multiple-ssh-keys.yaml | 5 +++ .../openstack-overwrite-cloud-config.yaml | 5 +++ pkg/userdata/ubuntu/testdata/openstack.yaml | 5 +++ .../ubuntu/testdata/version-1.19.15.yaml | 5 +++ .../ubuntu/testdata/version-1.20.11.yaml | 5 +++ .../ubuntu/testdata/version-1.21.5.yaml | 5 +++ .../ubuntu/testdata/version-1.22.2.yaml | 5 +++ .../ubuntu/testdata/vsphere-mirrors.yaml | 5 +++ .../ubuntu/testdata/vsphere-proxy.yaml | 5 +++ pkg/userdata/ubuntu/testdata/vsphere.yaml | 5 +++ 73 files changed, 371 insertions(+), 20 deletions(-) diff --git a/pkg/apis/cluster/common/consts.go b/pkg/apis/cluster/common/consts.go index a84c46314..056a80c21 100644 --- a/pkg/apis/cluster/common/consts.go +++ b/pkg/apis/cluster/common/consts.go @@ -134,10 +134,17 @@ const ( ExternalCloudProviderKubeletFlag KubeletFlags = "ExternalCloudProvider" ) +const ( + SystemReservedKubeletConfig = "SystemReserved" + KubeReservedKubeletConfig = "KubeReserved" + EvictionHardKubeletConfig = "EvictionHard" +) + const ( // Annotation prefixes, used on Machine objects to indicate the parameters that been used to create those Machines KubeletFeatureGatesAnnotationPrefixV1 = "v1.kubelet-featuregates.machine-controller.kubermatic.io" KubeletFlagsGroupAnnotationPrefixV1 = "v1.kubelet-flags.machine-controller.kubermatic.io" + KubeletConfigAnnotationPrefixV1 = "v1.kubelet-config.machine-controller.kubermatic.io" ) // SetKubeletFeatureGates marshal and save featureGates into metaobject annotations with @@ -165,11 +172,25 @@ func SetKubeletFlags(metaobj metav1.Object, flags map[KubeletFlags]string) { metaobj.SetAnnotations(annts) } -func GetKubeletFeatureGates(metaobj metav1.Object) map[string]bool { +func GetKubeletConfigs(annotations map[string]string) map[string]string { + configs := map[string]string{} + for name, value := range annotations { + if strings.HasPrefix(name, KubeletConfigAnnotationPrefixV1) { + nameConfigValue := strings.SplitN(name, "/", 2) + if len(nameConfigValue) != 2 { + continue + } + configs[nameConfigValue[1]] = value + } + } + return configs +} + +func GetKubeletFeatureGates(annotations map[string]string) map[string]bool { result := map[string]bool{} - for name, value := range metaobj.GetAnnotations() { + for name, value := range annotations { if strings.HasPrefix(name, KubeletFeatureGatesAnnotationPrefixV1) { - nameGateValue := strings.Split(name, "/") + nameGateValue := strings.SplitN(name, "/", 2) if len(nameGateValue) != 2 { continue } @@ -180,11 +201,11 @@ func GetKubeletFeatureGates(metaobj metav1.Object) map[string]bool { return result } -func GetKubeletFlags(metaobj metav1.Object) map[KubeletFlags]string { +func GetKubeletFlags(annotations map[string]string) map[KubeletFlags]string { result := map[KubeletFlags]string{} - for name, value := range metaobj.GetAnnotations() { + for name, value := range annotations { if strings.HasPrefix(name, KubeletFlagsGroupAnnotationPrefixV1) { - nameFlagValue := strings.Split(name, "/") + nameFlagValue := strings.SplitN(name, "/", 2) if len(nameFlagValue) != 2 { continue } diff --git a/pkg/apis/plugin/plugin.go b/pkg/apis/plugin/plugin.go index 18fe8a09f..5c5797e4d 100644 --- a/pkg/apis/plugin/plugin.go +++ b/pkg/apis/plugin/plugin.go @@ -51,6 +51,7 @@ type UserDataRequest struct { NoProxy string PauseImage string KubeletFeatureGates map[string]bool + KubeletConfigs map[string]string ContainerRuntime containerruntime.Config PodCIDR string NodePortRange string diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index c6c78f1a9..533b6d222 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -705,14 +705,15 @@ func (r *Reconciler) ensureInstanceExistsForMachine( } // grab kubelet featureGates from the annotations - kubeletFeatureGates := common.GetKubeletFeatureGates(machine) + kubeletFeatureGates := common.GetKubeletFeatureGates(machine.GetAnnotations()) if len(kubeletFeatureGates) == 0 { // fallback to command-line input kubeletFeatureGates = r.nodeSettings.KubeletFeatureGates } // grab kubelet general options from the annotations - kubeletFlags := common.GetKubeletFlags(machine) + kubeletFlags := common.GetKubeletFlags(machine.GetAnnotations()) + KubeletConfigs := common.GetKubeletConfigs(machine.GetAnnotations()) // look up for ExternalCloudProvider feature, with fallback to command-line input externalCloudProvider := r.nodeSettings.ExternalCloudProvider @@ -729,6 +730,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( DNSIPs: r.nodeSettings.ClusterDNSIPs, PauseImage: r.nodeSettings.PauseImage, KubeletFeatureGates: kubeletFeatureGates, + KubeletConfigs: KubeletConfigs, NoProxy: r.nodeSettings.NoProxy, HTTPProxy: r.nodeSettings.HTTPProxy, ContainerRuntime: r.nodeSettings.ContainerRuntime, diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index 554dbd415..05da14cc5 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -265,7 +265,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - path: "/etc/kubernetes/pki/ca.crt" content: | diff --git a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml index bd5a7d25f..d010fef17 100644 --- a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml @@ -300,6 +300,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml index 76d1bc1f9..55814f194 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml @@ -297,6 +297,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml index 6a4e85c10..54ccf66c0 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml @@ -297,6 +297,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml index 5876eaf44..2cc89d9f1 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml @@ -297,6 +297,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml index 36972a6bf..feccb64d3 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml @@ -297,6 +297,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 0d8ea0dac..14718cb55 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -314,6 +314,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml index 8b9e3525e..2ff8bba55 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -314,6 +314,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml index 9bc2df93d..d8dc13a75 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml @@ -305,6 +305,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml index 7804790e6..a5b6e8766 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml @@ -300,6 +300,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index b4a123e12..9bfeea643 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -265,7 +265,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - path: "/etc/kubernetes/pki/ca.crt" content: | diff --git a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml index 5f5b06e5e..b5f903ed5 100644 --- a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml @@ -301,6 +301,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml index fe2514bad..7ec997fd6 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml @@ -302,6 +302,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml index fbc87d610..b5351c639 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml @@ -302,6 +302,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml index 9d5cb61b1..445eb27cb 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml @@ -302,6 +302,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml index 6e2e89550..ed16be8d0 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml @@ -302,6 +302,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml index e7af63ef5..ba70e2b71 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -319,6 +319,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml index 38f2289ca..d9708de7a 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -319,6 +319,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml index 366dae7b0..5024529a9 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml @@ -310,6 +310,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index 6a3e2420f..0bb9b4392 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -301,6 +301,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index e72ad632d..f54814a29 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -277,7 +277,7 @@ storage: mode: 0644 contents: inline: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates | indent 10 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 10 }} - path: /opt/load-kernel-modules.sh filesystem: root @@ -551,7 +551,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" permissions: "0644" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - path: /opt/load-kernel-modules.sh permissions: "0755" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.19.15.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.19.15.yaml index 4148f486e..140e35117 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.19.15.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.19.15.yaml @@ -175,6 +175,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.11.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.11.yaml index 355e70c4d..b4461d4ed 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.11.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.11.yaml @@ -175,6 +175,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.5.yaml index 3017ec388..933d1b18d 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.5.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.5.yaml @@ -175,6 +175,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.2.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.2.yaml index f42f4375f..1c0acfdee 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.2.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.2.yaml @@ -175,6 +175,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index 64f5cfe67..63505cd17 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -158,6 +158,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.19.15.json b/pkg/userdata/flatcar/testdata/ignition_v1.19.15.json index fcf478903..fc862002c 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.19.15.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.19.15.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.19.15%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.19.15%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.20.11.json b/pkg/userdata/flatcar/testdata/ignition_v1.20.11.json index 48e966178..b9a3d8142 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.20.11.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.20.11.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.11%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.11%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.21.5.json b/pkg/userdata/flatcar/testdata/ignition_v1.21.5.json index 9c451364f..f47d95b64 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.21.5.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.21.5.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.22.2.json b/pkg/userdata/flatcar/testdata/ignition_v1.22.2.json index d16e759ed..546fb319a 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.22.2.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.22.2.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.2%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.2%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index e07f48412..11117b762 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -23,6 +23,7 @@ import ( "text/template" "github.com/Masterminds/semver/v3" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -162,7 +163,7 @@ func KubeletSystemdUnit(containerRuntime, kubeletVersion, cloudProvider, hostnam } // kubeletConfiguration returns marshaled kubelet.config.k8s.io/v1beta1 KubeletConfiguration -func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGates map[string]bool) (string, error) { +func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGates map[string]bool, kubeletConfigs map[string]string) (string, error) { clusterDNSstr := make([]string, 0, len(clusterDNS)) for _, ip := range clusterDNS { clusterDNSstr = append(clusterDNSstr, ip.String()) @@ -198,10 +199,41 @@ func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGate StaticPodPath: "/etc/kubernetes/manifests", KubeReserved: map[string]string{"cpu": "200m", "memory": "200Mi", "ephemeral-storage": "1Gi"}, SystemReserved: map[string]string{"cpu": "200m", "memory": "200Mi", "ephemeral-storage": "1Gi"}, + EvictionHard: map[string]string{"memory.available": "100Mi", "nodefs.available": "10%", "nodefs.inodesFree": "5%", "imagefs.available": "15%"}, VolumePluginDir: "/var/lib/kubelet/volumeplugins", TLSCipherSuites: kubeletTLSCipherSuites, } + if kubeReserved, ok := kubeletConfigs[common.KubeReservedKubeletConfig]; ok { + for _, krPair := range strings.Split(kubeReserved, ",") { + krKV := strings.SplitN(krPair, "=", 2) + if len(krKV) != 2 { + continue + } + cfg.KubeReserved[krKV[0]] = krKV[1] + } + } + + if systemReserved, ok := kubeletConfigs[common.SystemReservedKubeletConfig]; ok { + for _, srPair := range strings.Split(systemReserved, ",") { + srKV := strings.SplitN(srPair, "=", 2) + if len(srKV) != 2 { + continue + } + cfg.SystemReserved[srKV[0]] = srKV[1] + } + } + + if evictionHard, ok := kubeletConfigs[common.EvictionHardKubeletConfig]; ok { + for _, ehPair := range strings.Split(evictionHard, ",") { + ehKV := strings.SplitN(ehPair, "=", 2) + if len(ehKV) != 2 { + continue + } + cfg.EvictionHard[ehKV[0]] = ehKV[1] + } + } + buf, err := kyaml.Marshal(cfg) return string(buf), err } diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 46828984d..29fa0a540 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -277,7 +277,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - path: "/etc/kubernetes/pki/ca.crt" content: | diff --git a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.19-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.19-aws.yaml index 791db14cf..d978297ec 100644 --- a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.19-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.19-aws.yaml @@ -301,6 +301,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml index 86e5e7413..0c8b7b939 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml @@ -302,6 +302,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml index 163e3b9a3..05229a20c 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml @@ -302,6 +302,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml index c3fb9ed80..32c376dfb 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml @@ -302,6 +302,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws-external.yaml index f1bf1bed4..f30b07a67 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws-external.yaml @@ -301,6 +301,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index 9650e5ade..6cff9d189 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -301,6 +301,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-mirrors.yaml index 2d48e9e45..6282bf630 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-mirrors.yaml @@ -319,6 +319,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-proxy.yaml index b931b31a3..022c5fa40 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-proxy.yaml @@ -319,6 +319,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere.yaml index 6831995a1..1ca0c3548 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere.yaml @@ -310,6 +310,11 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/sles/provider.go b/pkg/userdata/sles/provider.go index ba5967e55..a3a5583e7 100644 --- a/pkg/userdata/sles/provider.go +++ b/pkg/userdata/sles/provider.go @@ -246,7 +246,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - path: "/etc/profile.d/opt-bin-path.sh" permissions: "0644" diff --git a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml index 140e154f4..4253fa908 100644 --- a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml @@ -314,6 +314,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml index 574dd0f87..d6eff7725 100644 --- a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml @@ -312,6 +312,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml index 2e62e8969..eed8e7c76 100644 --- a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml @@ -314,6 +314,11 @@ write_files: - 10.10.10.12 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml index 5140ac183..a1a41665e 100644 --- a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml @@ -314,6 +314,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml index ed9ac402b..67221d6bd 100644 --- a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml @@ -316,6 +316,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/sles/testdata/openstack.yaml b/pkg/userdata/sles/testdata/openstack.yaml index 057994ff1..25629422e 100644 --- a/pkg/userdata/sles/testdata/openstack.yaml +++ b/pkg/userdata/sles/testdata/openstack.yaml @@ -316,6 +316,11 @@ write_files: - 10.10.10.12 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/sles/testdata/version-1.19.15.yaml b/pkg/userdata/sles/testdata/version-1.19.15.yaml index a0089c221..08670aea6 100644 --- a/pkg/userdata/sles/testdata/version-1.19.15.yaml +++ b/pkg/userdata/sles/testdata/version-1.19.15.yaml @@ -312,6 +312,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/sles/testdata/version-1.20.11.yaml b/pkg/userdata/sles/testdata/version-1.20.11.yaml index f03bbff41..c5be4ae4a 100644 --- a/pkg/userdata/sles/testdata/version-1.20.11.yaml +++ b/pkg/userdata/sles/testdata/version-1.20.11.yaml @@ -312,6 +312,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/sles/testdata/version-1.21.5.yaml b/pkg/userdata/sles/testdata/version-1.21.5.yaml index 574dd0f87..d6eff7725 100644 --- a/pkg/userdata/sles/testdata/version-1.21.5.yaml +++ b/pkg/userdata/sles/testdata/version-1.21.5.yaml @@ -312,6 +312,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/sles/testdata/version-1.22.2.yaml b/pkg/userdata/sles/testdata/version-1.22.2.yaml index b96391053..99bb98744 100644 --- a/pkg/userdata/sles/testdata/version-1.22.2.yaml +++ b/pkg/userdata/sles/testdata/version-1.22.2.yaml @@ -312,6 +312,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml index c6566651f..4a76ef827 100644 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml @@ -327,6 +327,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml index 43f2453f6..fd2c37291 100644 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/sles/testdata/vsphere-proxy.yaml @@ -327,6 +327,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml index 506b3f04a..61eed97b4 100644 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ b/pkg/userdata/sles/testdata/vsphere.yaml @@ -317,6 +317,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index 137dbac74..803275db8 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -301,7 +301,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index 01e689249..6e65d14e6 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -387,6 +387,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 38d37914c..19e1119bf 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -370,6 +370,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index 36025c41a..70390d074 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -368,6 +368,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 6ea966ac1..4fa378eec 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -370,6 +370,11 @@ write_files: - 10.10.10.12 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 262cbb747..8d42cb184 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -370,6 +370,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index 4c0dc3370..58fb855ec 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -372,6 +372,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 57d49c306..c6d961936 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -372,6 +372,11 @@ write_files: - 10.10.10.12 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/ubuntu/testdata/version-1.19.15.yaml b/pkg/userdata/ubuntu/testdata/version-1.19.15.yaml index 23e84745c..86eca89b3 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.19.15.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.19.15.yaml @@ -368,6 +368,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/ubuntu/testdata/version-1.20.11.yaml b/pkg/userdata/ubuntu/testdata/version-1.20.11.yaml index e38c10e8e..11b7fb56f 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.20.11.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.20.11.yaml @@ -368,6 +368,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/ubuntu/testdata/version-1.21.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.21.5.yaml index 36025c41a..70390d074 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.21.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.21.5.yaml @@ -368,6 +368,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.2.yaml b/pkg/userdata/ubuntu/testdata/version-1.22.2.yaml index f8a5efd5d..80ae0e88c 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.22.2.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.22.2.yaml @@ -385,6 +385,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index f14361f5f..6365262aa 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -383,6 +383,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 5da5d1a6e..16c9b0989 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -383,6 +383,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 96cf423ec..1470cc9fe 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -373,6 +373,11 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% evictionPressureTransitionPeriod: 0s featureGates: RotateKubeletServerCertificate: true From d7b9c50b4482dbeea562948e46f2a9fe75e62c45 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 21 Dec 2021 21:20:39 +0500 Subject: [PATCH 041/489] Add validation for machinedeployments when using OSM (#1136) * Add validation for machinedeployments when using OSM * Bump OSM to v0.3.0 --- cmd/webhook/main.go | 20 ++++++++++-- go.mod | 4 +-- go.sum | 49 ++++++++++++++++++++++------- pkg/admission/admission.go | 6 ++++ pkg/admission/machinedeployments.go | 8 +++++ 5 files changed, 72 insertions(+), 15 deletions(-) diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 3b69d541f..5b893f62f 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -23,7 +23,9 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/node" userdatamanager "github.com/kubermatic/machine-controller/pkg/userdata/manager" + osmv1alpha1 "k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/clientcmd" "k8s.io/klog" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -36,6 +38,8 @@ var ( admissionTLSCertPath string admissionTLSKeyPath string caBundleFile string + useOSM bool + namespace string ) func main() { @@ -52,6 +56,11 @@ func main() { flag.StringVar(&admissionTLSCertPath, "tls-cert-path", "/tmp/cert/cert.pem", "The path of the TLS cert for the MutatingWebhook") flag.StringVar(&admissionTLSKeyPath, "tls-key-path", "/tmp/cert/key.pem", "The path of the TLS key for the MutatingWebhook") flag.StringVar(&caBundleFile, "ca-bundle", "", "path to a file containing all PEM-encoded CA certificates (will be used instead of the host's certificates if set)") + flag.StringVar(&namespace, "namespace", "kubermatic", "The namespace where the webhooks will run") + + // OSM specific flags + flag.BoolVar(&useOSM, "use-osm", false, "osm controller is enabled for node bootstrap") + flag.Parse() kubeconfig = flag.Lookup("kubeconfig").Value.(flag.Getter).Get().(string) masterURL = flag.Lookup("master").Value.(flag.Getter).Get().(string) @@ -67,7 +76,14 @@ func main() { klog.Fatalf("error building kubeconfig: %v", err) } - client, err := ctrlruntimeclient.New(cfg, ctrlruntimeclient.Options{}) + scheme := runtime.NewScheme() + if err := osmv1alpha1.AddToScheme(scheme); err != nil { + klog.Fatalf("failed to add osmv1alpha1 api to scheme: %v", err) + } + + client, err := ctrlruntimeclient.New(cfg, ctrlruntimeclient.Options{ + Scheme: scheme, + }) if err != nil { klog.Fatalf("failed to build client: %v", err) } @@ -77,7 +93,7 @@ func main() { klog.Fatalf("error initialising userdata plugins: %v", err) } - srv, err := admission.New(admissionListenAddress, client, um, nodeFlags) + srv, err := admission.New(admissionListenAddress, client, um, nodeFlags, useOSM, namespace) if err != nil { klog.Fatalf("failed to create admission hook: %v", err) } diff --git a/go.mod b/go.mod index 5b8e3ae72..2aec772cf 100644 --- a/go.mod +++ b/go.mod @@ -36,13 +36,13 @@ require ( github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda github.com/vmware/govmomi v0.23.1 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 - golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58 + golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c gomodules.xyz/jsonpatch/v2 v2.2.0 google.golang.org/api v0.36.0 google.golang.org/grpc v1.38.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b - k8c.io/operating-system-manager v0.1.0 + k8c.io/operating-system-manager v0.3.0 k8s.io/api v0.22.2 k8s.io/apiextensions-apiserver v0.22.2 k8s.io/apimachinery v0.22.2 diff --git a/go.sum b/go.sum index 14a291796..42cf16f38 100644 --- a/go.sum +++ b/go.sum @@ -161,8 +161,9 @@ github.com/a8m/expect v1.0.0/go.mod h1:4IwSCMumY49ScypDnjNbYEjgVeqy1/U2cEs3Lat96 github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= -github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083 h1:uwcvnXW76Y0rHM+qs7y8iHknWUWXYFNlD6FEVhc47TU= github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= +github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 h1:4SPQljF/GJ8Q+QlCWMWxRBepub4DresnOm4eI2ebFGc= +github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd h1:NlKlOv3aVJ5ODMC0JWPvddw05KENkL3cZttIuu8kJRo= github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd/go.mod h1:idhzw68Q7v4j+rQ2AGyq3OlZW2Jij9mdmGA4/Sk6J0E= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= @@ -170,8 +171,9 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4= +github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.61.751 h1:PX0jCn9kBBgaybsFltpmQ8F7O74hQXY/3yNyCjInDag= github.com/aliyun/alibaba-cloud-sdk-go v1.61.751/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= @@ -395,6 +397,7 @@ github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwo github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -531,8 +534,9 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M= -github.com/go-openapi/swag v0.19.14 h1:gm3vOOXfiuw5i9p5N9xJvfjvuofpyvLA9Wr6QfK5Fng= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/validate v0.17.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= @@ -677,8 +681,9 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-containerregistry v0.0.0-20200115214256-379933c9c22b/go.mod h1:Wtl/v6YdQxv397EREtzwgd9+Ud7Q5D8XMbi3Zazgkrs= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-licenses v0.0.0-20191112164736-212ea350c932/go.mod h1:16wa6pRqNDUIhOtwF0GcROVqMeXHZJ7H6eGDFUh5Pfk= @@ -881,6 +886,7 @@ github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0Lh github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kinvolk/container-linux-config-transpiler v0.9.1/go.mod h1:pjTzCvFfbXjWuMVNFjA9FdbsdmruK6+vki0hK0lDmnU= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -911,6 +917,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubermatic/machine-controller v1.23.1/go.mod h1:mXWbT7SjqpgFhzCFT3yMEHKdIlT+KkGy4KQCkNRM9Fc= github.com/kubermatic/machine-controller v1.26.0/go.mod h1:dcJ+GdDSCxCwM0poxwOK8hVO7epiOORDmNMmb2veyw4= +github.com/kubermatic/machine-controller v1.36.1/go.mod h1:6BFZEvEMZi8OT8aHOsS7DXYsF6ZSpmsNxsci7OLTTn8= github.com/kubernetes-csi/csi-lib-utils v0.7.0/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0= github.com/kubernetes-csi/csi-test v2.0.0+incompatible/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= github.com/kubernetes-csi/external-snapshotter/v2 v2.1.1/go.mod h1:dV5oB3U62KBdlf9ADWkMmjGd3USauqQtwIm2OZb5mqI= @@ -935,9 +942,10 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/markbates/inflect v1.0.4/go.mod h1:1fR9+pO2KHEO9ZRtto13gDwwZaAKstQzferVeWqbgNs= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= @@ -1056,6 +1064,7 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -1071,6 +1080,7 @@ github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= @@ -1171,6 +1181,7 @@ github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQ github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0/7TSWwj+ITvv0TnM= +github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -1194,6 +1205,8 @@ github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= +github.com/prometheus/common v0.25.0/go.mod h1:H6QK/N6XVT42whUeIdI3dp36w49c+/iMDk7UAI2qm7Q= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1236,6 +1249,7 @@ github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= @@ -1268,6 +1282,7 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= @@ -1358,8 +1373,9 @@ github.com/vbauerster/mpb/v5 v5.2.2/go.mod h1:W5Fvgw4dm3/0NhqzV8j6EacfuTe5SvnzBR github.com/vdemeester/k8s-pkg-credentialprovider v0.0.0-20200107171650-7c61ffa44238/go.mod h1:JwQJCMWpUDqjZrB5jpw0f5VbN7U95zxFy1ZDpoEarGo= github.com/vdemeester/k8s-pkg-credentialprovider v1.13.12-1/go.mod h1:Fko0rTxEtDW2kju5Ky7yFJNS3IcNvW8IPsp4/e9oev0= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= -github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb h1:lyL3z7vYwTWXf4/bI+A01+cCSnfhKIBhy+SQ46Z/ml8= github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= +github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50 h1:uxE3GYdXIOfhMv3unJKETJEhw78gvzuQqRX/rVirc2A= +github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= @@ -1460,8 +1476,9 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go4.org v0.0.0-20200104003542-c7e774b10ea0 h1:M6XsnQeLwG+rHQ+/rrGh3puBI3WZEy9TBWmf2H+enQA= go4.org v0.0.0-20200104003542-c7e774b10ea0/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= +go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1600,8 +1617,9 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1610,8 +1628,9 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58 h1:Mj83v+wSRNEar42a/MQgxk9X42TdEmrOl9i+y8WbxLo= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c h1:pkQiBZBvdos9qq4wBAHqlzuZHEXo07pqV06ef90u1WI= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1715,18 +1734,21 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo= golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210503060354-a79de5458b56 h1:b8jxX3zqjpqb2LklXPzKSGJhzyxCOZSz8ncv8Nv+y7w= +golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2064,9 +2086,12 @@ honnef.co/go/tools v0.0.1-2019.2.2/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.4/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= +k8c.io/kubermatic/v2 v2.16.2 h1:tjPfI+VV51pggXCvcDL/qG1r7KHDBQPSPYngPxpRtp8= k8c.io/kubermatic/v2 v2.16.2/go.mod h1:NdW+2mq4ynRtfZs9yPnvcnFWQpzmM7ngntW6GeuQicU= -k8c.io/operating-system-manager v0.1.0 h1:2/vmpWHOLm1j3YZ0qrlaW+ucydXXS83FF7pISDoWlKs= k8c.io/operating-system-manager v0.1.0/go.mod h1:ULyZQO1irKjsQTNjIdrHld7SZ+joHjmPnOEs5Db8G8M= +k8c.io/operating-system-manager v0.3.0 h1:xu1BA1Uj22MAeXSx9mNumfm63/6P8xjRcojel+QD5wI= +k8c.io/operating-system-manager v0.3.0/go.mod h1:ME5GOCNUrHG+57igEKP1JCJKVHynaLfodT8bRiYH3MY= k8s.io/api v0.0.0-20181018013834-843ad2d9b9ae/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20181115043458-b799cb063522/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20181203235848-2dd39edadc55/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= @@ -2272,6 +2297,7 @@ k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= knative.dev/caching v0.0.0-20190719140829-2032732871ff/go.mod h1:dHXFU6CGlLlbzaWc32g80cR92iuBSpsslDNBWI8C7eg= @@ -2324,6 +2350,7 @@ sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnM sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go index e31cbf78f..d944c1ece 100644 --- a/pkg/admission/admission.go +++ b/pkg/admission/admission.go @@ -45,6 +45,8 @@ type admissionData struct { client ctrlruntimeclient.Client userDataManager *userdatamanager.Manager nodeSettings machinecontroller.NodeSettings + useOSM bool + namespace string } var jsonPatch = admissionv1.PatchTypeJSONPatch @@ -54,11 +56,15 @@ func New( client ctrlruntimeclient.Client, um *userdatamanager.Manager, nodeFlags *node.Flags, + useOSM bool, + namespace string, ) (*http.Server, error) { mux := http.NewServeMux() ad := &admissionData{ client: client, userDataManager: um, + useOSM: useOSM, + namespace: namespace, } if err := nodeFlags.UpdateNodeSettings(&ad.nodeSettings); err != nil { diff --git a/pkg/admission/machinedeployments.go b/pkg/admission/machinedeployments.go index 7ea4e2dfb..81b907a55 100644 --- a/pkg/admission/machinedeployments.go +++ b/pkg/admission/machinedeployments.go @@ -21,6 +21,7 @@ import ( "fmt" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + osmadmission "k8c.io/operating-system-manager/pkg/admission" admissionv1 "k8s.io/api/admission/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -38,6 +39,13 @@ func (ad *admissionData) mutateMachineDeployments(ar admissionv1.AdmissionReques return nil, fmt.Errorf("validation failed: %v", errs) } + // If OSM is enabled then validate machine deployment against selected OSP + if ad.useOSM { + if errs := osmadmission.ValidateMachineDeployment(machineDeployment, ad.client, ad.namespace); len(errs) > 0 { + return nil, fmt.Errorf("validation failed: %v", errs) + } + } + // Do not validate the spec if it hasn't changed machineSpecNeedsValidation := true if ar.Operation == admissionv1.Update { From f6e499db31f20806aa59ac95eca73241146de7c6 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Tue, 21 Dec 2021 19:50:25 +0200 Subject: [PATCH 042/489] Fix evictionHard parsing (#1139) Signed-off-by: Artiom Diomin --- pkg/userdata/helper/kubelet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index 11117b762..f2e41a3dd 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -226,7 +226,7 @@ func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGate if evictionHard, ok := kubeletConfigs[common.EvictionHardKubeletConfig]; ok { for _, ehPair := range strings.Split(evictionHard, ",") { - ehKV := strings.SplitN(ehPair, "=", 2) + ehKV := strings.SplitN(ehPair, "<", 2) if len(ehKV) != 2 { continue } From 51af5078829c97572896651f9af3c8f944fe246a Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 27 Dec 2021 21:46:24 +0500 Subject: [PATCH 043/489] Support for Equinix Metal cloud provider (#1130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Equinix Metal cloud provider * refactor: fix invalid reference error * Add testdata for equinixmetal * Add mutation logic for facilitating machines and machiendeployments that use packet * Minor fixes Signed-off-by: Marko Mudrinić Co-authored-by: Marko Mudrinić --- .prow.yaml | 8 +- docs/cloud-provider.md | 44 +++++----- docs/operating-system.md | 2 +- ...ml => equinixmetal-machinedeployment.yaml} | 18 ++-- pkg/admission/machinedeployments.go | 5 ++ .../machinedeployments_validation.go | 27 ++++++ pkg/admission/machines.go | 9 ++ pkg/admission/util.go | 51 +++++++++++ pkg/cloudprovider/provider.go | 11 ++- .../{packet => equinixmetal}/provider.go | 86 +++++++++++-------- .../{packet => equinixmetal}/types/types.go | 2 +- pkg/providerconfig/types/types.go | 6 +- test/e2e/provisioning/all_e2e_test.go | 22 ++--- ...ml => machinedeployment-equinixmetal.yaml} | 6 +- 14 files changed, 205 insertions(+), 92 deletions(-) rename examples/{packet-machinedeployment.yaml => equinixmetal-machinedeployment.yaml} (74%) create mode 100644 pkg/admission/util.go rename pkg/cloudprovider/provider/{packet => equinixmetal}/provider.go (81%) rename pkg/cloudprovider/provider/{packet => equinixmetal}/types/types.go (93%) rename test/e2e/provisioning/testdata/{machinedeployment-packet.yaml => machinedeployment-equinixmetal.yaml} (86%) diff --git a/.prow.yaml b/.prow.yaml index 4b58cf1c3..98975b3a6 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -497,16 +497,16 @@ presubmits: memory: 1Gi cpu: 500m - - name: pull-machine-controller-e2e-packet + - name: pull-machine-controller-e2e-equinix-metal optional: true - run_if_changed: pkg\/cloudprovider\/provider\/packet\/.* + run_if_changed: pkg\/cloudprovider\/provider\/equinixmetal\/.* decorate: true error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-hetzner: "true" preset-e2e-ssh: "true" - preset-packet: "true" + preset-equinix-metal: "true" preset-goproxy: "true" spec: containers: @@ -514,7 +514,7 @@ presubmits: command: - "./hack/ci-e2e-test.sh" args: - - "TestPacketProvisioningE2E" + - "TestEquinixMetalProvisioningE2E" resources: requests: memory: 1Gi diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index b26359073..bdf56c9eb 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -106,7 +106,7 @@ projectName: "" # project id projectID: "" # tenant name (deprecated, should use projectName) -tenantName: "" +tenantName: "" # tenant Id (deprecated, should use projectID) tenantID: "" # image to use (currently only ubuntu is supported) @@ -247,7 +247,7 @@ subscriptionID: "<< AZURE_SUBSCRIPTION_ID >>" location: "westeurope" # Azure resource group resourceGroup: "<< YOUR_RESOURCE_GROUP >>" -# Azure resource group of the vnet +# Azure resource group of the vnet vnetResourceGroup: "<< YOUR_VNET_RESOURCE_GROUP >>" # Azure availability set availabilitySet: "<< YOUR AVAILABILITY SET >>" @@ -271,6 +271,26 @@ tags: "kubernetesCluster": "my-cluster" ``` +## Equinix Metal + +### machine.spec.providerConfig.cloudProviderSpec +```yaml +# If empty, can be set via METAL_AUTH_TOKEN env var +token: "<< METAL_AUTH_TOKEN >>" +# instance type +instanceType: "t1.small.x86" +# Equinix Metal project ID +projectID: "<< PROJECT_ID >>" +# Equinix Metal facilities +facilities: + - "ewr1" +# Equinix Metal billingCycle +billingCycle: "" +# node tags +tags: + "kubernetesCluster": "my-cluster" +``` + ## KubeVirt ### machine.spec.providerConfig.cloudProviderSpec @@ -290,26 +310,6 @@ cpus: "1" memory: "2048M" ``` -## Packet - -### machine.spec.providerConfig.cloudProviderSpec -```yaml -# If empty, can be set via PACKET_API_KEY env var -apiKey: "<< PACKET_API_KEY >>" -# instance type -instanceType: "t1.small.x86" -# packet project ID -projectID: "<< PROJECT_ID >>" -# packet facilities -facilities: - - "ewr1" -# packet billingCycle -billingCycle: "" -# node tags -tags: - "kubernetesCluster": "my-cluster" -``` - ## vSphere Refer to the [VSphere](./vsphere.md#provider-configuration) specific documentation. diff --git a/docs/operating-system.md b/docs/operating-system.md index 3dfe03dfe..fea7bb582 100644 --- a/docs/operating-system.md +++ b/docs/operating-system.md @@ -11,7 +11,7 @@ | Digitalocean | ✓ | ✓ | ✓ | x | x | x | x | | Google Cloud Platform | ✓ | ✓ | x | x | ✓ | x | x | | Hetzner | ✓ | x | ✓ | x | x | x | x | -| Packet | ✓ | ✓ | ✓ | x | x | x | x | +| Equinix Metal | ✓ | ✓ | ✓ | x | x | x | x | | Openstack | ✓ | ✓ | ✓ | x | ✓ | x | x | ## Configuring a operating system diff --git a/examples/packet-machinedeployment.yaml b/examples/equinixmetal-machinedeployment.yaml similarity index 74% rename from examples/packet-machinedeployment.yaml rename to examples/equinixmetal-machinedeployment.yaml index 1831f37ef..3300a951b 100644 --- a/examples/packet-machinedeployment.yaml +++ b/examples/equinixmetal-machinedeployment.yaml @@ -3,16 +3,16 @@ kind: Secret metadata: # If you change the namespace/name, you must also # adjust the rbac rules - name: machine-controller-hetzner + name: machine-controller-equinixmetal namespace: kube-system type: Opaque stringData: - apiKey: << PACKET_API_KEY >> + token: << METAL_AUTH_TOKEN >> --- apiVersion: "cluster.k8s.io/v1alpha1" kind: MachineDeployment metadata: - name: packet-machinedeployment + name: equinixmetal-machinedeployment namespace: kube-system spec: paused: false @@ -35,18 +35,18 @@ spec: value: sshPublicKeys: - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "packet" + cloudProvider: "equinixmetal" cloudProviderSpec: - # If empty, can be set via PACKET_API_KEY env var - apiKey: + # If empty, can be set via METAL_TOKEN env var + token: secretKeyRef: namespace: kube-system - name: machine-controller-packet - key: apiKey + name: machine-controller-equinixmetal + key: token instanceType: "t1.small.x86" projectID: "<< PROJECT_ID >>" facilities: - - "ewr1" + - "ewr1" operatingSystem: "ubuntu" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/pkg/admission/machinedeployments.go b/pkg/admission/machinedeployments.go index 81b907a55..76a6dc81f 100644 --- a/pkg/admission/machinedeployments.go +++ b/pkg/admission/machinedeployments.go @@ -35,6 +35,11 @@ func (ad *admissionData) mutateMachineDeployments(ar admissionv1.AdmissionReques machineDeploymentOriginal := machineDeployment.DeepCopy() machineDeploymentDefaultingFunction(&machineDeployment) + + if err := mutationsForMachineDeployment(&machineDeployment); err != nil { + return nil, fmt.Errorf("mutation failed: %v", err) + } + if errs := validateMachineDeployment(machineDeployment); len(errs) > 0 { return nil, fmt.Errorf("validation failed: %v", errs) } diff --git a/pkg/admission/machinedeployments_validation.go b/pkg/admission/machinedeployments_validation.go index 4772496b0..73e6cfdb7 100644 --- a/pkg/admission/machinedeployments_validation.go +++ b/pkg/admission/machinedeployments_validation.go @@ -17,8 +17,12 @@ limitations under the License. package admission import ( + "encoding/json" + "fmt" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" @@ -109,3 +113,26 @@ func getIntOrPercent(s *intstr.IntOrString, roundUp bool) (int, error) { func machineDeploymentDefaultingFunction(md *v1alpha1.MachineDeployment) { v1alpha1.PopulateDefaultsMachineDeployment(md) } + +func mutationsForMachineDeployment(md *v1alpha1.MachineDeployment) error { + providerConfig, err := providerconfigtypes.GetConfig(md.Spec.Template.Spec.ProviderSpec) + if err != nil { + return fmt.Errorf("failed to read MachineDeployment.Spec.Template.Spec.ProviderSpec: %v", err) + } + + // Packet has been renamed to Equinix Metal + if providerConfig.CloudProvider == cloudProviderPacket { + err = migrateToEquinixMetal(providerConfig) + if err != nil { + return fmt.Errorf("failed to migrate packet to equinix metal: %v", err) + } + } + + // Update value in original object + md.Spec.Template.Spec.ProviderSpec.Value.Raw, err = json.Marshal(providerConfig) + if err != nil { + return fmt.Errorf("failed to json marshal machine.spec.providerSpec: %v", err) + } + + return nil +} diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index a955938b2..cf9d4b7b7 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -103,6 +103,15 @@ func (ad *admissionData) defaultAndValidateMachineSpec(spec *clusterv1alpha1.Mac if err != nil { return fmt.Errorf("failed to read machine.spec.providerSpec: %v", err) } + + // Packet has been renamed to Equinix Metal + if providerConfig.CloudProvider == cloudProviderPacket { + err = migrateToEquinixMetal(providerConfig) + if err != nil { + return fmt.Errorf("failed to migrate packet to equinix metal: %v", err) + } + } + skg := providerconfig.NewConfigVarResolver(ad.ctx, ad.client) prov, err := cloudprovider.ForProvider(providerConfig.CloudProvider, skg) if err != nil { diff --git a/pkg/admission/util.go b/pkg/admission/util.go new file mode 100644 index 000000000..ed9f6d77b --- /dev/null +++ b/pkg/admission/util.go @@ -0,0 +1,51 @@ +/* +Copyright 2021 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package admission + +import ( + "encoding/json" + "fmt" + + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" +) + +const cloudProviderPacket = "packet" + +func migrateToEquinixMetal(providerConfig *providerconfigtypes.Config) (err error) { + providerConfig.CloudProvider = providerconfigtypes.CloudProviderEquinixMetal + + // Field .spec.providerSpec.cloudProviderSpec.apiKey has been replaced with .spec.providerSpec.cloudProviderSpec.token + // We first need to perform in-place replacement for this field + rawConfig := map[string]interface{}{} + if err := json.Unmarshal(providerConfig.CloudProviderSpec.Raw, &rawConfig); err != nil { + return fmt.Errorf("failed to unmarshal providerConfig.CloudProviderSpec.Raw: %v", err) + } + // NB: We have to set the token only if apiKey existed, otherwise, migrated + // machines will not create at all (authentication errors). + apiKey, ok := rawConfig["apiKey"] + if ok { + rawConfig["token"] = apiKey + delete(rawConfig, "apiKey") + } + + // Update original object + providerConfig.CloudProviderSpec.Raw, err = json.Marshal(rawConfig) + if err != nil { + return fmt.Errorf("failed to json marshal providerConfig.CloudProviderSpec.Raw: %v", err) + } + return nil +} diff --git a/pkg/cloudprovider/provider.go b/pkg/cloudprovider/provider.go index 9b2c93b6e..78f9ba116 100644 --- a/pkg/cloudprovider/provider.go +++ b/pkg/cloudprovider/provider.go @@ -26,13 +26,13 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/azure" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/digitalocean" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/equinixmetal" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/fake" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/gce" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/hetzner" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/kubevirt" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/linode" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/openstack" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/packet" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/scaleway" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vsphere" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" @@ -71,8 +71,15 @@ var ( providerconfigtypes.CloudProviderAzure: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return azure.New(cvr) }, + providerconfigtypes.CloudProviderEquinixMetal: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return equinixmetal.New(cvr) + }, + // NB: This is explicitly left to allow old Packet machines to be deleted. + // We can handle those machines in the same way as Equinix Metal machines + // because there are no API changes. + // TODO: Remove this after deprecation period. providerconfigtypes.CloudProviderPacket: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return packet.New(cvr) + return equinixmetal.New(cvr) }, providerconfigtypes.CloudProviderFake: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return fake.New(cvr) diff --git a/pkg/cloudprovider/provider/packet/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go similarity index 81% rename from pkg/cloudprovider/provider/packet/provider.go rename to pkg/cloudprovider/provider/equinixmetal/provider.go index 576aed263..d045435be 100644 --- a/pkg/cloudprovider/provider/packet/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package packet +package equinixmetal import ( "encoding/json" @@ -29,7 +29,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - packettypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/packet/types" + equinixmetaltypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/equinixmetal/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -45,13 +45,13 @@ const ( defaultBillingCycle = "hourly" ) -// New returns a Packet provider +// New returns a Equinix Metal provider func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } type Config struct { - APIKey string + Token string ProjectID string BillingCycle string InstanceType string @@ -67,7 +67,7 @@ func (c *Config) populateDefaults() { } } -func populateDefaults(c *packettypes.RawConfig) { +func populateDefaults(c *equinixmetaltypes.RawConfig) { if c.BillingCycle.Value == "" { c.BillingCycle.Value = defaultBillingCycle } @@ -77,7 +77,7 @@ type provider struct { configVarResolver *providerconfig.ConfigVarResolver } -func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *packettypes.RawConfig, *providerconfigtypes.Config, error) { +func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *equinixmetaltypes.RawConfig, *providerconfigtypes.Config, error) { if s.Value == nil { return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } @@ -87,7 +87,7 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *packettypes.Raw return nil, nil, nil, err } - rawConfig := packettypes.RawConfig{} + rawConfig := equinixmetaltypes.RawConfig{} if err = json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawConfig); err != nil { return nil, nil, nil, err } @@ -97,13 +97,25 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *packettypes.Raw } c := Config{} - c.APIKey, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.APIKey, "PACKET_API_KEY") - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"apiKey\" field, error = %v", err) + c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, "METAL_AUTH_TOKEN") + if err != nil || len(c.Token) == 0 { + // This retry is temporary and is only required to facilitate migration from Packet to Equinix Metal + // We look for env variable PACKET_API_KEY associated with Packet to ensure that nothing breaks during automated migration for the Machines + // TODO(@ahmedwaleedmalik) Remove this after a release period + c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, "PACKET_API_KEY") + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get the value of \"apiKey\" field, error = %v", err) + } } - c.ProjectID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProjectID, "PACKET_PROJECT_ID") - if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"projectID\" field, error = %v", err) + c.ProjectID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProjectID, "METAL_PROJECT_ID") + if err != nil || len(c.ProjectID) == 0 { + // This retry is temporary and is only required to facilitate migration from Packet to Equinix Metal + // We look for env variable PACKET_PROJECT_ID associated with Packet to ensure that nothing breaks during automated migration for the Machines + // TODO(@ahmedwaleedmalik) Remove this after a release period + c.ProjectID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProjectID, "PACKET_PROJECT_ID") + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get the value of \"apiKey\" field, error = %v", err) + } } c.InstanceType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.InstanceType) if err != nil { @@ -134,7 +146,7 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *packettypes.Raw return &c, &rawConfig, &pconfig, err } -func (p *provider) getPacketDevice(machine *v1alpha1.Machine) (*packngo.Device, *packngo.Client, error) { +func (p *provider) getMetalDevice(machine *v1alpha1.Machine) (*packngo.Device, *packngo.Client, error) { c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, nil, cloudprovidererrors.TerminalError{ @@ -143,7 +155,7 @@ func (p *provider) getPacketDevice(machine *v1alpha1.Machine) (*packngo.Device, } } - client := getClient(c.APIKey) + client := getClient(c.Token) device, err := getDeviceByTag(client, c.ProjectID, generateTag(string(machine.UID))) if err != nil { return nil, nil, err @@ -157,7 +169,7 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { return fmt.Errorf("failed to parse config: %v", err) } - if c.APIKey == "" { + if c.Token == "" { return errors.New("apiKey is missing") } if c.InstanceType == "" { @@ -172,7 +184,7 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { return fmt.Errorf("invalid/not supported operating system specified %q: %v", pc.OperatingSystem, err) } - client := getClient(c.APIKey) + client := getClient(c.Token) if len(c.Facilities) == 0 || c.Facilities[0] == "" { return fmt.Errorf("must have at least one non-blank facility") @@ -211,7 +223,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provi } } - client := getClient(c.APIKey) + client := getClient(c.Token) imageName, err := getNameForOS(pc.OperatingSystem) if err != nil { @@ -236,10 +248,10 @@ func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provi device, res, err := client.Devices.Create(serverCreateOpts) if err != nil { - return nil, packetErrorToTerminalError(err, res, "failed to create server") + return nil, metalErrorToTerminalError(err, res, "failed to create server") } - return &packetDevice{device: device}, nil + return &metalDevice{device: device}, nil } func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { @@ -259,10 +271,10 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.P } } - client := getClient(c.APIKey) - res, err := client.Devices.Delete(instance.(*packetDevice).device.ID) + client := getClient(c.Token) + res, err := client.Devices.Delete(instance.(*metalDevice).device.ID) if err != nil { - return false, packetErrorToTerminalError(err, res, "failed to delete the server") + return false, metalErrorToTerminalError(err, res, "failed to delete the server") } return false, nil @@ -282,19 +294,19 @@ func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, } func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - device, _, err := p.getPacketDevice(machine) + device, _, err := p.getMetalDevice(machine) if err != nil { return nil, err } if device != nil { - return &packetDevice{device: device}, nil + return &metalDevice{device: device}, nil } return nil, cloudprovidererrors.ErrInstanceNotFound } func (p *provider) MigrateUID(machine *v1alpha1.Machine, newID types.UID) error { - device, client, err := p.getPacketDevice(machine) + device, client, err := p.getMetalDevice(machine) if err != nil { return err } @@ -321,7 +333,7 @@ func (p *provider) MigrateUID(machine *v1alpha1.Machine, newID types.UID) error } _, response, err := client.Devices.Update(device.ID, dur) if err != nil { - return packetErrorToTerminalError(err, response, "failed to update UID label") + return metalErrorToTerminalError(err, response, "failed to update UID label") } klog.Infof("Successfully set UID label for machine %s", machine.Name) @@ -348,19 +360,19 @@ func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { return nil } -type packetDevice struct { +type metalDevice struct { device *packngo.Device } -func (s *packetDevice) Name() string { +func (s *metalDevice) Name() string { return s.device.Hostname } -func (s *packetDevice) ID() string { +func (s *metalDevice) ID() string { return s.device.ID } -func (s *packetDevice) Addresses() map[string]v1.NodeAddressType { +func (s *metalDevice) Addresses() map[string]v1.NodeAddressType { // returns addresses in CIDR format addresses := map[string]v1.NodeAddressType{} for _, ip := range s.device.Network { @@ -374,7 +386,7 @@ func (s *packetDevice) Addresses() map[string]v1.NodeAddressType { return addresses } -func (s *packetDevice) Status() instance.Status { +func (s *metalDevice) Status() instance.Status { switch s.device.State { case "provisioning": return instance.StatusCreating @@ -388,7 +400,7 @@ func (s *packetDevice) Status() instance.Status { /****** CONVENIENCE INTERNAL FUNCTIONS ******/ -func setProviderSpec(rawConfig packettypes.RawConfig, s v1alpha1.ProviderSpec) (*runtime.RawExtension, error) { +func setProviderSpec(rawConfig equinixmetaltypes.RawConfig, s v1alpha1.ProviderSpec) (*runtime.RawExtension, error) { if s.Value == nil { return nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } @@ -412,7 +424,7 @@ func setProviderSpec(rawConfig packettypes.RawConfig, s v1alpha1.ProviderSpec) ( func getDeviceByTag(client *packngo.Client, projectID, tag string) (*packngo.Device, error) { devices, response, err := client.Devices.List(projectID, nil) if err != nil { - return nil, packetErrorToTerminalError(err, response, "failed to list devices") + return nil, metalErrorToTerminalError(err, response, "failed to list devices") } for _, device := range devices { @@ -423,7 +435,7 @@ func getDeviceByTag(client *packngo.Client, projectID, tag string) (*packngo.Dev return nil, nil } -// given a defined Kubermatic constant for an operating system, return the canonical slug for Packet +// given a defined Kubermatic constant for an operating system, return the canonical slug for Equinix Metal func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { switch os { case providerconfigtypes.OperatingSystemUbuntu: @@ -452,11 +464,11 @@ func getTagUID(tag string) (string, error) { return parts[1], nil } -// packetErrorToTerminalError judges if the given error +// metalErrorToTerminalError judges if the given error // can be qualified as a "terminal" error, for more info see v1alpha1.MachineStatus // // if the given error doesn't qualify the error passed as an argument will be returned -func packetErrorToTerminalError(err error, response *packngo.Response, msg string) error { +func metalErrorToTerminalError(err error, response *packngo.Response, msg string) error { prepareAndReturnError := func() error { return fmt.Errorf("%s, due to %s", msg, err) } diff --git a/pkg/cloudprovider/provider/packet/types/types.go b/pkg/cloudprovider/provider/equinixmetal/types/types.go similarity index 93% rename from pkg/cloudprovider/provider/packet/types/types.go rename to pkg/cloudprovider/provider/equinixmetal/types/types.go index 5ccdeeb2d..4ac292e0c 100644 --- a/pkg/cloudprovider/provider/packet/types/types.go +++ b/pkg/cloudprovider/provider/equinixmetal/types/types.go @@ -21,7 +21,7 @@ import ( ) type RawConfig struct { - APIKey providerconfigtypes.ConfigVarString `json:"apiKey,omitempty"` + Token providerconfigtypes.ConfigVarString `json:"token,omitempty"` ProjectID providerconfigtypes.ConfigVarString `json:"projectID,omitempty"` BillingCycle providerconfigtypes.ConfigVarString `json:"billingCycle"` InstanceType providerconfigtypes.ConfigVarString `json:"instanceType"` diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 7ac78c9ca..2bc79bd19 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -48,11 +48,12 @@ const ( CloudProviderAzure CloudProvider = "azure" CloudProviderDigitalocean CloudProvider = "digitalocean" CloudProviderGoogle CloudProvider = "gce" + CloudProviderEquinixMetal CloudProvider = "equinixmetal" + CloudProviderPacket CloudProvider = "packet" CloudProviderHetzner CloudProvider = "hetzner" CloudProviderKubeVirt CloudProvider = "kubevirt" CloudProviderLinode CloudProvider = "linode" CloudProviderOpenstack CloudProvider = "openstack" - CloudProviderPacket CloudProvider = "packet" CloudProviderVsphere CloudProvider = "vsphere" CloudProviderFake CloudProvider = "fake" CloudProviderAlibaba CloudProvider = "alibaba" @@ -80,12 +81,13 @@ var ( CloudProviderAWS, CloudProviderAzure, CloudProviderDigitalocean, + CloudProviderEquinixMetal, + CloudProviderPacket, CloudProviderGoogle, CloudProviderHetzner, CloudProviderKubeVirt, CloudProviderLinode, CloudProviderOpenstack, - CloudProviderPacket, CloudProviderVsphere, CloudProviderFake, CloudProviderAlibaba, diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index a760be5db..2f6a46d5d 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -59,9 +59,9 @@ const ( AzureManifest = "./testdata/machinedeployment-azure.yaml" AzureRedhatSatelliteManifest = "./testdata/machinedeployment-azure.yaml" AzureCustomImageReferenceManifest = "./testdata/machinedeployment-azure-custom-image-reference.yaml" + EquinixMetalManifest = "./testdata/machinedeployment-equinixmetal.yaml" GCEManifest = "./testdata/machinedeployment-gce.yaml" HZManifest = "./testdata/machinedeployment-hetzner.yaml" - PacketManifest = "./testdata/machinedeployment-packet.yaml" LinodeManifest = "./testdata/machinedeployment-linode.yaml" VSPhereManifest = "./testdata/machinedeployment-vsphere.yaml" VSPhereDSCManifest = "./testdata/machinedeployment-vsphere-datastore-cluster.yaml" @@ -712,30 +712,30 @@ func TestHetznerProvisioningE2E(t *testing.T) { runScenarios(t, selector, params, HZManifest, fmt.Sprintf("hz-%s", *testRunIdentifier)) } -// TestPacketProvisioning - a test suite that exercises Packet provider +// TestEquinixMetalProvisioningE2E - a test suite that exercises Equinix Metal provider // by requesting nodes with different combination of container runtime type, container runtime version and the OS flavour. -func TestPacketProvisioningE2E(t *testing.T) { +func TestEquinixMetalProvisioningE2E(t *testing.T) { t.Parallel() // test data - apiKey := os.Getenv("PACKET_API_KEY") - if len(apiKey) == 0 { - t.Fatal("unable to run the test suite, PACKET_API_KEY environment variable cannot be empty") + token := os.Getenv("METAL_AUTH_TOKEN") + if len(token) == 0 { + t.Fatal("unable to run the test suite, METAL_AUTH_TOKEN environment variable cannot be empty") } - projectID := os.Getenv("PACKET_PROJECT_ID") + projectID := os.Getenv("METAL_PROJECT_ID") if len(projectID) == 0 { - t.Fatal("unable to run the test suite, PACKET_PROJECT_ID environment variable cannot be empty") + t.Fatal("unable to run the test suite, METAL_PROJECT_ID environment variable cannot be empty") } selector := Not(OsSelector("sles", "rhel", "amzn2")) // act params := []string{ - fmt.Sprintf("<< PACKET_API_KEY >>=%s", apiKey), - fmt.Sprintf("<< PACKET_PROJECT_ID >>=%s", projectID), + fmt.Sprintf("<< METAL_AUTH_TOKEN >>=%s", token), + fmt.Sprintf("<< METAL_PROJECT_ID >>=%s", projectID), } - runScenarios(t, selector, params, PacketManifest, fmt.Sprintf("packet-%s", *testRunIdentifier)) + runScenarios(t, selector, params, EquinixMetalManifest, fmt.Sprintf("equinixmetal-%s", *testRunIdentifier)) } func TestAlibabaProvisioningE2E(t *testing.T) { diff --git a/test/e2e/provisioning/testdata/machinedeployment-packet.yaml b/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml similarity index 86% rename from test/e2e/provisioning/testdata/machinedeployment-packet.yaml rename to test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml index d487b98f7..28e52c4fa 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-packet.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml @@ -22,10 +22,10 @@ spec: value: sshPublicKeys: - "<< YOUR_PUBLIC_KEY >>" - cloudProvider: "packet" + cloudProvider: "equinixmetal" cloudProviderSpec: - apiKey: << PACKET_API_KEY >> - projectID: << PACKET_PROJECT_ID >> + token: << METAL_AUTH_TOKEN >> + projectID: << METAL_PROJECT_ID >> instanceType: "c1.small.x86" facilities: - "ams1" From c3488bde1d9c840caaf57109719c68c701b13f05 Mon Sep 17 00:00:00 2001 From: Marcin Franczyk Date: Tue, 28 Dec 2021 17:22:24 +0100 Subject: [PATCH 044/489] use kubevirt/api instead of kubevirt/client-go (#1141) that fixes a problem with dependencies. KubeVirt client-go is not compatible with k8s/client-go v0.22.x Signed-off-by: Marcin Franczyk --- go.mod | 9 +- go.sum | 181 +----------------- .../provider/kubevirt/provider.go | 18 +- 3 files changed, 21 insertions(+), 187 deletions(-) diff --git a/go.mod b/go.mod index 2aec772cf..8eed6d998 100644 --- a/go.mod +++ b/go.mod @@ -50,18 +50,13 @@ require ( k8s.io/klog v1.0.0 k8s.io/kubelet v0.22.2 k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a - kubevirt.io/client-go v0.30.0 - kubevirt.io/containerized-data-importer v1.40.0 + kubevirt.io/api v0.48.1 + kubevirt.io/containerized-data-importer-api v1.41.0 sigs.k8s.io/controller-runtime v0.10.2 sigs.k8s.io/yaml v1.2.0 ) replace ( - github.com/openshift/api => github.com/openshift/api v0.0.0-20210428205234-a8389931bee7 - github.com/openshift/client-go => github.com/openshift/client-go v0.0.0-20210112165513-ebc401615f47 - github.com/openshift/library-go => github.com/mhenriks/library-go v0.0.0-20210511195009-51ba86622560 - github.com/operator-framework/operator-lifecycle-manager => github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190128024246-5eb7ae5bdb7a - // the following replacements are only here to make kubevirt.io/containerized-data-importer work github.com/packethost/packngo => github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888 k8s.io/client-go => k8s.io/client-go v0.22.2 diff --git a/go.sum b/go.sum index 42cf16f38..1a382b237 100644 --- a/go.sum +++ b/go.sum @@ -50,7 +50,6 @@ contrib.go.opencensus.io/exporter/stackdriver v0.12.8/go.mod h1:XyyafDnFOsqoxHJg dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-sdk-for-go v28.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -138,9 +137,7 @@ github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmy github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= -github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -155,10 +152,8 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/Venafi/vcert/v4 v4.11.0/go.mod h1:OE+UZ0cj8qqVUuk0u7R4GIk4ZB6JMSf/WySqnBPNwws= -github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/a8m/expect v1.0.0/go.mod h1:4IwSCMumY49ScypDnjNbYEjgVeqy1/U2cEs3Lat96eA= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= @@ -199,7 +194,6 @@ github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:o github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-k8s-tester v0.0.0-20190114231546-b411acf57dfe/go.mod h1:1ADF5tAtU1/mVtfMcHAYSm2fPw71DA7fFk0yed64/0I= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.15.77/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= github.com/aws/aws-sdk-go v1.16.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -231,7 +225,6 @@ github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20180905225744-ee1a9a0726d2/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -255,26 +248,11 @@ github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/container-storage-interface/spec v1.1.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= -github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4= -github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containers/image/v5 v5.5.1/go.mod h1:4PyNYR0nwlGq/ybVJD9hWlhmIsNra4Q8uOQX2s6E2uM= -github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= -github.com/containers/ocicrypt v1.0.2/go.mod h1:nsOhbP19flrX6rE7ieGFvBlr7modwmNjsqWarIUce4M= -github.com/containers/storage v1.20.2/go.mod h1:oOB9Ie8OVPojvoaKWEGSEtHbXUAs+tSyr7RO7ZGteMc= github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -293,7 +271,6 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -332,23 +309,16 @@ github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dteve github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/distribution v0.0.0-20180920194744-16128bbac47f/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.7.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v1.4.2-0.20191219165747-a9416c67da9f/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= -github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libnetwork v0.0.0-20190731215715-7f13a5c99f4b/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8= -github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= @@ -362,16 +332,11 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/elazarl/goproxy v0.0.0-20190911111923-ecfe977594f1/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= -github.com/elazarl/goproxy/ext v0.0.0-20190711103511-473e67f1d7d2/go.mod h1:gNh8nYJoAm43RfaxurUnxr+N1PwuFV3ZMl/efxlIlY8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.8.1+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.6+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.10.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.11.2+incompatible h1:Z4Z0K2AuOw+QtgwkkJnwpT165MBr12qS8rnBwjP/Pzs= github.com/emicklei/go-restful v2.11.2+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful-openapi v1.2.0/go.mod h1:cy7o3Ge8ZWZ5E90mpEY81sJZZFs2pkuYcLvfngYy1l0= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -411,9 +376,6 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsouza/fake-gcs-server v0.0.0-20180612165233-e85be23bdaa8/go.mod h1:1/HufuJ+eaDf4KTnYdS6HJMGvMRU8d4cYTuu/1QaBbI= -github.com/fsouza/go-dockerclient v0.0.0-20171004212419-da3951ba2e9e/go.mod h1:KpcjM623fQYE9MZiTGzKhjfxXAV9wbyX2C1cyRHfhl0= -github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= -github.com/getsentry/raven-go v0.0.0-20190513200303-c977f96e1095/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -431,13 +393,11 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-ini/ini v1.62.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -470,7 +430,6 @@ github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpX github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4= -github.com/go-openapi/jsonpointer v0.0.0-20180322222829-3a0015ad55fa/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= @@ -478,7 +437,6 @@ github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDB github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.0.0-20180322222742-3fb327e6747d/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= @@ -505,7 +463,6 @@ github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2g github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98= github.com/go-openapi/runtime v0.19.20/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= -github.com/go-openapi/spec v0.0.0-20180415031709-bcff419492ee/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.17.2/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= @@ -524,7 +481,6 @@ github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6 github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= -github.com/go-openapi/swag v0.0.0-20180405201759-811b1089cde9/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= @@ -556,7 +512,6 @@ github.com/go-sql-driver/mysql v0.0.0-20160411075031-7ebe0a500653/go.mod h1:zAC/ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-swagger/go-swagger v0.25.0/go.mod h1:9639ioXrPX9E6BbnbaDklGXjNz7upAXoNBwL4Ok11Vk= github.com/go-swagger/scan-repo-boundary v0.0.0-20180623220736-973b3573c013/go.mod h1:b65mBPzqzZWxOZGxSWrqs4GInLIn+u99Q9q7p+GKni0= @@ -599,7 +554,6 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kETFqjFoFlOc= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godror/godror v0.13.3/go.mod h1:2ouUT4kdhUBk7TAkHWD4SN0CdI0pgEQbo8FVHhbSKWg= @@ -617,7 +571,6 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -657,17 +610,10 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/gomodule/redigo v1.7.0/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= -github.com/gonum/blas v0.0.0-20181208220705-f22b278b28ac/go.mod h1:P32wAyui1PQ58Oce/KYkOqQv8cVw1zAapXOl+dRFGbc= -github.com/gonum/floats v0.0.0-20181209220543-c233463c7e82/go.mod h1:PxC8OnwL11+aosOB5+iEPoV3picfs8tUpkVd0pDo+Kg= -github.com/gonum/graph v0.0.0-20170401004347-50b27dea7ebb/go.mod h1:ye018NnX1zrbOLqwBvs2HqyyTouQgnL8C+qzYk1snPY= -github.com/gonum/internal v0.0.0-20181124074243-f884aa714029/go.mod h1:Pu4dmpkhSyOzRwuXkOgAvijx4o+4YMUJJo9OvPYMkks= -github.com/gonum/lapack v0.0.0-20181123203213-e4cdc5a0bff9/go.mod h1:XA3DeT6rxh2EAE789SSiSJNqxPaC0aE9J8NTOI0Jo/A= -github.com/gonum/matrix v0.0.0-20181209220409-c518dec07be9/go.mod h1:0EXg4mc1CNP0HCqCz+K4ts155PXIlUywf0wqN+GfPZw= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -742,10 +688,8 @@ github.com/gorilla/csrf v1.6.2/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAk github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v0.0.0-20181024020800-521ea7b17d02/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v0.0.0-20191024121256-f395758b854c/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= @@ -763,14 +707,12 @@ github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170330212424-2500245aa611/ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/grpc-ecosystem/grpc-gateway v1.6.3/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.15.2/go.mod h1:vO11I9oWA+KsxmfFQPhLnnIb1VDE24M+pdxZFiuZcA8= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-health-probe v0.2.0/go.mod h1:4GVx/bTCtZaSzhjbGueDY5YgBdsmKeVx+LErv/n0L6s= github.com/h2non/gock v1.0.9/go.mod h1:CZMcB0Lg5IWnr9bF79pPMg9WeV6WumxQiUJ1UvdO1iE= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -827,7 +769,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.0.0-20171009183408-7fe0c75c13ab/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -848,7 +789,6 @@ github.com/jetstack/cert-manager v1.1.0/go.mod h1:GULIHTGjSc2LjlgBCLhQ8u5WmQ95hk github.com/jinzhu/gorm v0.0.0-20170316141641-572d0a0ab1eb/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= github.com/jinzhu/inflection v0.0.0-20190603042836-f5c5f50e6090/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -884,7 +824,6 @@ github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-2019 github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kinvolk/container-linux-config-transpiler v0.9.1/go.mod h1:pjTzCvFfbXjWuMVNFjA9FdbsdmruK6+vki0hK0lDmnU= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -893,11 +832,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/knative/build v0.1.2/go.mod h1:/sU74ZQkwlYA5FwYDJhYTy61i/Kn+5eWfln2jDbw3Qo= github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -918,9 +854,6 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubermatic/machine-controller v1.23.1/go.mod h1:mXWbT7SjqpgFhzCFT3yMEHKdIlT+KkGy4KQCkNRM9Fc= github.com/kubermatic/machine-controller v1.26.0/go.mod h1:dcJ+GdDSCxCwM0poxwOK8hVO7epiOORDmNMmb2veyw4= github.com/kubermatic/machine-controller v1.36.1/go.mod h1:6BFZEvEMZi8OT8aHOsS7DXYsF6ZSpmsNxsci7OLTTn8= -github.com/kubernetes-csi/csi-lib-utils v0.7.0/go.mod h1:bze+2G9+cmoHxN6+WyG1qT4MDxgZJMLGwc7V4acPNm0= -github.com/kubernetes-csi/csi-test v2.0.0+incompatible/go.mod h1:YxJ4UiuPWIhMBkxUKY5c267DyA0uDZ/MtAimhx/2TA0= -github.com/kubernetes-csi/external-snapshotter/v2 v2.1.1/go.mod h1:dV5oB3U62KBdlf9ADWkMmjGd3USauqQtwIm2OZb5mqI= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -935,7 +868,6 @@ github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20180323154445-8b799c424f57/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -966,23 +898,17 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.5/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v0.0.0-20160514122348-38ee283dabf1/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/maxbrunsfeld/counterfeiter v0.0.0-20181017030959-1aadac120687/go.mod h1:aoVsckWnsNzazwF2kmD+bzgdr4GBlbK91zsdivQJ2eU= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= -github.com/mhenriks/library-go v0.0.0-20210511195009-51ba86622560/go.mod h1:udseDnqxn5ON8i+NBjDp00fBTK0JRu1/6Y6tf6EivDE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= -github.com/mistifyio/go-zfs v2.1.1+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= @@ -1016,8 +942,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mrnold/go-libnbd v1.4.1-cdi/go.mod h1:t/zovtHFkgtIy65eJ+Ay1mNBFz+yO6ESu6r6CluGzdI= -github.com/mtrmac/gpgme v0.1.2/go.mod h1:GYYHnGSuS7HK3zVS2n3y73y0okK/BeKzwnn5jgiVFNI= github.com/munnerz/crd-schema-fuzz v1.0.0/go.mod h1:4z/rcm37JxUkSsExFcLL6ZIT1SgDRdLiu7qq1evdVS0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -1058,7 +982,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= @@ -1070,7 +993,6 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.2-0.20180831124310-ae19f1b56d53/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -1091,21 +1013,12 @@ github.com/open-policy-agent/gatekeeper v0.0.0-20201111000257-4450f08fa95e/go.mo github.com/open-policy-agent/opa v0.19.1/go.mod h1:rrwxoT/b011T0cyj+gg2VvxqTtn6N3gp/jzmr3fjW44= github.com/open-policy-agent/opa v0.21.0/go.mod h1:cZaTfhxsj7QdIiUI0U9aBtOLLTqVNe+XE60+9kZKLHw= github.com/open-policy-agent/opa v0.24.0/go.mod h1:qEyD/i8j+RQettHGp4f86yjrjvv+ZYia+JHCMv2G7wA= -github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v0.0.0-20191031171055-b133feaeeb2e/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc90/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.5.2/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g= -github.com/openshift/api v0.0.0-20210428205234-a8389931bee7/go.mod h1:aqU5Cq+kqKKPbDMqxo9FojgDeSpNJI7iuskjXjtojDg= -github.com/openshift/build-machinery-go v0.0.0-20200917070002-f171684f77ab/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE= -github.com/openshift/client-go v0.0.0-20210112165513-ebc401615f47/go.mod h1:u7NRAjtYVAKokiI9LouzTv4mhds8P4S1TwdVAfbjKSk= +github.com/openshift/api v0.0.0-20191219222812-2987a591a72c/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= +github.com/openshift/client-go v0.0.0-20191125132246-f6563a70e19a/go.mod h1:6rzn+JTr7+WYS2E1TExP4gByoABxMznR6y2SnUIkmxk= github.com/openshift/custom-resource-status v0.0.0-20190822192428-e62f2f3b79f3/go.mod h1:GDjWl0tX6FNIj82vIxeudWeSx2Ff6nDZ8uJn0ohUFvo= github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca h1:F1MEnOMwSrTA0YAkO0he9ip9w0JhYzI/iCB2mXmaSPg= github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca/go.mod h1:GDjWl0tX6FNIj82vIxeudWeSx2Ff6nDZ8uJn0ohUFvo= @@ -1120,13 +1033,9 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ github.com/openzipkin/zipkin-go v0.2.0/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/operator-framework/operator-lifecycle-manager v0.0.0-20190128024246-5eb7ae5bdb7a/go.mod h1:vq6TTFvg6ti1Bn6ACsZneZTmjTsURgDD6tQtVDbEgsU= -github.com/operator-framework/operator-registry v1.0.4/go.mod h1:hve6YwcjM2nGVlscLtNsp9sIIBkNZo6jlJgzWw7vP9s= -github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc= github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= -github.com/ovirt/go-ovirt v4.3.4+incompatible/go.mod h1:r33ZGjVKCPMiI6hw791/Zx8tNKk0Gn+4VFWbOfyIvZQ= github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888 h1:ARzSqjQJcSR9IXGMEkXN1IQEZChwd2MqXWm6YNGvO5o= github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888/go.mod h1:RQHg5xR1F614BwJyepfMqrKN+32IH0i7yX+ey43rEeQ= github.com/packethost/pkg v0.0.0-20200903155310-0433e0605550/go.mod h1:GSv7cTtIjns4yc0pyajaM1RE/KE4djJONoblFIRDrxA= @@ -1158,7 +1067,6 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= -github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -1167,8 +1075,6 @@ github.com/poy/onpar v0.0.0-20200406201722-06f95a1c68e8/go.mod h1:nSbFQvMj97ZyhF github.com/poy/onpar v1.0.1/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= -github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/prometheus/client_golang v0.0.0-20181025174421-f30f42803563/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -1197,7 +1103,6 @@ github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7q github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20190104105734-b1c43a6df3ae/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -1209,12 +1114,10 @@ github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16 github.com/prometheus/common v0.25.0/go.mod h1:H6QK/N6XVT42whUeIdI3dp36w49c+/iMDk7UAI2qm7Q= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190104112138-b1a0a9a36d74/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -1237,14 +1140,12 @@ github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfm github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= -github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg= github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -1257,7 +1158,6 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7 h1:Do8ksLD4Nr3pA0x0hnLOLftZgkiTDvwPDShRTUxtXpE= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8= -github.com/sclevine/spec v1.0.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -1335,13 +1235,10 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/tchap/go-patricia v2.3.0+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/tektoncd/pipeline v0.10.1/go.mod h1:D2X0exT46zYx95BU7ByM8+erpjoN7thmUBvlKThOszU= github.com/tektoncd/plumbing v0.0.0-20191216083742-847dcf196de9/go.mod h1:QZHgU07PRBTRF6N57w4+ApRu8OgfYLFNqCDlfEZaD9Y= github.com/tektoncd/plumbing/pipelinerun-logs v0.0.0-20191206114338-712d544c2c21/go.mod h1:S62EUWtqmejjJgUMOGB1CCCHRp6C706laH06BoALkzU= @@ -1356,28 +1253,20 @@ github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJ github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v0.0.0-20181022190402-e5e69e061d4f/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.18.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= -github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g= -github.com/vbauerster/mpb/v5 v5.2.2/go.mod h1:W5Fvgw4dm3/0NhqzV8j6EacfuTe5SvnzBRwiXxDR9ww= github.com/vdemeester/k8s-pkg-credentialprovider v0.0.0-20200107171650-7c61ffa44238/go.mod h1:JwQJCMWpUDqjZrB5jpw0f5VbN7U95zxFy1ZDpoEarGo= github.com/vdemeester/k8s-pkg-credentialprovider v1.13.12-1/go.mod h1:Fko0rTxEtDW2kju5Ky7yFJNS3IcNvW8IPsp4/e9oev0= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50 h1:uxE3GYdXIOfhMv3unJKETJEhw78gvzuQqRX/rVirc2A= github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= -github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/vmware/govmomi v0.23.1 h1:vU09hxnNR/I7e+4zCJvW+5vHu5dO64Aoe2Lw7Yi/KRg= github.com/vmware/govmomi v0.23.1/go.mod h1:Y+Wq4lst78L85Ge/F8+ORXIWiKYqaro1vhAulACy9Lc= @@ -1386,10 +1275,6 @@ github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0B github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= @@ -1404,7 +1289,6 @@ github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wK go.etcd.io/bbolt v1.3.1-etcd.7/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.0.0-20181031231232-83304cfc808c/go.mod h1:weASp41xM3dk0YHg1s/W8ecdGP5G4teSTMBPpYAaUgA= @@ -1507,7 +1391,6 @@ golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -1558,11 +1441,9 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1670,7 +1551,6 @@ golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1685,7 +1565,6 @@ golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1693,19 +1572,16 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1773,7 +1649,6 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20180810170437-e96c4e24768d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011152555-a398e557df60/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1926,7 +1801,6 @@ google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191220175831-5c49e3ecc1c1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -1961,7 +1835,6 @@ google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxH google.golang.org/grpc v1.13.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= -google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -2036,7 +1909,6 @@ gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eR gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= -gopkg.in/ldap.v2 v2.5.1/go.mod h1:oI0cpe/D7HRtBQl8aTg+ZmzFUAvu4lsv3eLXMLGFxWk= gopkg.in/natefinch/lumberjack.v2 v2.0.0-20150622162204-20b71e5b60d7/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -2094,8 +1966,6 @@ k8c.io/operating-system-manager v0.3.0 h1:xu1BA1Uj22MAeXSx9mNumfm63/6P8xjRcojel+ k8c.io/operating-system-manager v0.3.0/go.mod h1:ME5GOCNUrHG+57igEKP1JCJKVHynaLfodT8bRiYH3MY= k8s.io/api v0.0.0-20181018013834-843ad2d9b9ae/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20181115043458-b799cb063522/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= -k8s.io/api v0.0.0-20181203235848-2dd39edadc55/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= -k8s.io/api v0.0.0-20190118113203-912cbe2bfef3/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20190725062911-6607c48751ae/go.mod h1:1O0xzX/RAtnm7l+5VEUxZ1ysO2ghatfq/OZED4zM9kA= k8s.io/api v0.0.0-20190918155943-95b840bb6a1f/go.mod h1:uWuOHnjmNrtQomJrvEBg0c0HRNyQ+8KTEERVsK0PW48= k8s.io/api v0.0.0-20190918195907-bd6ac527cfd2/go.mod h1:AOxZTnaXR/xiarlQL0JUfwQPxjmKDvVYoRp58cA7lUo= @@ -2103,7 +1973,6 @@ k8s.io/api v0.16.4/go.mod h1:AtzMnsR45tccQss5q8RnF+W8L81DH6XwXwo/joEx9u0= k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= k8s.io/api v0.17.2/go.mod h1:BS9fjjLc4CMuqfSO8vgbHPKMt5+SF0ET6u/RVDihTo4= -k8s.io/api v0.18.0-beta.2/go.mod h1:2oeNnWEqcSmaM/ibSh3t7xcIqbkGXhzZdn4ezV9T4m0= k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= k8s.io/api v0.18.4/go.mod h1:lOIQAKYgai1+vz9J7YcDZwC26Z0zQewYOGWdyIPUUQ4= @@ -2112,19 +1981,15 @@ k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw= k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= k8s.io/api v0.19.4/go.mod h1:SbtJ2aHCItirzdJ36YslycFNzWADYH3tgOhvBEFtZAk= -k8s.io/api v0.20.0/go.mod h1:HyLC5l5eoS/ygQYl1BXBgFzWNlkHiAuyNAbevIn+FKg= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= k8s.io/api v0.22.2 h1:M8ZzAD0V6725Fjg53fKeTJxGsJvRbk4TEm/fexHMtfw= k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= -k8s.io/apiextensions-apiserver v0.0.0-20181204003618-e419c5771cdc/go.mod h1:IxkesAMoaCRoLrPJdZNZUQp9NfZnzqaVzLhb2VEQzXE= k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY= k8s.io/apiextensions-apiserver v0.0.0-20190918201827-3de75813f604/go.mod h1:7H8sjDlWQu89yWB3FhZfsLyRCRLuoXoCoY5qtwW1q6I= k8s.io/apiextensions-apiserver v0.16.4/go.mod h1:HYQwjujEkXmQNhap2C9YDdIVOSskGZ3et0Mvjcyjbto= -k8s.io/apiextensions-apiserver v0.17.0/go.mod h1:XiIFUakZywkUl54fVXa7QTEHcqQz9HG55nHd1DCoHj8= k8s.io/apiextensions-apiserver v0.17.2/go.mod h1:4KdMpjkEjjDI2pPfBA15OscyNldHWdBCfsWMDWAmSTs= -k8s.io/apiextensions-apiserver v0.18.0-beta.2/go.mod h1:Hnrg5jx8/PbxRbUoqDGxtQkULjwx8FDW4WYJaKNK+fk= k8s.io/apiextensions-apiserver v0.18.0/go.mod h1:18Cwn1Xws4xnWQNC00FLq1E350b9lUF+aOdIWDOZxgo= k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY= k8s.io/apiextensions-apiserver v0.18.4/go.mod h1:NYeyeYq4SIpFlPxSAB6jHPIdvu3hL0pc36wuRChybio= @@ -2132,15 +1997,12 @@ k8s.io/apiextensions-apiserver v0.18.6/go.mod h1:lv89S7fUysXjLZO7ke783xOwVTm6lKi k8s.io/apiextensions-apiserver v0.19.0/go.mod h1:znfQxNpjqz/ZehvbfMg5N6fvBJW5Lqu5HVLTJQdP4Fs= k8s.io/apiextensions-apiserver v0.19.2/go.mod h1:EYNjpqIAvNZe+svXVx9j4uBaVhTB4C94HkY3w058qcg= k8s.io/apiextensions-apiserver v0.19.4/go.mod h1:B9rpH/nu4JBCtuUp3zTTk8DEjZUupZTBEec7/2zNRYw= -k8s.io/apiextensions-apiserver v0.20.0/go.mod h1:ZH+C33L2Bh1LY1+HphoRmN1IQVLTShVcTojivK3N9xg= k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= k8s.io/apiextensions-apiserver v0.20.2/go.mod h1:F6TXp389Xntt+LUq3vw6HFOLttPa0V8821ogLGwb6Zs= k8s.io/apiextensions-apiserver v0.22.2 h1:zK7qI8Ery7j2CaN23UCFaC1hj7dMiI87n01+nKuewd4= k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA= k8s.io/apimachinery v0.0.0-20181015213631-60666be32c5d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/apimachinery v0.0.0-20181110190943-2a7c93004028/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= -k8s.io/apimachinery v0.0.0-20181203235515-3d8ee2261517/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= -k8s.io/apimachinery v0.0.0-20190118094746-1525e4dadd2d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/apimachinery v0.0.0-20190703205208-4cfb76a8bf76/go.mod h1:M2fZgZL9DbLfeJaPBCDqSqNsdsmLN+V29knYJnIXlMA= k8s.io/apimachinery v0.0.0-20190719140911-bfcf53abc9f8/go.mod h1:sBJWIJZfxLhp7mRsRyuAE/NfKTr3kXGR1iaqg8O0gJo= k8s.io/apimachinery v0.0.0-20190816221834-a9f1d8a9c101/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= @@ -2149,10 +2011,8 @@ k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT k8s.io/apimachinery v0.16.4/go.mod h1:llRdnznGEAqC3DcNm6yEj472xaFVfLM7hnYofMb12tQ= k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.17.1-beta.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apimachinery v0.17.1/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apimachinery v0.17.2/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.18.0-beta.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= k8s.io/apimachinery v0.18.4/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= @@ -2161,20 +2021,17 @@ k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCk k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apimachinery v0.19.4/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= -k8s.io/apimachinery v0.20.0/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.22.2 h1:ejz6y/zNma8clPVfNDLnPbleBo6MpoFy/HBiBqCouVk= k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apiserver v0.0.0-20181026151315-13cfe3978170/go.mod h1:6bqaTSOSJavUIXUtfaR9Os9JtTCm8ZqH2SUl2S60C4w= k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg= k8s.io/apiserver v0.0.0-20190918200908-1e17798da8c1/go.mod h1:4FuDU+iKPjdsdQSN3GsEKZLB/feQsj1y9dhhBDVV2Ns= k8s.io/apiserver v0.16.4/go.mod h1:kbLJOak655g6W7C+muqu1F76u9wnEycfKMqbVaXIdAc= k8s.io/apiserver v0.17.0/go.mod h1:ABM+9x/prjINN6iiffRVNCBR2Wk7uY4z+EtEGZD48cg= k8s.io/apiserver v0.17.0/go.mod h1:ABM+9x/prjINN6iiffRVNCBR2Wk7uY4z+EtEGZD48cg= k8s.io/apiserver v0.17.2/go.mod h1:lBmw/TtQdtxvrTk0e2cgtOxHizXI+d0mmGQURIHQZlo= -k8s.io/apiserver v0.18.0-beta.2/go.mod h1:bnblMkMoCFnIfVnVftd0SXJPzyvrk3RtaqSbblphF/A= k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw= k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw= k8s.io/apiserver v0.18.4/go.mod h1:q+zoFct5ABNnYkGIaGQ3bcbUNdmPyOCoEBcg51LChY8= @@ -2182,7 +2039,6 @@ k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg= k8s.io/apiserver v0.19.0/go.mod h1:XvzqavYj73931x7FLtyagh8WibHpePJ1QwWrSJs2CLk= k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= k8s.io/apiserver v0.19.4/go.mod h1:X8WRHCR1UGZDd7HpV0QDc1h/6VbbpAeAGyxSh8yzZXw= -k8s.io/apiserver v0.20.0/go.mod h1:6gRIWiOkvGvQt12WTYmsiYoUyYW0FXSiMdNl4m+sxY8= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.2/go.mod h1:2nKd93WyMhZx4Hp3RfgH2K5PhwyTrprrkWYnI7id7jA= k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= @@ -2192,18 +2048,13 @@ k8s.io/cli-runtime v0.19.4/go.mod h1:m8G32dVbKOeaX1foGhleLEvNd6REvU7YnZyWn5//9rw k8s.io/client-go v0.22.2 h1:DaSQgs02aCC1QcwUdkKZWOeaVsQjYvWv8ZazcZ6JcHc= k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= k8s.io/cloud-provider v0.17.0/go.mod h1:Ze4c3w2C0bRsjkBUoHpFi+qWe3ob1wI2/7cUn+YQIDE= -k8s.io/cluster-bootstrap v0.22.2/go.mod h1:ZkmQKprEqvrUccMnbRHISsMscA1dsQ8SffM9nHq6CgE= k8s.io/code-generator v0.0.0-20181114232248-ae218e241252/go.mod h1:IPqxl/YHk05nodzupwjke6ctMjyNRdV2zZ5/j3/F204= -k8s.io/code-generator v0.0.0-20181203235156-f8cba74510f3/go.mod h1:MYiN+ZJZ9HkETbgVZdWw2AsuAi9PZ4V80cwfuf2axe8= k8s.io/code-generator v0.0.0-20190612205613-18da4a14b22b/go.mod h1:G8bQwmHm2eafm5bgtX67XDZQ8CWKSGu9DekI+yN4Y5I= k8s.io/code-generator v0.0.0-20190717022600-77f3a1fe56bb/go.mod h1:cDx5jQmWH25Ff74daM7NVYty9JWw9dvIS9zT9eIubCY= k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE= -k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= k8s.io/code-generator v0.16.4/go.mod h1:mJUgkl06XV4kstAnLHAIzJPVCOzVR+ZcfPIv4fUsFCY= -k8s.io/code-generator v0.17.0/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= k8s.io/code-generator v0.17.1/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= k8s.io/code-generator v0.17.2/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= -k8s.io/code-generator v0.18.0-beta.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= k8s.io/code-generator v0.18.4/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= @@ -2211,7 +2062,6 @@ k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8 k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= k8s.io/code-generator v0.19.4/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= -k8s.io/code-generator v0.20.0/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.2/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.4/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= @@ -2222,7 +2072,6 @@ k8s.io/component-base v0.16.4/go.mod h1:GYQ+4hlkEwdlpAp59Ztc4gYuFhdoZqiAJD1unYDJ k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= k8s.io/component-base v0.17.2/go.mod h1:zMPW3g5aH7cHJpKYQ/ZsGMcgbsA/VyhEugF3QT1awLs= -k8s.io/component-base v0.18.0-beta.2/go.mod h1:HVk5FpRnyzQ/MjBr9//e/yEBjTVa2qjGXCTuUzcD7ks= k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c= k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM= k8s.io/component-base v0.18.4/go.mod h1:7jr/Ef5PGmKwQhyAz/pjByxJbC58mhKAhiaDu0vXfPk= @@ -2230,14 +2079,12 @@ k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4I k8s.io/component-base v0.19.0/go.mod h1:dKsY8BxkA+9dZIAh2aWJLL/UdASFDNtGYTCItL4LM7Y= k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo= k8s.io/component-base v0.19.4/go.mod h1:ZzuSLlsWhajIDEkKF73j64Gz/5o0AgON08FgRbEPI70= -k8s.io/component-base v0.20.0/go.mod h1:wKPj+RHnAr8LW2EIBIK7AxOHPde4gme2lzXwVSoRXeA= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= k8s.io/component-base v0.22.2 h1:vNIvE0AIrLhjX8drH0BgCNJcR4QZxMXcJzBsDplDx9M= k8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug= k8s.io/csi-translation-lib v0.17.0/go.mod h1:HEF7MEz7pOLJCnxabi45IPkhSsE/KmxPQksuCrHKWls= k8s.io/gengo v0.0.0-20181106084056-51747d6e00da/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20181113154421-fd15ee9cc2f7/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -2249,7 +2096,6 @@ k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v0.0.0-20190306015804-8e90cee79f82/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.1.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= @@ -2262,12 +2108,8 @@ k8s.io/klog/v2 v2.3.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= -k8s.io/kube-aggregator v0.0.0-20181204002017-122bac39d429/go.mod h1:8sbzT4QQKDEmSCIbfqjV0sd97GpUT7A4W626sBiYJmU= -k8s.io/kube-aggregator v0.18.0-beta.2/go.mod h1:O3Td9mheraINbLHH4pzoFP2gRzG0Wk1COqzdSL4rBPk= k8s.io/kube-aggregator v0.19.0/go.mod h1:1Ln45PQggFAG8xOqWPIYMxUq8WNtpPnYsbUJ39DpF/A= k8s.io/kube-aggregator v0.19.4/go.mod h1:cTkvun110194d797AuThyydBBlgm+cKIFUeS2uzGJfU= -k8s.io/kube-aggregator v0.20.0/go.mod h1:3Is/gzzWmhhG/rA3CpA1+eVye87lreBQDFGcAGT7gzo= -k8s.io/kube-aggregator v0.20.2/go.mod h1:j7ks4pWm6cjXzlVZB9tewvUdg2njjbiFuHp575ZKnqc= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= @@ -2277,8 +2119,6 @@ k8s.io/kubectl v0.19.4/go.mod h1:XPmlu4DJEYgD83pvZFeKF8+MSvGnYGqunbFSrJsqHv0= k8s.io/kubelet v0.19.4/go.mod h1:zJnPeb7nJCRvtAwxJhe9fFCtMLXL3cXbQiczPmpDrLU= k8s.io/kubelet v0.22.2 h1:7ol5AXXxcW97dUE8W/QiPjkXu1ZuGshG5VmgDmviZsc= k8s.io/kubelet v0.22.2/go.mod h1:ORIRua2/wTcx5UnEvxWosu650/8fatmzbMRC7m6WjAM= -k8s.io/kubernetes v1.11.8-beta.0.0.20190124204751-3a10094374f2/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= -k8s.io/kubernetes v1.14.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/legacy-cloud-providers v0.17.0/go.mod h1:DdzaepJ3RtRy+e5YhNtrCYwlgyK87j/5+Yfp0L9Syp8= k8s.io/metrics v0.19.0/go.mod h1:WykpW8B60OeAJx1imdwUgyOID2kDljr/Q+1zrPJ98Wo= k8s.io/metrics v0.19.4/go.mod h1:a0gvAzrxQPw2ouBqnXI7X9qlggpPkKAFgWU/Py+KZiU= @@ -2290,7 +2130,6 @@ k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -2303,14 +2142,14 @@ k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ knative.dev/caching v0.0.0-20190719140829-2032732871ff/go.mod h1:dHXFU6CGlLlbzaWc32g80cR92iuBSpsslDNBWI8C7eg= knative.dev/eventing-contrib v0.6.1-0.20190723221543-5ce18048c08b/go.mod h1:SnXZgSGgMSMLNFTwTnpaOH7hXDzTFtw0J8OmHflNx3g= knative.dev/pkg v0.0.0-20191111150521-6d806b998379/go.mod h1:pgODObA1dTyhNoFxPZTTjNWfx6F0aKsKzn+vaT9XO/Q= -kubevirt.io/client-go v0.30.0 h1:0jUvTa/Ev03lCN+Dr4mH22ipoJ9otAOkpFh6wA66b5M= +kubevirt.io/api v0.48.1 h1:C5i9h8ea7Xy3fJMoKEuzjRP74GnVMF7u2mQV8FGf2XE= +kubevirt.io/api v0.48.1/go.mod h1:RoYMmFt76vWvFtw/FSiL0YUHZ2Ao6UfXlgpZAQnRswo= kubevirt.io/client-go v0.30.0/go.mod h1:JY7hQq+SUT0aLvleXrW/+28fDfZ6BPe4E6f8FyC8jkY= kubevirt.io/containerized-data-importer v1.10.6/go.mod h1:qF594BtRRkruyrqLwt3zbLCWdPIQNs1qWh4LR1cOzy0= -kubevirt.io/containerized-data-importer v1.40.0 h1:EjrlOxWKe/gktOC4elC1JCfbynS8erdf9SfmfXJpJ3c= -kubevirt.io/containerized-data-importer v1.40.0/go.mod h1:IlhJj5CBgyzXSfgrPBDmHqCmadEl19Vs04YE3fgtsf8= -kubevirt.io/controller-lifecycle-operator-sdk v0.2.1-0.20210723143736-64585ea1d1bd h1:QCXqLkTzaBTpPw4Onzh31I5L72orz6ItavRyc6TEe2c= -kubevirt.io/controller-lifecycle-operator-sdk v0.2.1-0.20210723143736-64585ea1d1bd/go.mod h1:ZJhLceiY2Gl5CXFGSp5eMGt/sksOiJP0289nAZFCQf0= -kubevirt.io/qe-tools v0.1.6/go.mod h1:PJyH/YXC4W0AmxfheDmXWMbLNsMSboVGXKpMAwfKzVE= +kubevirt.io/containerized-data-importer-api v1.41.0 h1:VdEwYP36N+4asMnTBSadVH4SF7OVPvvraEQMtOd7Vlk= +kubevirt.io/containerized-data-importer-api v1.41.0/go.mod h1:0xadDFtaMd8iy+/oD2+dYoPxACZ/YizKqay5QIrQ6cw= +kubevirt.io/controller-lifecycle-operator-sdk v0.2.1 h1:I1b14fnhwrVvQLmgksMo9vgje42hmH4QN5kqyYDqbMA= +kubevirt.io/controller-lifecycle-operator-sdk v0.2.1/go.mod h1:ZJhLceiY2Gl5CXFGSp5eMGt/sksOiJP0289nAZFCQf0= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= @@ -2335,11 +2174,9 @@ sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf sigs.k8s.io/controller-runtime v0.10.2 h1:jW8qiY+yMnnPx6O9hu63tgcwaKzd1yLYui+mpvClOOc= sigs.k8s.io/controller-runtime v0.10.2/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= sigs.k8s.io/controller-tools v0.2.4/go.mod h1:m/ztfQNocGYBgTTCmFdnK94uVvgxeZeE3LtJvd/jIzA= -sigs.k8s.io/controller-tools v0.2.8/go.mod h1:9VKHPszmf2DHz/QmHkcfZoewO6BL7pPs9uAiBVsaJSE= sigs.k8s.io/controller-tools v0.2.9-0.20200414181213-645d44dca7c0/go.mod h1:YKE/iHvcKITCljdnlqHYe+kAt7ZldvtAwUzQff0k1T0= sigs.k8s.io/controller-tools v0.4.1/go.mod h1:G9rHdZMVlBDocIxGkK3jHLWqcTMNvveypYJwrvYKjWU= sigs.k8s.io/controller-tools v0.5.0/go.mod h1:JTsstrMpxs+9BUj6eGuAaEb6SDSPTeVtUyp0jmnAM/I= -sigs.k8s.io/kube-storage-version-migrator v0.0.3/go.mod h1:mXfSLkx9xbJHQsgNDDUZK/iQTs2tMbx/hsJlWe6Fthw= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index a98dde1f9..f8818e2c7 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -25,8 +25,8 @@ import ( "strings" "time" - kubevirtv1 "kubevirt.io/client-go/api/v1" - cdi "kubevirt.io/containerized-data-importer/pkg/apis/core/v1alpha1" + kubevirtv1 "kubevirt.io/api/core/v1" + cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -45,13 +45,15 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog" utilpointer "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" ) func init() { - // Workaround until we have https://github.com/kubevirt/kubevirt/pull/2841 - metav1.AddToGroupVersion(scheme.Scheme, kubevirtv1.GroupVersion) + if err := kubevirtv1.AddToScheme(scheme.Scheme); err != nil { + klog.Fatalf("failed to add kubevirtv1 to scheme: %v", err) + } } var supportedOS = map[providerconfigtypes.OperatingSystem]*struct{}{ @@ -408,12 +410,12 @@ func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provi DNSConfig: c.DNSConfig, }, }, - DataVolumeTemplates: []cdi.DataVolume{ + DataVolumeTemplates: []kubevirtv1.DataVolumeTemplateSpec{ { ObjectMeta: metav1.ObjectMeta{ Name: dataVolumeName, }, - Spec: cdi.DataVolumeSpec{ + Spec: cdiv1beta1.DataVolumeSpec{ PVC: &corev1.PersistentVolumeClaimSpec{ StorageClassName: utilpointer.StringPtr(c.StorageClassName), AccessModes: []corev1.PersistentVolumeAccessMode{ @@ -423,8 +425,8 @@ func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provi Requests: pvcRequest, }, }, - Source: cdi.DataVolumeSource{ - HTTP: &cdi.DataVolumeSourceHTTP{ + Source: &cdiv1beta1.DataVolumeSource{ + HTTP: &cdiv1beta1.DataVolumeSourceHTTP{ URL: c.SourceURL, }, }, From 86b1621658bb2e4a38ce3ff3c7c8d0c621aff1b0 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Mon, 3 Jan 2022 16:25:32 +0100 Subject: [PATCH 045/489] Nutanix cloud provider support (#1132) * skeleton setup of nutanix cloud provider Signed-off-by: Marvin Beckers * implement waiting for VM to be fully up Signed-off-by: Marvin Beckers * implement logic for getting VM information Signed-off-by: Marvin Beckers * add cleanup and metrics labels Signed-off-by: Marvin Beckers * allow loading client-related configuration from env variables Signed-off-by: Marvin Beckers * improve construction of VM request Signed-off-by: Marvin Beckers * merge utils.go into client.go Signed-off-by: Marvin Beckers * add support for passing a storage container ID Signed-off-by: Marvin Beckers * remove 'omitempty' from provider config type Signed-off-by: Marvin Beckers * add support for cpu passthrough Signed-off-by: Marvin Beckers * implement port configuration field Signed-off-by: Marvin Beckers * add initial documentation and example Signed-off-by: Marvin Beckers * Validate that disk size is not smaller than the image Signed-off-by: Marvin Beckers * ensure that image status contains size Signed-off-by: Marvin Beckers * update docs to clarify on requirements Signed-off-by: Marvin Beckers * update API call to define disk and VM name Signed-off-by: Marvin Beckers * add note for API support to doc Signed-off-by: Marvin Beckers * Wrap and return errors correctly and read name from status Signed-off-by: Marvin Beckers * fix name source, image kind and power state Signed-off-by: Marvin Beckers * don't use generated spec.name for VM name Signed-off-by: Marvin Beckers * Add allowInsecure setting to example Signed-off-by: Marvin Beckers * update memory in example; 1GB resulted in unstable node Signed-off-by: Marvin Beckers * make projectName optional Signed-off-by: Marvin Beckers * return cloudprovidererrors.TerminalError where appropriate Signed-off-by: Marvin Beckers * fix CI failures Signed-off-by: Marvin Beckers * replace github.com/nutanix/terraform-provider-nutanix with temporary github.com/embik/nutanix-client-go Signed-off-by: Marvin Beckers * fix license check Signed-off-by: Marvin Beckers --- .wwhrd.yml | 3 + docs/nutanix.md | 17 + examples/nutanix-machinedeployment.yaml | 81 ++++ go.mod | 1 + go.sum | 7 +- pkg/cloudprovider/provider.go | 4 + pkg/cloudprovider/provider/nutanix/client.go | 456 ++++++++++++++++++ .../provider/nutanix/provider.go | 423 ++++++++++++++++ .../provider/nutanix/types/types.go | 66 +++ pkg/providerconfig/types/types.go | 2 + 10 files changed, 1058 insertions(+), 2 deletions(-) create mode 100644 docs/nutanix.md create mode 100644 examples/nutanix-machinedeployment.yaml create mode 100644 pkg/cloudprovider/provider/nutanix/client.go create mode 100644 pkg/cloudprovider/provider/nutanix/provider.go create mode 100644 pkg/cloudprovider/provider/nutanix/types/types.go diff --git a/.wwhrd.yml b/.wwhrd.yml index 32147a2ba..d3c17d045 100644 --- a/.wwhrd.yml +++ b/.wwhrd.yml @@ -27,4 +27,7 @@ allowlist: exceptions: - github.com/hashicorp/golang-lru # MPL-2.0 - github.com/hashicorp/golang-lru/simplelru # MPL-2.0 + - github.com/embik/nutanix-client-go/pkg/client # MPL-2.0 + - github.com/embik/nutanix-client-go/pkg/client/v3 # MPL-2.0 + - github.com/embik/nutanix-client-go/internal/utils # MPL-2.0 - github.com/ajeddeloh/go-json # Since it's a fork, https://github.com/golang/go/blob/master/LICENSE diff --git a/docs/nutanix.md b/docs/nutanix.md new file mode 100644 index 000000000..046beb63f --- /dev/null +++ b/docs/nutanix.md @@ -0,0 +1,17 @@ +# Nutanix Prism Central + +This provider implementation is currently in **alpha** stage. Currently, the only supported API version is [Prism v3](https://www.nutanix.dev/reference/prism_central/v3/). + +## Prerequisites + +The `nutanix` provider assumes several things to be preexisting. You need: + +- Credentials and access information for a Nutanix Prism Central instance (endpoint, port, username and password). +- The name of a Nutanix cluster to create the VMs for Machines on. +- The name of a subnet on the given Nutanix cluster that the VMs' network interfaces will be created on. +- An image name that will be used to create the VM for (must match the configured operating system). +- **Optional**: The name of a project that the given credentials have access to, to create the VMs in. If none is provided, the VMs are created without a project. + +## Configuration Options + +An example `MachineDeployment` can be found [here](../examples/nutanix-machinedeployment.yaml). diff --git a/examples/nutanix-machinedeployment.yaml b/examples/nutanix-machinedeployment.yaml new file mode 100644 index 000000000..d9c42139c --- /dev/null +++ b/examples/nutanix-machinedeployment.yaml @@ -0,0 +1,81 @@ +apiVersion: v1 +kind: Secret +metadata: + # If you change the namespace/name, you must also + # adjust the rbac rules + name: machine-controller-nutanix + namespace: kube-system +type: Opaque +stringData: + password: << NUTANIX_PASSWORD >> +--- +apiVersion: "cluster.k8s.io/v1alpha1" +kind: MachineDeployment +metadata: + name: nutanix-machinedeployment + namespace: kube-system +spec: + paused: false + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + minReadySeconds: 0 + selector: + matchLabels: + foo: bar + template: + metadata: + labels: + foo: bar + spec: + providerSpec: + value: + sshPublicKeys: + - "<< YOUR_PUBLIC_KEY >>" + cloudProvider: "nutanix" + cloudProviderSpec: + # Can also be set via the env var 'NUTANIX_USERNAME' on the machine-controller + username: '<< NUTANIX_USERNAME >>' + # Can also be set via the env var 'NUTANIX_ENDPOINT' on the machine-controller + # example: 'your-nutanix-host' or '10.0.1.5'. No protocol or port should be passed. + endpoint: '<< NUTANIX_ENDPOINT >>' + # Can also be set via the env var 'NUTANIX_PORT' on the machine-controller + # if not set, defaults to 9440 (default Nutanix port) + port: '<< NUTANIX_PORT >>' + # Optional: Allow insecure connections to endpoint if no valid TLS certificate is presented + allowInsecure: true + # Can also be set via the env var 'NUTANIX_PASSWORD' on the machine-controller + password: + secretKeyRef: + namespace: kube-system + name: machine-controller-nutanix + key: password + # Can also be set via the env var 'NUTANIX_CLUSTER_NAME' on the machine-controller + # this refers to a Nutanix cluster, not a Kubernetes cluster + clusterName: nutanix-cluster + # Optional: Sets the project that the VM is deployed into. If none is provided, the VM will be created without a project + projectName: project1 + # Sets the subnet that the VM is connected to. Must exist in the given Nutanix cluster + subnetName: subnet1 + # Provides the image used to create the VM + imageName: ubuntu-20.04 + # Sets the vCPU count for this VM + cpus: 2 + # Sets the CPU cores per vCPUs + cpuCores: 1 + # Memory configuration in MiB + memoryMB: 2048 + # Optional: Enable Nutanix' CPU passthrough feature + #cpuPassthrough: true + # Optional: Set up system disk size in GB. If not set, will be based on image size. + # Cannot be smaller than the image size. + diskSize: 20 + operatingSystem: "ubuntu" + operatingSystemSpec: + distUpgradeOnBoot: false + disableAutoUpdate: true + versions: + kubelet: 1.22.5 diff --git a/go.mod b/go.mod index 8eed6d998..4a1f7806a 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/coreos/container-linux-config-transpiler v0.9.0 github.com/davecgh/go-spew v1.1.1 github.com/digitalocean/godo v1.54.0 + github.com/embik/nutanix-client-go v0.0.0-20220103122158-dbb64d7901ab github.com/ghodss/yaml v1.0.0 github.com/go-test/deep v1.0.7 github.com/google/uuid v1.1.2 diff --git a/go.sum b/go.sum index 1a382b237..2bdb6b1c8 100644 --- a/go.sum +++ b/go.sum @@ -332,6 +332,8 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/embik/nutanix-client-go v0.0.0-20220103122158-dbb64d7901ab h1:WrwH567h9IFvCcAFORWFl+jOZqfvQc47LfnxN0nQzIk= +github.com/embik/nutanix-client-go v0.0.0-20220103122158-dbb64d7901ab/go.mod h1:gkKNSxfEt3QtYG3S/wKiN8OmrJ4fpU7JbTlbnrMDOL8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.10.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -843,6 +845,7 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -963,7 +966,6 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/nelsam/hel v0.0.0-20200611165952-2d829bae0c66/go.mod h1:Rl/hm4V2s75ScsPmI9cNz87HLNg5MoFAMJwA90fzbkw= github.com/nelsam/hel/v2 v2.3.2/go.mod h1:1ZTGfU2PFTOd5mx22i5O0Lc2GY933lQ2wb/ggy+rL3w= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -1885,8 +1887,9 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/pkg/cloudprovider/provider.go b/pkg/cloudprovider/provider.go index 78f9ba116..3e6756465 100644 --- a/pkg/cloudprovider/provider.go +++ b/pkg/cloudprovider/provider.go @@ -32,6 +32,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/hetzner" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/kubevirt" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/linode" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/nutanix" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/openstack" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/scaleway" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vsphere" @@ -100,6 +101,9 @@ var ( // TODO(MQ): add a baremetal driver. return baremetal.New(cvr) }, + providerconfigtypes.CloudProviderNutanix: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return nutanix.New(cvr) + }, } ) diff --git a/pkg/cloudprovider/provider/nutanix/client.go b/pkg/cloudprovider/provider/nutanix/client.go new file mode 100644 index 000000000..2dd3b88a7 --- /dev/null +++ b/pkg/cloudprovider/provider/nutanix/client.go @@ -0,0 +1,456 @@ +/* +Copyright 2021 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nutanix + +import ( + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + nutanixclient "github.com/embik/nutanix-client-go/pkg/client" + nutanixv3 "github.com/embik/nutanix-client-go/pkg/client/v3" + + "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" + cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" + nutanixtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/nutanix/types" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/utils/pointer" +) + +const ( + invalidCredentials = "invalid Nutanix Credentials" +) + +type ClientSet struct { + Prism *nutanixv3.Client +} + +func GetClientSet(config *Config) (*ClientSet, error) { + if config == nil { + return nil, errors.New("no configuration passed") + } + + if config.Username == "" { + return nil, errors.New("no username specified") + } + + if config.Password == "" { + return nil, errors.New("no password specificed") + } + + if config.Endpoint == "" { + return nil, errors.New("no endpoint specified") + } + + // set up 9440 as default port if none is passed via config + port := 9440 + if config.Port != nil { + port = *config.Port + } + + credentials := nutanixclient.Credentials{ + URL: fmt.Sprintf("%s:%d", config.Endpoint, port), + Endpoint: config.Endpoint, + Port: fmt.Sprint(port), + Username: config.Username, + Password: config.Password, + Insecure: config.AllowInsecure, + } + + clientV3, err := nutanixv3.NewV3Client(credentials) + if err != nil { + return nil, err + } + + return &ClientSet{ + Prism: clientV3, + }, nil +} + +func createVM(client *ClientSet, name string, conf Config, os providerconfigtypes.OperatingSystem, userdata string) (instance.Instance, error) { + cluster, err := getClusterByName(client, conf.ClusterName) + if err != nil { + return nil, err + } + + subnet, err := getSubnetByName(client, conf.SubnetName, *cluster.Metadata.UUID) + if err != nil { + return nil, err + } + + image, err := getImageByName(client, conf.ImageName) + if err != nil { + return nil, err + } + + request := &nutanixv3.VMIntentInput{ + Metadata: &nutanixv3.Metadata{ + Kind: pointer.String(nutanixtypes.VMKind), + Categories: conf.Categories, + }, + Spec: &nutanixv3.VM{ + Name: pointer.String(name), + ClusterReference: &nutanixv3.Reference{ + Kind: pointer.String(nutanixtypes.ClusterKind), + UUID: cluster.Metadata.UUID, + }, + }, + } + + resources := &nutanixv3.VMResources{ + PowerState: pointer.String("ON"), + NumSockets: pointer.Int64(conf.CPUs), + MemorySizeMib: pointer.Int64(conf.MemoryMB), + NicList: []*nutanixv3.VMNic{ + { + SubnetReference: &nutanixv3.Reference{ + Kind: pointer.String(nutanixtypes.SubnetKind), + UUID: subnet.Metadata.UUID, + }, + }, + }, + DiskList: []*nutanixv3.VMDisk{ + { + DeviceProperties: &nutanixv3.VMDiskDeviceProperties{ + DeviceType: pointer.String("DISK"), + DiskAddress: &nutanixv3.DiskAddress{ + DeviceIndex: pointer.Int64(0), + AdapterType: pointer.String("SCSI"), + }, + }, + DataSourceReference: &nutanixv3.Reference{ + Kind: pointer.String(nutanixtypes.ImageKind), + UUID: image.Metadata.UUID, + }, + }, + }, + GuestCustomization: &nutanixv3.GuestCustomization{ + CloudInit: &nutanixv3.GuestCustomizationCloudInit{ + UserData: pointer.String(base64.StdEncoding.EncodeToString([]byte(userdata))), + }, + }, + } + + if conf.ProjectName != "" { + project, err := getProjectByName(client, conf.ProjectName) + if err != nil { + return nil, fmt.Errorf("failed to get project: %v", err) + } + + request.Metadata.ProjectReference = &nutanixv3.Reference{ + Kind: pointer.String(nutanixtypes.ProjectKind), + UUID: project.Metadata.UUID, + } + } + + if conf.CPUCores != nil { + resources.NumVcpusPerSocket = conf.CPUCores + } + + if conf.CPUPassthrough != nil { + resources.EnableCPUPassthrough = conf.CPUPassthrough + } + + if conf.DiskSizeGB != nil { + resources.DiskList[0].DiskSizeMib = pointer.Int64(*conf.DiskSizeGB * 1024) + } + + request.Spec.Resources = resources + + resp, err := client.Prism.V3.CreateVM(request) + if err != nil { + return nil, wrapNutanixError(err) + } + + taskUUID := resp.Status.ExecutionContext.TaskUUID.(string) + + if err := waitForCompletion(client, taskUUID, time.Second*10, time.Minute*15); err != nil { + return nil, fmt.Errorf("failed to wait for task: %v", err) + } + + if resp.Metadata.UUID == nil { + return nil, errors.New("did not get response with UUID") + } + + if err := waitForPowerState(client, *resp.Metadata.UUID, time.Second*10, time.Minute*10); err != nil { + return nil, fmt.Errorf("failed to wait for power state: %v", err) + } + + vm, err := client.Prism.V3.GetVM(*resp.Metadata.UUID) + if err != nil { + return nil, wrapNutanixError(err) + } + + if vm.Status.Name == nil { + return nil, fmt.Errorf("request for VM UUID '%s' did not return name", *resp.Metadata.UUID) + } + + addresses, err := getIPs(client, *vm.Metadata.UUID, time.Second*5, time.Minute*10) + if err != nil { + return nil, fmt.Errorf("failed to get addresses: %v", err) + } + + return Server{ + name: *vm.Status.Name, + id: *resp.Metadata.UUID, + status: instance.StatusRunning, + addresses: addresses, + }, nil +} + +func getSubnetByName(client *ClientSet, name, clusterID string) (*nutanixv3.SubnetIntentResponse, error) { + filter := fmt.Sprintf("name==%s", name) + subnets, err := client.Prism.V3.ListAllSubnet(filter) + + if err != nil { + return nil, wrapNutanixError(err) + } + + for _, subnet := range subnets.Entities { + if *subnet.Status.Name == name && *subnet.Status.ClusterReference.UUID == clusterID { + return subnet, nil + } + } + + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("no subnet found for name==%s", name), + } +} + +func getProjectByName(client *ClientSet, name string) (*nutanixv3.Project, error) { + filter := fmt.Sprintf("name==%s", name) + projects, err := client.Prism.V3.ListAllProject(filter) + + if err != nil { + return nil, wrapNutanixError(err) + } + + if projects == nil || projects.Entities == nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("no project found for name==%s", name), + } + + } + + for _, project := range projects.Entities { + if project == nil { + return nil, errors.New("project is nil") + } + + if project.Status == nil { + return nil, errors.New("project status is nil") + } + + if project.Status.Name == name { + return project, nil + } + } + + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("no project found for name==%s", name), + } +} + +func getClusterByName(client *ClientSet, name string) (*nutanixv3.ClusterIntentResponse, error) { + filter := fmt.Sprintf("name==%s", name) + clusters, err := client.Prism.V3.ListAllCluster(filter) + + if err != nil { + return nil, wrapNutanixError(err) + } + + if clusters == nil || clusters.Entities == nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("no cluster found for name==%s", name), + } + } + + for _, cluster := range clusters.Entities { + if *cluster.Status.Name == name { + return cluster, nil + } + } + + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("no cluster found for name==%s", name), + } +} + +func getImageByName(client *ClientSet, name string) (*nutanixv3.ImageIntentResponse, error) { + filter := fmt.Sprintf("name==%s", name) + images, err := client.Prism.V3.ListAllImage(filter) + + if err != nil { + return nil, wrapNutanixError(err) + } + + if images == nil || images.Entities == nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("no image found for name==%s", name), + } + } + + for _, image := range images.Entities { + if *image.Status.Name == name { + return image, nil + } + } + + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("no image found for name==%s", name), + } +} + +func getVMByName(client *ClientSet, name string, projectID *string) (*nutanixv3.VMIntentResource, error) { + filter := fmt.Sprintf("vm_name==%s", name) + vms, err := client.Prism.V3.ListAllVM(filter) + + if err != nil { + return nil, wrapNutanixError(err) + } + + for _, vm := range vms.Entities { + if *vm.Status.Name == name { + if projectID != nil && *vm.Metadata.ProjectReference.UUID != *projectID { + continue + } + return vm, nil + } + } + + return nil, cloudprovidererrors.ErrInstanceNotFound +} + +func getIPs(client *ClientSet, vmID string, interval time.Duration, timeout time.Duration) (map[string]corev1.NodeAddressType, error) { + addresses := make(map[string]corev1.NodeAddressType) + + if err := wait.Poll(interval, timeout, func() (bool, error) { + vm, err := client.Prism.V3.GetVM(vmID) + if err != nil { + return false, wrapNutanixError(err) + } + + if len(vm.Status.Resources.NicList) == 0 || len(vm.Status.Resources.NicList[0].IPEndpointList) == 0 { + return false, nil + } + + ip := *vm.Status.Resources.NicList[0].IPEndpointList[0].IP + addresses[ip] = corev1.NodeInternalIP + + return true, nil + }); err != nil { + return map[string]corev1.NodeAddressType{}, err + } + + return addresses, nil +} + +func waitForCompletion(client *ClientSet, taskID string, interval time.Duration, timeout time.Duration) error { + return wait.Poll(interval, timeout, func() (bool, error) { + task, err := client.Prism.V3.GetTask(taskID) + if err != nil { + return false, wrapNutanixError(err) + } + + if task.Status == nil { + return false, nil + } + + switch *task.Status { + case "INVALID_UUID", "FAILED": + return false, fmt.Errorf("bad status: %s", *task.Status) + case "QUEUED", "RUNNING": + return false, nil + case "SUCCEEDED": + return true, nil + default: + return false, fmt.Errorf("unknown status: %s", *task.Status) + } + + }) +} + +func waitForPowerState(client *ClientSet, vmID string, interval time.Duration, timeout time.Duration) error { + return wait.Poll(interval, timeout, func() (bool, error) { + vm, err := client.Prism.V3.GetVM(vmID) + if err != nil { + return false, wrapNutanixError(err) + } + + if vm.Status == nil || vm.Status.Resources == nil || vm.Status.Resources.PowerState == nil { + return false, nil + } + + switch *vm.Status.Resources.PowerState { + case "ON": + return true, nil + case "OFF": + return false, nil + default: + return false, fmt.Errorf("unexpected power state: %s", *vm.Status.Resources.PowerState) + } + }) +} + +func wrapNutanixError(initialErr error) error { + if initialErr == nil { + return nil + } + + var resp nutanixtypes.ErrorResponse + + if err := json.Unmarshal([]byte(initialErr.Error()), &resp); err != nil { + // invalid credentials are returned with a simple string + if strings.Contains(initialErr.Error(), invalidCredentials) { + return cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: initialErr.Error(), + } + } + + // failed to parse error, let's make sure it doesnt't have new lines at least + return fmt.Errorf("api error: %s", strings.ReplaceAll(initialErr.Error(), "\n", "")) + } + + // TODO: handle different states by potentially returning a TerminalError + // this needs experience with errors coming from Nutanix because the state + // values are not defined anywhere. So if you hit an error that qualifies, + // why not add something handling it! + switch resp.State { + default: + var msgs []string + for _, msg := range resp.MessageList { + msgs = append(msgs, fmt.Sprintf("%s: %s", msg.Message, msg.Reason)) + } + + return fmt.Errorf("api error (%s, code %d): %s", resp.State, resp.Code, strings.Join(msgs, ", ")) + } +} diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go new file mode 100644 index 000000000..3a3c01a18 --- /dev/null +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -0,0 +1,423 @@ +/* +Copyright 2021 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nutanix + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "time" + + "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" + nutanixtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/nutanix/types" + cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "github.com/kubermatic/machine-controller/pkg/providerconfig" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + + corev1 "k8s.io/api/core/v1" + ktypes "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" +) + +type Config struct { + Endpoint string + Port *int + Username string + Password string + AllowInsecure bool + + ClusterName string + ProjectName string + SubnetName string + ImageName string + + Categories map[string]string + + CPUs int64 + CPUCores *int64 + CPUPassthrough *bool + MemoryMB int64 + DiskSizeGB *int64 +} + +type provider struct { + configVarResolver *providerconfig.ConfigVarResolver +} + +// Server holds Nutanix server information. +type Server struct { + name string + id string + status instance.Status + addresses map[string]corev1.NodeAddressType +} + +// Ensures that Server implements Instance interface. +var _ instance.Instance = &Server{} + +// Ensures that provider implements Provider interface. +var _ cloudprovidertypes.Provider = &provider{} + +func (nutanixServer Server) Name() string { + return nutanixServer.name +} + +func (nutanixServer Server) ID() string { + return nutanixServer.id +} + +func (nutanixServer Server) Addresses() map[string]corev1.NodeAddressType { + return nutanixServer.addresses +} + +func (nutanixServer Server) Status() instance.Status { + return nutanixServer.status +} + +// New returns a nutanix provider. +func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + provider := &provider{configVarResolver: configVarResolver} + return provider +} + +func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *nutanixtypes.RawConfig, error) { + if s.Value == nil { + return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") + } + pconfig := providerconfigtypes.Config{} + err := json.Unmarshal(s.Value.Raw, &pconfig) + if err != nil { + return nil, nil, nil, err + } + + if pconfig.OperatingSystemSpec.Raw == nil { + return nil, nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") + } + + rawConfig := nutanixtypes.RawConfig{} + err = json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawConfig) + if err != nil { + return nil, nil, nil, err + } + + c := Config{} + + c.Endpoint, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Endpoint, "NUTANIX_ENDPOINT") + if err != nil { + return nil, nil, nil, err + } + + port, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Port, "NUTANIX_PORT") + if err != nil { + return nil, nil, nil, err + } + + if port != "" { + // we parse the port into an int to make sure we're being passed a somewhat valid port value + portInt, err := strconv.Atoi(port) + if err != nil { + return nil, nil, nil, err + } + c.Port = pointer.Int(portInt) + } + + c.Username, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Username, "NUTANIX_USERNAME") + if err != nil { + return nil, nil, nil, err + } + + c.Password, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Password, "NUTANIX_PASSWORD") + if err != nil { + return nil, nil, nil, err + } + + c.AllowInsecure, err = p.configVarResolver.GetConfigVarBoolValueOrEnv(rawConfig.AllowInsecure, "NUTANIX_ALLOW_INSECURE") + if err != nil { + return nil, nil, nil, err + } + + c.ClusterName, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ClusterName, "NUTANIX_CLUSTER_NAME") + if err != nil { + return nil, nil, nil, err + } + + if rawConfig.ProjectName != nil { + c.ProjectName, err = p.configVarResolver.GetConfigVarStringValue(*rawConfig.ProjectName) + if err != nil { + return nil, nil, nil, err + } + } + + c.SubnetName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.SubnetName) + if err != nil { + return nil, nil, nil, err + } + + c.ImageName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ImageName) + if err != nil { + return nil, nil, nil, err + } + + c.Categories = rawConfig.Categories + + c.CPUs = rawConfig.CPUs + c.CPUCores = rawConfig.CPUCores + c.CPUPassthrough = rawConfig.CPUPassthrough + c.MemoryMB = rawConfig.MemoryMB + c.DiskSizeGB = rawConfig.DiskSize + + return &c, &pconfig, &rawConfig, nil +} + +func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { + return spec, nil +} + +func (p *provider) Validate(spec v1alpha1.MachineSpec) error { + config, _, _, err := p.getConfig(spec.ProviderSpec) + if err != nil { + return fmt.Errorf("failed to parse machineSpec: %v", err) + } + + client, err := GetClientSet(config) + if err != nil { + return fmt.Errorf("failed to construct client: %v", err) + } + + cluster, err := getClusterByName(client, config.ClusterName) + if err != nil { + return fmt.Errorf("failed to get cluster: %v", err) + } + + if config.ProjectName != "" { + if _, err := getProjectByName(client, config.ProjectName); err != nil { + return fmt.Errorf("failed to get project: %v", err) + } + } + + if _, err := getSubnetByName(client, config.SubnetName, *cluster.Metadata.UUID); err != nil { + return fmt.Errorf("failed to get subnet: %v", err) + } + + image, err := getImageByName(client, config.ImageName) + if err != nil { + return fmt.Errorf("failed to get image: %v", err) + } + + var imageSizeBytes int64 + + if image.Status != nil && image.Status.Resources.SizeBytes != nil { + imageSizeBytes = *image.Status.Resources.SizeBytes + } else { + return fmt.Errorf("failed to read image size for '%s'", config.ImageName) + } + + if config.DiskSizeGB != nil && *config.DiskSizeGB*1024*1024 < imageSizeBytes { + return fmt.Errorf("requested disk size (%d bytes) is smaller than image size (%d bytes)", *config.DiskSizeGB*1024*1024, *image.Status.Resources.SizeBytes) + } + + return nil +} + +func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { + vm, err := p.create(machine, userdata) + if err != nil { + _, cleanupErr := p.Cleanup(machine, data) + if cleanupErr != nil { + return nil, fmt.Errorf("cleaning up failed with err %v after creation failed with err %v", cleanupErr, err) + } + return nil, err + } + return vm, nil +} + +func (p *provider) create(machine *v1alpha1.Machine, userdata string) (instance.Instance, error) { + config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("failed to parse machineSpec: %v", err), + } + } + + client, err := GetClientSet(config) + if err != nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("failed to construct client: %v", err), + } + } + + return createVM(client, machine.Name, *config, pc.OperatingSystem, userdata) +} + +func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { + return p.cleanup(machine, data) +} + +func (p *provider) cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { + config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return false, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("failed to parse machineSpec: %v", err), + } + } + + client, err := GetClientSet(config) + if err != nil { + return false, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("failed to construct client: %v", err), + } + } + + var projectID *string + + if config.ProjectName != "" { + project, err := getProjectByName(client, config.ProjectName) + if err != nil { + return false, err + } + + projectID = project.Metadata.UUID + } + + vm, err := getVMByName(client, machine.Name, projectID) + if err != nil { + if err == cloudprovidererrors.ErrInstanceNotFound { + // VM is gone already + return true, nil + } + + return false, err + } + + // TODO: figure out if VM is already in deleting state + + resp, err := client.Prism.V3.DeleteVM(*vm.Metadata.UUID) + if err != nil { + return false, err + } + + taskID, ok := resp.Status.ExecutionContext.TaskUUID.(string) + if !ok { + return false, errors.New("failed to parse deletion task UUID") + } + + if err := waitForCompletion(client, taskID, time.Second*5, time.Minute*10); err != nil { + return false, fmt.Errorf("failed to wait for completion: %v", err) + } + + return true, nil +} + +func (p *provider) Get(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { + config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("failed to parse machineSpec: %v", err), + } + } + + client, err := GetClientSet(config) + if err != nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("failed to construct client: %v", err), + } + } + + var projectID *string + + if config.ProjectName != "" { + project, err := getProjectByName(client, config.ProjectName) + if err != nil { + return nil, err + } + + projectID = project.Metadata.UUID + } + + vm, err := getVMByName(client, machine.Name, projectID) + if err != nil { + return nil, err + } + + if vm.Status == nil || vm.Status.Resources == nil || vm.Status.Resources.PowerState == nil { + return nil, fmt.Errorf("could not read power state for VM '%s'", machine.Name) + } + + var status instance.Status + + switch *vm.Status.Resources.PowerState { + case "ON": + status = instance.StatusRunning + case "OFF": + status = instance.StatusCreating + default: + status = instance.StatusUnknown + } + + addresses := make(map[string]corev1.NodeAddressType) + + if len(vm.Status.Resources.NicList) > 0 && len(vm.Status.Resources.NicList[0].IPEndpointList) > 0 { + ip := *vm.Status.Resources.NicList[0].IPEndpointList[0].IP + addresses[ip] = corev1.NodeInternalIP + } else { + return nil, fmt.Errorf("could not find any IP addresses for VM '%s'", machine.Name) + } + + return Server{ + name: *vm.Status.Name, + id: *vm.Metadata.UUID, + status: status, + addresses: addresses, + }, nil +} + +func (p *provider) MigrateUID(machine *v1alpha1.Machine, new ktypes.UID) error { + return nil +} + +// GetCloudConfig returns an empty cloud configuration for Nutanix as no CCM exists +func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) { + return "", "", nil +} + +func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { + labels := make(map[string]string) + + config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return labels, fmt.Errorf("failed to parse config: %v", err) + } + + labels["size"] = fmt.Sprintf("%d-cpus-%d-mb", config.CPUs, config.MemoryMB) + labels["cluster"] = config.ClusterName + + return labels, nil +} + +func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { + return nil +} diff --git a/pkg/cloudprovider/provider/nutanix/types/types.go b/pkg/cloudprovider/provider/nutanix/types/types.go new file mode 100644 index 000000000..821d3f5bc --- /dev/null +++ b/pkg/cloudprovider/provider/nutanix/types/types.go @@ -0,0 +1,66 @@ +/* +Copyright 2021 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" +) + +const ( + VMKind = "vm" + ProjectKind = "project" + ClusterKind = "cluster" + SubnetKind = "subnet" + DiskKind = "disk" + ImageKind = "image" +) + +type RawConfig struct { + Endpoint providerconfigtypes.ConfigVarString `json:"endpoint"` + Port providerconfigtypes.ConfigVarString `json:"port"` + Username providerconfigtypes.ConfigVarString `json:"username"` + Password providerconfigtypes.ConfigVarString `json:"password"` + AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` + + ClusterName providerconfigtypes.ConfigVarString `json:"clusterName"` + ProjectName *providerconfigtypes.ConfigVarString `json:"projectName,omitempty"` + SubnetName providerconfigtypes.ConfigVarString `json:"subnetName"` + ImageName providerconfigtypes.ConfigVarString `json:"imageName"` + + // VM sizing configuration + CPUs int64 `json:"cpus"` + CPUCores *int64 `json:"cpuCores,omitempty"` + CPUPassthrough *bool `json:"cpuPassthrough,omitempty"` + MemoryMB int64 `json:"memoryMB"` + DiskSize *int64 `json:"diskSize,omitempty"` + + // Metadata related configuration + Categories map[string]string `json:"categories,omitempty"` +} + +type ErrorResponse struct { + APIVersion string `json:"api_version"` + Kind string `json:"kind"` + State string `json:"state"` + MessageList []ErrorResponseMsg `json:"message_list"` + Code int32 `json:"code"` +} + +type ErrorResponseMsg struct { + Message string `json:"message"` + Reason string `json:"reason"` +} diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 2bc79bd19..340e0a72b 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -53,6 +53,7 @@ const ( CloudProviderHetzner CloudProvider = "hetzner" CloudProviderKubeVirt CloudProvider = "kubevirt" CloudProviderLinode CloudProvider = "linode" + CloudProviderNutanix CloudProvider = "nutanix" CloudProviderOpenstack CloudProvider = "openstack" CloudProviderVsphere CloudProvider = "vsphere" CloudProviderFake CloudProvider = "fake" @@ -87,6 +88,7 @@ var ( CloudProviderHetzner, CloudProviderKubeVirt, CloudProviderLinode, + CloudProviderNutanix, CloudProviderOpenstack, CloudProviderVsphere, CloudProviderFake, From ee2d1ba74fcefc6bce486a284ae5064f783fcbaa Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 5 Jan 2022 22:34:44 +0500 Subject: [PATCH 046/489] Add SIG osm for overseeing changes for userdata and cloudprovider (#1146) * Add SIG osm for overseeing changes for userdata and cloudprovider Signed-off-by: Waleed Malik * Update label for sig-osm --- OWNERS_ALIASES | 6 ++++++ pkg/cloudprovider/provider/OWNERS | 13 +++++++++++++ pkg/userdata/OWNERS | 13 +++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 pkg/cloudprovider/provider/OWNERS create mode 100644 pkg/userdata/OWNERS diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 9c55374f6..23f2f754c 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -12,3 +12,9 @@ aliases: - xmudrii - xrstf - youssefazrak + + # Temporary SIG to oversee changes in userdata and cloudprovider sub-directories + # This SIG is responsible for ensuring that OSM and machine-controller are in sync + sig-osm: + - ahmedwaleedmalik + - moadqassem diff --git a/pkg/cloudprovider/provider/OWNERS b/pkg/cloudprovider/provider/OWNERS new file mode 100644 index 000000000..31bc1a729 --- /dev/null +++ b/pkg/cloudprovider/provider/OWNERS @@ -0,0 +1,13 @@ +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md + +approvers: + - sig-osm + +reviewers: + - sig-osm + +labels: + - sig/osm + +options: + no_parent_owners: true \ No newline at end of file diff --git a/pkg/userdata/OWNERS b/pkg/userdata/OWNERS new file mode 100644 index 000000000..31bc1a729 --- /dev/null +++ b/pkg/userdata/OWNERS @@ -0,0 +1,13 @@ +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md + +approvers: + - sig-osm + +reviewers: + - sig-osm + +labels: + - sig/osm + +options: + no_parent_owners: true \ No newline at end of file From cc1d43fd898c2b98f32cae415a6fe79d750a61d1 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Wed, 5 Jan 2022 20:05:36 +0100 Subject: [PATCH 047/489] Move spot instance e2e test to different AZ in eu-central-1 (#1150) * bump spot price to always be able to run e2e tests Signed-off-by: Marvin Beckers * Use a different AZ for spot instances Signed-off-by: Marvin Beckers --- test/e2e/provisioning/helper.go | 2 +- .../testdata/machinedeployment-aws-spot-instances.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index d8bdb5493..a3b709918 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -171,7 +171,7 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar scenarioParams = append(scenarioParams, fmt.Sprintf("<< RHEL_SUBSCRIPTION_MANAGER_USER >>=%s", "")) scenarioParams = append(scenarioParams, fmt.Sprintf("<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>=%s", "")) scenarioParams = append(scenarioParams, fmt.Sprintf("<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>=%s", "")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.02")) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.03")) } // only used by assume role scenario, otherwise empty (disabled) diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml index 0214fbd9f..258099e24 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml @@ -27,7 +27,7 @@ spec: accessKeyId: << AWS_ACCESS_KEY_ID >> secretAccessKey: << AWS_SECRET_ACCESS_KEY >> region: "eu-central-1" - availabilityZone: "eu-central-1a" + availabilityZone: "eu-central-1b" vpcId: "vpc-819f62e9" instanceType: "t2.medium" instanceProfile: "kubernetes-v1" From 590868abb2b86193c1e16af1fcddb03007867a61 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Thu, 6 Jan 2022 10:29:34 +0100 Subject: [PATCH 048/489] Fix disk size calculation in Nutanix spec validation (#1148) * Fix disk size calculation in Nutanix spec validation Signed-off-by: Marvin Beckers * Correctly calculate output for error message as well Signed-off-by: Marvin Beckers --- pkg/cloudprovider/provider/nutanix/provider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 3a3c01a18..ce7578db8 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -230,8 +230,8 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { return fmt.Errorf("failed to read image size for '%s'", config.ImageName) } - if config.DiskSizeGB != nil && *config.DiskSizeGB*1024*1024 < imageSizeBytes { - return fmt.Errorf("requested disk size (%d bytes) is smaller than image size (%d bytes)", *config.DiskSizeGB*1024*1024, *image.Status.Resources.SizeBytes) + if config.DiskSizeGB != nil && *config.DiskSizeGB*1024*1024*1024 < imageSizeBytes { + return fmt.Errorf("requested disk size (%d bytes) is smaller than image size (%d bytes)", *config.DiskSizeGB*1024*1024*1024, *image.Status.Resources.SizeBytes) } return nil From 31596ff8b9341957ca46cc56ae53f5f9a8c6d525 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Thu, 6 Jan 2022 11:25:36 +0100 Subject: [PATCH 049/489] Use default scheme and pass request context to ConfigVarResolver (#1147) * Add corev1 to scheme to access ConfigMaps and Secrets Signed-off-by: Marvin Beckers * Pass context to ConfigVarResolver that comes from corresponding http.Request Signed-off-by: Marvin Beckers * Use default scheme instead of a new one Signed-off-by: Marvin Beckers --- cmd/webhook/main.go | 9 +++------ pkg/admission/admission.go | 5 ++--- pkg/admission/machinedeployments.go | 5 +++-- pkg/admission/machines.go | 9 +++++---- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 5b893f62f..739f7d61a 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -25,7 +25,7 @@ import ( userdatamanager "github.com/kubermatic/machine-controller/pkg/userdata/manager" osmv1alpha1 "k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1" - "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/clientcmd" "k8s.io/klog" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -76,14 +76,11 @@ func main() { klog.Fatalf("error building kubeconfig: %v", err) } - scheme := runtime.NewScheme() - if err := osmv1alpha1.AddToScheme(scheme); err != nil { + if err := osmv1alpha1.AddToScheme(scheme.Scheme); err != nil { klog.Fatalf("failed to add osmv1alpha1 api to scheme: %v", err) } - client, err := ctrlruntimeclient.New(cfg, ctrlruntimeclient.Options{ - Scheme: scheme, - }) + client, err := ctrlruntimeclient.New(cfg, ctrlruntimeclient.Options{}) if err != nil { klog.Fatalf("failed to build client: %v", err) } diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go index d944c1ece..110c6ace6 100644 --- a/pkg/admission/admission.go +++ b/pkg/admission/admission.go @@ -41,7 +41,6 @@ import ( ) type admissionData struct { - ctx context.Context client ctrlruntimeclient.Client userDataManager *userdatamanager.Manager nodeSettings machinecontroller.NodeSettings @@ -125,7 +124,7 @@ func createAdmissionResponse(original, mutated runtime.Object) (*admissionv1.Adm return response, nil } -type mutator func(admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error) +type mutator func(context.Context, admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error) func handleFuncFactory(mutate mutator) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { @@ -143,7 +142,7 @@ func handleFuncFactory(mutate mutator) func(http.ResponseWriter, *http.Request) } // run the mutation logic - response, err := mutate(*review.Request) + response, err := mutate(r.Context(), *review.Request) if err != nil { response = &admissionv1.AdmissionResponse{} response.Result = &metav1.Status{Message: err.Error()} diff --git a/pkg/admission/machinedeployments.go b/pkg/admission/machinedeployments.go index 76a6dc81f..9cbb0d2e4 100644 --- a/pkg/admission/machinedeployments.go +++ b/pkg/admission/machinedeployments.go @@ -17,6 +17,7 @@ limitations under the License. package admission import ( + "context" "encoding/json" "fmt" @@ -27,7 +28,7 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" ) -func (ad *admissionData) mutateMachineDeployments(ar admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error) { +func (ad *admissionData) mutateMachineDeployments(ctx context.Context, ar admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error) { machineDeployment := clusterv1alpha1.MachineDeployment{} if err := json.Unmarshal(ar.Object.Raw, &machineDeployment); err != nil { return nil, fmt.Errorf("failed to unmarshal: %v", err) @@ -64,7 +65,7 @@ func (ad *admissionData) mutateMachineDeployments(ar admissionv1.AdmissionReques } if machineSpecNeedsValidation { - if err := ad.defaultAndValidateMachineSpec(&machineDeployment.Spec.Template.Spec); err != nil { + if err := ad.defaultAndValidateMachineSpec(ctx, &machineDeployment.Spec.Template.Spec); err != nil { return nil, err } } diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index cf9d4b7b7..fc8b4c539 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -17,6 +17,7 @@ limitations under the License. package admission import ( + "context" "encoding/json" "fmt" @@ -38,7 +39,7 @@ import ( // the `providerConfig` field to `providerSpec` const BypassSpecNoModificationRequirementAnnotation = "kubermatic.io/bypass-no-spec-mutation-requirement" -func (ad *admissionData) mutateMachines(ar admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error) { +func (ad *admissionData) mutateMachines(ctx context.Context, ar admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error) { machine := clusterv1alpha1.Machine{} if err := json.Unmarshal(ar.Object.Raw, &machine); err != nil { return nil, fmt.Errorf("failed to unmarshal: %v", err) @@ -80,7 +81,7 @@ func (ad *admissionData) mutateMachines(ar admissionv1.AdmissionRequest) (*admis // Default and verify .Spec on CREATE only, its expensive and not required to do it on UPDATE // as we disallow .Spec changes anyways if ar.Operation == admissionv1.Create { - if err := ad.defaultAndValidateMachineSpec(&machine.Spec); err != nil { + if err := ad.defaultAndValidateMachineSpec(ctx, &machine.Spec); err != nil { return nil, err } @@ -98,7 +99,7 @@ func (ad *admissionData) mutateMachines(ar admissionv1.AdmissionRequest) (*admis return createAdmissionResponse(machineOriginal, &machine) } -func (ad *admissionData) defaultAndValidateMachineSpec(spec *clusterv1alpha1.MachineSpec) error { +func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec *clusterv1alpha1.MachineSpec) error { providerConfig, err := providerconfigtypes.GetConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to read machine.spec.providerSpec: %v", err) @@ -112,7 +113,7 @@ func (ad *admissionData) defaultAndValidateMachineSpec(spec *clusterv1alpha1.Mac } } - skg := providerconfig.NewConfigVarResolver(ad.ctx, ad.client) + skg := providerconfig.NewConfigVarResolver(ctx, ad.client) prov, err := cloudprovider.ForProvider(providerConfig.CloudProvider, skg) if err != nil { return fmt.Errorf("failed to get cloud provider %q: %v", providerConfig.CloudProvider, err) From b208d1a236e486cd81acd83383be658b21b04fda Mon Sep 17 00:00:00 2001 From: Simon Bein Date: Thu, 6 Jan 2022 14:59:37 +0100 Subject: [PATCH 050/489] Use appropriate base64 flags (#1075) Adds detection whether gnu or bsd base64 package is available and selects the correct flag. This is necessary as MacOs (or BSD) uses the -b flag while gnu uses the -w flag. Unfortunately there is no flag for line-wrapping that both of them support Signed-off-by: Simon Bein --- Makefile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9cec18e19..0fa1477c1 100644 --- a/Makefile +++ b/Makefile @@ -38,6 +38,9 @@ IMAGE_NAME ?= $(REGISTRY)/$(REGISTRY_NAMESPACE)/machine-controller:$(IMAGE_TAG) OS = amzn2 centos ubuntu sles rhel flatcar USERDATA_BIN = $(patsubst %, machine-controller-userdata-%, $(OS)) +BASE64_ENC = \ + $(shell if base64 -w0 <(echo "") &> /dev/null; then echo "base64 -w0"; else echo "base64 -b0"; fi) + .PHONY: all all: build-machine-controller webhook @@ -147,9 +150,9 @@ clean-certs: .PHONY: deploy deploy: examples/admission-cert.pem @cat examples/machine-controller.yaml \ - |sed "s/__admission_ca_cert__/$(shell cat examples/ca-cert.pem|base64 -w0)/g" \ - |sed "s/__admission_cert__/$(shell cat examples/admission-cert.pem|base64 -w0)/g" \ - |sed "s/__admission_key__/$(shell cat examples/admission-key.pem|base64 -w0)/g" \ + |sed "s/__admission_ca_cert__/$(shell cat examples/ca-cert.pem|$(BASE64_ENC))/g" \ + |sed "s/__admission_cert__/$(shell cat examples/admission-cert.pem|$(BASE64_ENC))/g" \ + |sed "s/__admission_key__/$(shell cat examples/admission-key.pem|$(BASE64_ENC))/g" \ |kubectl apply -f - .PHONY: check-dependencies From d780011d879b87aa7fe2a2c8291ec9e0af142b2c Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 6 Jan 2022 20:58:32 +0500 Subject: [PATCH 051/489] Add mutating logic to populate OSP annotation based on OS flavor (#1149) * Add mutating logic to populate OSP annotation based on OS flavor Signed-off-by: Waleed Malik * Add comments for clarifying special case for amzn2 Signed-off-by: Waleed Malik --- pkg/admission/machinedeployments.go | 2 +- .../machinedeployments_validation.go | 36 ++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/pkg/admission/machinedeployments.go b/pkg/admission/machinedeployments.go index 9cbb0d2e4..541bacf0b 100644 --- a/pkg/admission/machinedeployments.go +++ b/pkg/admission/machinedeployments.go @@ -37,7 +37,7 @@ func (ad *admissionData) mutateMachineDeployments(ctx context.Context, ar admiss machineDeploymentDefaultingFunction(&machineDeployment) - if err := mutationsForMachineDeployment(&machineDeployment); err != nil { + if err := mutationsForMachineDeployment(&machineDeployment, ad.useOSM); err != nil { return nil, fmt.Errorf("mutation failed: %v", err) } diff --git a/pkg/admission/machinedeployments_validation.go b/pkg/admission/machinedeployments_validation.go index 73e6cfdb7..e04970b03 100644 --- a/pkg/admission/machinedeployments_validation.go +++ b/pkg/admission/machinedeployments_validation.go @@ -20,6 +20,8 @@ import ( "encoding/json" "fmt" + osmresources "k8c.io/operating-system-manager/pkg/controllers/osc/resources" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -32,6 +34,8 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" ) +const ospNamePattern = "osp-%s" + func validateMachineDeployment(md v1alpha1.MachineDeployment) field.ErrorList { allErrs := field.ErrorList{} allErrs = append(allErrs, validateMachineDeploymentSpec(&md.Spec, field.NewPath("spec"))...) @@ -114,12 +118,19 @@ func machineDeploymentDefaultingFunction(md *v1alpha1.MachineDeployment) { v1alpha1.PopulateDefaultsMachineDeployment(md) } -func mutationsForMachineDeployment(md *v1alpha1.MachineDeployment) error { +func mutationsForMachineDeployment(md *v1alpha1.MachineDeployment, useOSM bool) error { providerConfig, err := providerconfigtypes.GetConfig(md.Spec.Template.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to read MachineDeployment.Spec.Template.Spec.ProviderSpec: %v", err) } + if useOSM { + err = ensureOSPAnnotation(md, *providerConfig) + if err != nil { + return err + } + } + // Packet has been renamed to Equinix Metal if providerConfig.CloudProvider == cloudProviderPacket { err = migrateToEquinixMetal(providerConfig) @@ -136,3 +147,26 @@ func mutationsForMachineDeployment(md *v1alpha1.MachineDeployment) error { return nil } + +func ensureOSPAnnotation(md *v1alpha1.MachineDeployment, providerConfig providerconfigtypes.Config) error { + // Check for existing annotation + if _, ok := md.Annotations[osmresources.MachineDeploymentOSPAnnotation]; !ok { + if md.Annotations == nil { + md.Annotations = make(map[string]string) + } + // Annotation not specified, populate default OSP annotation + switch providerConfig.OperatingSystem { + case providerconfigtypes.OperatingSystemUbuntu, providerconfigtypes.OperatingSystemCentOS, providerconfigtypes.OperatingSystemFlatcar, + providerconfigtypes.OperatingSystemRHEL, providerconfigtypes.OperatingSystemSLES: + md.Annotations[osmresources.MachineDeploymentOSPAnnotation] = fmt.Sprintf(ospNamePattern, providerConfig.OperatingSystem) + return nil + case providerconfigtypes.OperatingSystemAmazonLinux2: + // This is a special case where the OS name suffix in OSP is different then the actual OS name + md.Annotations[osmresources.MachineDeploymentOSPAnnotation] = fmt.Sprintf(ospNamePattern, "amazon-linux") + return nil + default: + return fmt.Errorf("failed to populate OSP annotation for machinedeployment with unsupported Operating System %s", providerConfig.OperatingSystem) + } + } + return nil +} From a2bf73322a0f9c19eecc184e455960cf67c61935 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Thu, 6 Jan 2022 17:46:34 +0100 Subject: [PATCH 052/489] update cri defaulting (#1140) * update cri defaulting Signed-off-by: Moath Qasim * Update fixtures Signed-off-by: Waleed Malik Co-authored-by: Waleed Malik --- pkg/containerruntime/containerruntime.go | 4 +- .../amzn2/testdata/kubelet-v1.22-aws.yaml | 46 +++++----------- .../centos/testdata/kubelet-v1.22-aws.yaml | 46 ++++++---------- .../flatcar/testdata/cloud-init_v1.22.2.yaml | 41 +++----------- .../flatcar/testdata/ignition_v1.22.2.json | 2 +- .../testdata/kubelet-v1.22-aws-external.yaml | 46 ++++++---------- .../rhel/testdata/kubelet-v1.22-aws.yaml | 46 ++++++---------- .../kubelet-v1.22-vsphere-mirrors.yaml | 46 ++++++---------- .../testdata/kubelet-v1.22-vsphere-proxy.yaml | 53 ++++++------------- .../rhel/testdata/kubelet-v1.22-vsphere.yaml | 46 ++++++---------- .../ubuntu/testdata/version-1.22.2.yaml | 45 +++++----------- 11 files changed, 127 insertions(+), 294 deletions(-) diff --git a/pkg/containerruntime/containerruntime.go b/pkg/containerruntime/containerruntime.go index dccc48eec..a8d1a3da3 100644 --- a/pkg/containerruntime/containerruntime.go +++ b/pkg/containerruntime/containerruntime.go @@ -108,10 +108,10 @@ func (cfg Config) Engine(kubeletVersion *semver.Version) Engine { sandboxImage: cfg.SandboxImage, } - moreThan122, _ := semver.NewConstraint(">= 1.22") + moreThan124, _ := semver.NewConstraint(">= 1.24") switch { - case moreThan122.Check(kubeletVersion): + case moreThan124.Check(kubeletVersion): return containerd case cfg.Docker != nil: return docker diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml index a5b6e8766..77ab23022 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml @@ -81,25 +81,22 @@ write_files: ipvsadm - mkdir -p /etc/systemd/system/containerd.service.d + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - cat < Date: Thu, 6 Jan 2022 19:07:33 +0100 Subject: [PATCH 053/489] Add support for proxy URL and use client that doesn't leak URL to logs (#1152) Signed-off-by: Marvin Beckers --- go.mod | 2 +- go.sum | 4 ++-- pkg/cloudprovider/provider/nutanix/client.go | 4 ++++ pkg/cloudprovider/provider/nutanix/provider.go | 6 ++++++ pkg/cloudprovider/provider/nutanix/types/types.go | 1 + 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 4a1f7806a..185e024cd 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/coreos/container-linux-config-transpiler v0.9.0 github.com/davecgh/go-spew v1.1.1 github.com/digitalocean/godo v1.54.0 - github.com/embik/nutanix-client-go v0.0.0-20220103122158-dbb64d7901ab + github.com/embik/nutanix-client-go v0.0.0-20220106131900-50b8f27e5f60 github.com/ghodss/yaml v1.0.0 github.com/go-test/deep v1.0.7 github.com/google/uuid v1.1.2 diff --git a/go.sum b/go.sum index 2bdb6b1c8..70df54ce8 100644 --- a/go.sum +++ b/go.sum @@ -332,8 +332,8 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/embik/nutanix-client-go v0.0.0-20220103122158-dbb64d7901ab h1:WrwH567h9IFvCcAFORWFl+jOZqfvQc47LfnxN0nQzIk= -github.com/embik/nutanix-client-go v0.0.0-20220103122158-dbb64d7901ab/go.mod h1:gkKNSxfEt3QtYG3S/wKiN8OmrJ4fpU7JbTlbnrMDOL8= +github.com/embik/nutanix-client-go v0.0.0-20220106131900-50b8f27e5f60 h1:0FVKOkpksULFs6F7Kfd8ClBXVTvtiIKl07uV3HinOHk= +github.com/embik/nutanix-client-go v0.0.0-20220106131900-50b8f27e5f60/go.mod h1:gkKNSxfEt3QtYG3S/wKiN8OmrJ4fpU7JbTlbnrMDOL8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.10.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= diff --git a/pkg/cloudprovider/provider/nutanix/client.go b/pkg/cloudprovider/provider/nutanix/client.go index 2dd3b88a7..760dab46b 100644 --- a/pkg/cloudprovider/provider/nutanix/client.go +++ b/pkg/cloudprovider/provider/nutanix/client.go @@ -78,6 +78,10 @@ func GetClientSet(config *Config) (*ClientSet, error) { Insecure: config.AllowInsecure, } + if config.ProxyURL != "" { + credentials.ProxyURL = config.ProxyURL + } + clientV3, err := nutanixv3.NewV3Client(credentials) if err != nil { return nil, err diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index ce7578db8..8b02aa204 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -43,6 +43,7 @@ type Config struct { Username string Password string AllowInsecure bool + ProxyURL string ClusterName string ProjectName string @@ -154,6 +155,11 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return nil, nil, nil, err } + c.ProxyURL, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProxyURL, "NUTANIX_PROXY_URL") + if err != nil { + return nil, nil, nil, err + } + c.ClusterName, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ClusterName, "NUTANIX_CLUSTER_NAME") if err != nil { return nil, nil, nil, err diff --git a/pkg/cloudprovider/provider/nutanix/types/types.go b/pkg/cloudprovider/provider/nutanix/types/types.go index 821d3f5bc..59afc898d 100644 --- a/pkg/cloudprovider/provider/nutanix/types/types.go +++ b/pkg/cloudprovider/provider/nutanix/types/types.go @@ -35,6 +35,7 @@ type RawConfig struct { Username providerconfigtypes.ConfigVarString `json:"username"` Password providerconfigtypes.ConfigVarString `json:"password"` AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` + ProxyURL providerconfigtypes.ConfigVarString `json:"proxyURL,omitempty"` ClusterName providerconfigtypes.ConfigVarString `json:"clusterName"` ProjectName *providerconfigtypes.ConfigVarString `json:"projectName,omitempty"` From 9dba8cee0a86b23396ee89052a916a17703287d7 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Fri, 7 Jan 2022 11:17:36 +0200 Subject: [PATCH 054/489] Containerd daemon level registry auth configuration (#1144) * Resolve containerd registry authentication from the Secret Signed-off-by: Artiom Diomin * Review fixes Signed-off-by: Artiom Diomin --- .gimps.yaml | 6 +- cmd/machine-controller/main.go | 40 +++-- docs/registry-authentication.md | 47 ++++++ .../machinedeployments_validation.go | 3 +- pkg/containerruntime/containerd.go | 143 +++++++++++++++++- pkg/containerruntime/containerruntime.go | 24 ++- pkg/controller/machine/machine_controller.go | 27 +++- pkg/userdata/helper/helper.go | 108 ------------- pkg/userdata/helper/kubelet.go | 1 + pkg/userdata/ubuntu/provider_test.go | 12 ++ pkg/userdata/ubuntu/testdata/containerd.yaml | 10 ++ 11 files changed, 277 insertions(+), 144 deletions(-) create mode 100644 docs/registry-authentication.md diff --git a/.gimps.yaml b/.gimps.yaml index 2e7ba0d4f..7856ca9c5 100644 --- a/.gimps.yaml +++ b/.gimps.yaml @@ -14,8 +14,12 @@ # This is the configuration for https://github.com/xrstf/gimps. -importOrder: [std, external, project, kubernetes] +importOrder: [std, external, kubermatic, kubernetes] sets: + - name: kubermatic + patterns: + - 'k8c.io/**' + - 'github.com/kubermatic/**' - name: kubernetes patterns: - 'k8s.io/**' diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 455c25b03..e111dc3ce 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -28,7 +28,6 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - osmv1alpha1 "k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1/migrations" @@ -45,6 +44,8 @@ import ( "github.com/kubermatic/machine-controller/pkg/node" "github.com/kubermatic/machine-controller/pkg/signals" + osmv1alpha1 "k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" @@ -69,20 +70,20 @@ var ( workerCount int bootstrapTokenServiceAccountName string skipEvictionAfter time.Duration - nodeCSRApprover bool caBundleFile string useOSM bool - nodeHTTPProxy string - nodeNoProxy string - nodeInsecureRegistries string - nodeRegistryMirrors string - nodePauseImage string - nodeContainerRuntime string - podCidr string - nodePortRange string - + nodeCSRApprover bool + nodeHTTPProxy string + nodeNoProxy string + nodeInsecureRegistries string + nodeRegistryMirrors string + nodePauseImage string + nodeContainerRuntime string + podCidr string + nodePortRange string + nodeRegistryCredentialsSecret string nodeContainerdRegistryMirrors = registryMirrorsFlags{} ) @@ -169,7 +170,7 @@ func main() { flag.BoolVar(&nodeCSRApprover, "node-csr-approver", true, "Enable NodeCSRApprover controller to automatically approve node serving certificate requests") flag.StringVar(&podCidr, "pod-cidr", "172.25.0.0/16", "The network ranges from which POD networks are allocated") flag.StringVar(&nodePortRange, "node-port-range", "30000-32767", "A port range to reserve for services with NodePort visibility") - + flag.StringVar(&nodeRegistryCredentialsSecret, "node-registry-credentials-secret", "", "A Secret object reference, that containt auth info for image registry in namespace/secret-name form, example: kube-system/registry-credentials. See doc at https://github.com/kubermaric/machine-controller/blob/master/docs/registry-authentication.md") flag.BoolVar(&useOSM, "use-osm", false, "use osm controller for node bootstrap") flag.Parse() @@ -263,6 +264,12 @@ func main() { nodeContainerdRegistryMirrors["docker.io"] = registryMirrors } + if nodeRegistryCredentialsSecret != "" { + if secRef := strings.Split(nodeRegistryCredentialsSecret, "/"); len(secRef) != 2 { + klog.Fatalf("-node-registry-credentials-secret is in incorrect format %q, should be in 'namespace/secretname'", nodeRegistryCredentialsSecret) + } + } + runOptions := controllerRunOptions{ kubeClient: kubeClient, kubeconfigProvider: kubeconfigProvider, @@ -273,10 +280,11 @@ func main() { skipEvictionAfter: skipEvictionAfter, nodeCSRApprover: nodeCSRApprover, node: machinecontroller.NodeSettings{ - ClusterDNSIPs: clusterDNSIPs, - HTTPProxy: nodeHTTPProxy, - NoProxy: nodeNoProxy, - PauseImage: nodePauseImage, + ClusterDNSIPs: clusterDNSIPs, + HTTPProxy: nodeHTTPProxy, + NoProxy: nodeNoProxy, + PauseImage: nodePauseImage, + RegistryCredentialsSecretRef: nodeRegistryCredentialsSecret, ContainerRuntime: containerruntime.Get( nodeContainerRuntime, containerruntime.WithInsecureRegistries(insecureRegistries), diff --git a/docs/registry-authentication.md b/docs/registry-authentication.md new file mode 100644 index 000000000..c96d5f293 --- /dev/null +++ b/docs/registry-authentication.md @@ -0,0 +1,47 @@ +# Registry Authentication + +Machine-controller supports configuring container runtime with authentication +information. Flag `-node-registry-credentials-secret` can take a secret +reference in form `namespace/secret-name` where authentication info will be +stored. During the VM creation this info will be used to configure container +runtime. + +Secret format is serialized +`map[string]github.com/containerd/containerd/pkg/cri/config.AuthConfig`, where +`AuthConfig` is defined as + +```go +type AuthConfig struct { + // Username is the username to login the registry. + Username string `toml:"username" json:"username"` + // Password is the password to login the registry. + Password string `toml:"password" json:"password"` + // Auth is a base64 encoded string from the concatenation of the username, + // a colon, and the password. + Auth string `toml:"auth" json:"auth"` + // IdentityToken is used to authenticate the user and get + // an access token for the registry. + IdentityToken string `toml:"identitytoken" json:"identitytoken"` +} +``` + +Original source: https://github.com/containerd/containerd/blob/v1.5.9/pkg/cri/config/config.go#L126-L137 + + +Example: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: my-registries + namespace: kube-system +data: + gcr.io: | + eyJ1c2VybmFtZSI6ImwwZzFuIiwicGFzc3dvcmQiOiJjMDBscDQ1NXc + wcmQiLCJhdXRoIjoiIiwiaWRlbnRpdHl0b2tlbiI6IiJ9Cg== + +``` + +Now having this saved in the Kubernetes API, launch machine-controller with +`-node-registry-credentials-secret=kube-system/my-registries` flag. diff --git a/pkg/admission/machinedeployments_validation.go b/pkg/admission/machinedeployments_validation.go index e04970b03..7a32952fb 100644 --- a/pkg/admission/machinedeployments_validation.go +++ b/pkg/admission/machinedeployments_validation.go @@ -20,11 +20,10 @@ import ( "encoding/json" "fmt" - osmresources "k8c.io/operating-system-manager/pkg/controllers/osc/resources" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + osmresources "k8c.io/operating-system-manager/pkg/controllers/osc/resources" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index 7a84a65f4..749ccd041 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -21,8 +21,9 @@ import ( "strings" "text/template" + "github.com/BurntSushi/toml" + "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - "github.com/kubermatic/machine-controller/pkg/userdata/helper" ) const ( @@ -30,13 +31,10 @@ const ( ) type Containerd struct { - insecureRegistries []string - registryMirrors map[string][]string - sandboxImage string -} - -func (eng *Containerd) Config() (string, error) { - return helper.ContainerdConfig(eng.insecureRegistries, eng.registryMirrors, eng.sandboxImage) + insecureRegistries []string + registryMirrors map[string][]string + sandboxImage string + registryCredentials map[string]AuthConfig } func (eng *Containerd) ConfigFileName() string { @@ -172,3 +170,132 @@ systemctl enable --now containerd func (eng *Containerd) String() string { return containerdName } + +type containerdConfigManifest struct { + Version int `toml:"version"` + Metrics *containerdMetrics `toml:"metrics"` + Plugins map[string]interface{} `toml:"plugins"` +} + +type containerdMetrics struct { + Address string `toml:"address"` +} + +type containerdCRIPlugin struct { + Containerd *containerdCRISettings `toml:"containerd"` + Registry *containerdCRIRegistry `toml:"registry"` + SandboxImage string `toml:"sandbox_image,omitempty"` +} + +type containerdCRISettings struct { + Runtimes map[string]containerdCRIRuntime `toml:"runtimes"` +} + +type containerdCRIRuntime struct { + RuntimeType string `toml:"runtime_type"` + Options interface{} `toml:"options"` +} + +type containerdCRIRuncOptions struct { + SystemdCgroup bool +} + +type containerdCRIRegistry struct { + Mirrors map[string]containerdRegistryMirror `toml:"mirrors"` + Configs map[string]containerdRegistryConfig `toml:"configs"` +} + +type containerdRegistryMirror struct { + Endpoint []string `toml:"endpoint"` +} + +type containerdRegistryConfig struct { + TLS *containerdRegistryTLSConfig `toml:"tls"` + Auth *AuthConfig `toml:"auth"` +} + +type containerdRegistryTLSConfig struct { + InsecureSkipVerify bool `toml:"insecure_skip_verify"` +} + +// AuthConfig is a COPY of github.com/containerd/containerd/pkg/cri/config.AuthConfig. +// AuthConfig contains the config related to authentication to a specific registry +type AuthConfig struct { + // Username is the username to login the registry. + Username string `toml:"username,omitempty" json:"username,omitempty"` + // Password is the password to login the registry. + Password string `toml:"password,omitempty" json:"password,omitempty"` + // Auth is a base64 encoded string from the concatenation of the username, + // a colon, and the password. + Auth string `toml:"auth,omitempty" json:"auth,omitempty"` + // IdentityToken is used to authenticate the user and get + // an access token for the registry. + IdentityToken string `toml:"identitytoken,omitempty" json:"identitytoken,omitempty"` +} + +func (eng *Containerd) Config() (string, error) { + criPlugin := containerdCRIPlugin{ + SandboxImage: eng.sandboxImage, + Containerd: &containerdCRISettings{ + Runtimes: map[string]containerdCRIRuntime{ + "runc": { + RuntimeType: "io.containerd.runc.v2", + Options: containerdCRIRuncOptions{ + SystemdCgroup: true, + }, + }, + }, + }, + Registry: &containerdCRIRegistry{ + Mirrors: map[string]containerdRegistryMirror{ + "docker.io": { + Endpoint: []string{"https://registry-1.docker.io/"}, + }, + }, + }, + } + + for registryName := range eng.registryMirrors { + registry := criPlugin.Registry.Mirrors[registryName] + registry.Endpoint = eng.registryMirrors[registryName] + criPlugin.Registry.Mirrors[registryName] = registry + } + + if len(eng.insecureRegistries) != 0 || len(eng.registryCredentials) != 0 { + criPlugin.Registry.Configs = map[string]containerdRegistryConfig{} + } + + for _, registry := range eng.insecureRegistries { + criPlugin.Registry.Configs[registry] = containerdRegistryConfig{ + TLS: &containerdRegistryTLSConfig{ + InsecureSkipVerify: true, + }, + } + } + + for registry, auth := range eng.registryCredentials { + regConfig := criPlugin.Registry.Configs[registry] + auth := auth + regConfig.Auth = &auth + criPlugin.Registry.Configs[registry] = regConfig + } + + cfg := containerdConfigManifest{ + Version: 2, + Metrics: &containerdMetrics{ + // metrics available at http://127.0.0.1:1338/v1/metrics + Address: "127.0.0.1:1338", + }, + + Plugins: map[string]interface{}{ + "io.containerd.grpc.v1.cri": criPlugin, + }, + } + + var buf strings.Builder + enc := toml.NewEncoder(&buf) + enc.Indent = "" + err := enc.Encode(cfg) + + return buf.String(), err +} diff --git a/pkg/containerruntime/containerruntime.go b/pkg/containerruntime/containerruntime.go index a8d1a3da3..05edcfee3 100644 --- a/pkg/containerruntime/containerruntime.go +++ b/pkg/containerruntime/containerruntime.go @@ -49,6 +49,12 @@ func WithRegistryMirrors(mirrors map[string][]string) Opt { } } +func WithRegistryCredentials(auth map[string]AuthConfig) Opt { + return func(cfg *Config) { + cfg.RegistryCredentials = auth + } +} + func WithSandboxImage(image string) Opt { return func(cfg *Config) { cfg.SandboxImage = image @@ -78,11 +84,12 @@ func Get(containerRuntimeName string, opts ...Opt) Config { } type Config struct { - Docker *Docker `json:",omitempty"` - Containerd *Containerd `json:",omitempty"` - InsecureRegistries []string `json:",omitempty"` - RegistryMirrors map[string][]string `json:",omitempty"` - SandboxImage string `json:",omitempty"` + Docker *Docker `json:",omitempty"` + Containerd *Containerd `json:",omitempty"` + InsecureRegistries []string `json:",omitempty"` + RegistryMirrors map[string][]string `json:",omitempty"` + RegistryCredentials map[string]AuthConfig `json:",omitempty"` + SandboxImage string `json:",omitempty"` } func (cfg Config) String() string { @@ -103,9 +110,10 @@ func (cfg Config) Engine(kubeletVersion *semver.Version) Engine { } containerd := &Containerd{ - insecureRegistries: cfg.InsecureRegistries, - registryMirrors: cfg.RegistryMirrors, - sandboxImage: cfg.SandboxImage, + insecureRegistries: cfg.InsecureRegistries, + registryMirrors: cfg.RegistryMirrors, + sandboxImage: cfg.SandboxImage, + registryCredentials: cfg.RegistryCredentials, } moreThan124, _ := semver.NewConstraint(">= 1.24") diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 533b6d222..b014f3c9c 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -18,6 +18,7 @@ package controller import ( "context" + "encoding/json" "errors" "fmt" "net" @@ -140,6 +141,8 @@ type NodeSettings struct { ExternalCloudProvider bool // container runtime to install ContainerRuntime containerruntime.Config + // Registry credentials secret object reference + RegistryCredentialsSecretRef string } type KubeconfigProvider interface { @@ -721,6 +724,27 @@ func (r *Reconciler) ensureInstanceExistsForMachine( externalCloudProvider, _ = strconv.ParseBool(val) } + registryCredentials := map[string]containerruntime.AuthConfig{} + + if secRef := strings.SplitN(r.nodeSettings.RegistryCredentialsSecretRef, "/", 2); len(secRef) == 2 { + var credsSecret corev1.Secret + err := r.client.Get(ctx, types.NamespacedName{Namespace: secRef[0], Name: secRef[1]}, &credsSecret) + if err != nil { + return nil, fmt.Errorf("failed to retrieve registry credentials secret object: %w", err) + } + + for registry, data := range credsSecret.Data { + var regCred containerruntime.AuthConfig + if err := json.Unmarshal(data, ®Cred); err != nil { + return nil, fmt.Errorf("failed to unmarshal registry credentials: %w", err) + } + registryCredentials[registry] = regCred + } + } + + crRuntime := r.nodeSettings.ContainerRuntime + crRuntime.RegistryCredentials = registryCredentials + req := plugin.UserDataRequest{ MachineSpec: machine.Spec, Kubeconfig: kubeconfig, @@ -733,10 +757,11 @@ func (r *Reconciler) ensureInstanceExistsForMachine( KubeletConfigs: KubeletConfigs, NoProxy: r.nodeSettings.NoProxy, HTTPProxy: r.nodeSettings.HTTPProxy, - ContainerRuntime: r.nodeSettings.ContainerRuntime, + ContainerRuntime: crRuntime, PodCIDR: r.podCIDR, NodePortRange: r.nodePortRange, } + // Here we do stuff! var userdata string diff --git a/pkg/userdata/helper/helper.go b/pkg/userdata/helper/helper.go index ea6798fde..5d22d4c81 100644 --- a/pkg/userdata/helper/helper.go +++ b/pkg/userdata/helper/helper.go @@ -21,8 +21,6 @@ import ( "fmt" "strings" - "github.com/BurntSushi/toml" - "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" ) @@ -103,112 +101,6 @@ SystemMaxUse=5G ` } -type containerdConfigManifest struct { - Version int `toml:"version"` - Metrics *containerdMetrics `toml:"metrics"` - Plugins map[string]interface{} `toml:"plugins"` -} - -type containerdMetrics struct { - Address string `toml:"address"` -} - -type containerdCRIPlugin struct { - Containerd *containerdCRISettings `toml:"containerd"` - Registry *containerdCRIRegistry `toml:"registry"` - SandboxImage string `toml:"sandbox_image,omitempty"` -} - -type containerdCRISettings struct { - Runtimes map[string]containerdCRIRuntime `toml:"runtimes"` -} - -type containerdCRIRuntime struct { - RuntimeType string `toml:"runtime_type"` - Options interface{} `toml:"options"` -} - -type containerdCRIRuncOptions struct { - SystemdCgroup bool -} - -type containerdCRIRegistry struct { - Mirrors map[string]containerdRegistryMirror `toml:"mirrors"` - Configs map[string]containerdRegistryConfig `toml:"configs"` -} - -type containerdRegistryMirror struct { - Endpoint []string `toml:"endpoint"` -} - -type containerdRegistryConfig struct { - TLS *containerdRegistryTLSConfig `toml:"tls"` -} - -type containerdRegistryTLSConfig struct { - InsecureSkipVerify bool `toml:"insecure_skip_verify"` -} - -func ContainerdConfig(insecureRegistries []string, registryMirrors map[string][]string, sandboxImage string) (string, error) { - criPlugin := containerdCRIPlugin{ - SandboxImage: sandboxImage, - Containerd: &containerdCRISettings{ - Runtimes: map[string]containerdCRIRuntime{ - "runc": { - RuntimeType: "io.containerd.runc.v2", - Options: containerdCRIRuncOptions{ - SystemdCgroup: true, - }, - }, - }, - }, - Registry: &containerdCRIRegistry{ - Mirrors: map[string]containerdRegistryMirror{ - "docker.io": { - Endpoint: []string{"https://registry-1.docker.io/"}, - }, - }, - }, - } - - for registryName := range registryMirrors { - registry := criPlugin.Registry.Mirrors[registryName] - registry.Endpoint = registryMirrors[registryName] - criPlugin.Registry.Mirrors[registryName] = registry - } - - if len(insecureRegistries) > 0 { - criPlugin.Registry.Configs = map[string]containerdRegistryConfig{} - } - - for _, registry := range insecureRegistries { - criPlugin.Registry.Configs[registry] = containerdRegistryConfig{ - TLS: &containerdRegistryTLSConfig{ - InsecureSkipVerify: true, - }, - } - } - - cfg := containerdConfigManifest{ - Version: 2, - Metrics: &containerdMetrics{ - // metrics available at http://127.0.0.1:1338/v1/metrics - Address: "127.0.0.1:1338", - }, - - Plugins: map[string]interface{}{ - "io.containerd.grpc.v1.cri": criPlugin, - }, - } - - var buf strings.Builder - enc := toml.NewEncoder(&buf) - enc.Indent = "" - err := enc.Encode(cfg) - - return buf.String(), err -} - type dockerConfig struct { ExecOpts []string `json:"exec-opts,omitempty"` StorageDriver string `json:"storage-driver,omitempty"` diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index f2e41a3dd..d243acdf4 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -23,6 +23,7 @@ import ( "text/template" "github.com/Masterminds/semver/v3" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" corev1 "k8s.io/api/core/v1" diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index e5dedb836..6bc781b42 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -119,6 +119,7 @@ type userDataTestCase struct { noProxy string insecureRegistries []string registryMirrors map[string][]string + registryCredentials map[string]containerruntime.AuthConfig pauseImage string containerruntime string } @@ -410,6 +411,16 @@ func TestUserDataGeneration(t *testing.T) { { name: "containerd", containerruntime: "containerd", + registryCredentials: map[string]containerruntime.AuthConfig{ + "docker.io": { + Username: "login1", + Password: "passwd1", + }, + }, + insecureRegistries: []string{"k8s.gcr.io"}, + registryMirrors: map[string][]string{ + "k8s.gcr.io": {"https://intranet.local/"}, + }, providerSpec: &providerconfigtypes.Config{ CloudProvider: "", SSHPublicKeys: []string{"ssh-rsa AAABBB"}, @@ -477,6 +488,7 @@ func TestUserDataGeneration(t *testing.T) { test.containerruntime, containerruntime.WithInsecureRegistries(test.insecureRegistries), containerruntime.WithRegistryMirrors(test.registryMirrors), + containerruntime.WithRegistryCredentials(test.registryCredentials), ), } s, err := provider.UserData(req) diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index 6e65d14e6..4011665c1 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -364,6 +364,16 @@ write_files: [plugins."io.containerd.grpc.v1.cri".registry.mirrors] [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] endpoint = ["https://registry-1.docker.io/"] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"] + endpoint = ["https://intranet.local/"] + [plugins."io.containerd.grpc.v1.cri".registry.configs] + [plugins."io.containerd.grpc.v1.cri".registry.configs."docker.io"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."docker.io".auth] + username = "login1" + password = "passwd1" + [plugins."io.containerd.grpc.v1.cri".registry.configs."k8s.gcr.io"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."k8s.gcr.io".tls] + insecure_skip_verify = true - path: "/etc/kubernetes/kubelet.conf" From 4f5f591971751cfe8fd847cd9e692743a8dffd74 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 7 Jan 2022 15:32:34 +0500 Subject: [PATCH 055/489] Split clients into worker and seed(default) clients for webhooks (#1153) * Split clients into worker and seed(default) clients for webhooks Signed-off-by: Waleed Malik * Handle PR feedback --- cmd/webhook/main.go | 82 ++++++++++++++++++++++++-------------- pkg/admission/admission.go | 3 ++ pkg/admission/machines.go | 2 +- 3 files changed, 56 insertions(+), 31 deletions(-) diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 739f7d61a..871b8e897 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -31,71 +31,93 @@ import ( ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) -var ( - masterURL string - kubeconfig string - admissionListenAddress string - admissionTLSCertPath string - admissionTLSKeyPath string - caBundleFile string - useOSM bool - namespace string -) +type options struct { + masterURL string + kubeconfig string + admissionListenAddress string + admissionTLSCertPath string + admissionTLSKeyPath string + caBundleFile string + useOSM bool + namespace string + workerClusterKubeconfig string +} func main() { nodeFlags := node.NewFlags(flag.CommandLine) + opt := &options{} klog.InitFlags(nil) if flag.Lookup("kubeconfig") == nil { - flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") + flag.StringVar(&opt.kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") } if flag.Lookup("master") == nil { - flag.StringVar(&masterURL, "master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.") + flag.StringVar(&opt.masterURL, "master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.") } - flag.StringVar(&admissionListenAddress, "listen-address", ":9876", "The address on which the MutatingWebhook will listen on") - flag.StringVar(&admissionTLSCertPath, "tls-cert-path", "/tmp/cert/cert.pem", "The path of the TLS cert for the MutatingWebhook") - flag.StringVar(&admissionTLSKeyPath, "tls-key-path", "/tmp/cert/key.pem", "The path of the TLS key for the MutatingWebhook") - flag.StringVar(&caBundleFile, "ca-bundle", "", "path to a file containing all PEM-encoded CA certificates (will be used instead of the host's certificates if set)") - flag.StringVar(&namespace, "namespace", "kubermatic", "The namespace where the webhooks will run") + flag.StringVar(&opt.admissionListenAddress, "listen-address", ":9876", "The address on which the MutatingWebhook will listen on") + flag.StringVar(&opt.admissionTLSCertPath, "tls-cert-path", "/tmp/cert/cert.pem", "The path of the TLS cert for the MutatingWebhook") + flag.StringVar(&opt.admissionTLSKeyPath, "tls-key-path", "/tmp/cert/key.pem", "The path of the TLS key for the MutatingWebhook") + flag.StringVar(&opt.caBundleFile, "ca-bundle", "", "path to a file containing all PEM-encoded CA certificates (will be used instead of the host's certificates if set)") + flag.StringVar(&opt.namespace, "namespace", "kubermatic", "The namespace where the webhooks will run") + flag.StringVar(&opt.workerClusterKubeconfig, "worker-cluster-kubeconfig", "", "Path to kubeconfig of worker/user cluster where machines and machinedeployments exist. If not specified, value from --kubeconfig or in-cluster config will be used") // OSM specific flags - flag.BoolVar(&useOSM, "use-osm", false, "osm controller is enabled for node bootstrap") + flag.BoolVar(&opt.useOSM, "use-osm", false, "osm controller is enabled for node bootstrap") flag.Parse() - kubeconfig = flag.Lookup("kubeconfig").Value.(flag.Getter).Get().(string) - masterURL = flag.Lookup("master").Value.(flag.Getter).Get().(string) + opt.kubeconfig = flag.Lookup("kubeconfig").Value.(flag.Getter).Get().(string) + opt.masterURL = flag.Lookup("master").Value.(flag.Getter).Get().(string) - if caBundleFile != "" { - if err := util.SetCABundleFile(caBundleFile); err != nil { + if opt.caBundleFile != "" { + if err := util.SetCABundleFile(opt.caBundleFile); err != nil { klog.Fatalf("-ca-bundle is invalid: %v", err) } } - cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) - if err != nil { - klog.Fatalf("error building kubeconfig: %v", err) - } - + // Add osmv1alpha1 to scheme if err := osmv1alpha1.AddToScheme(scheme.Scheme); err != nil { klog.Fatalf("failed to add osmv1alpha1 api to scheme: %v", err) } + cfg, err := clientcmd.BuildConfigFromFlags(opt.masterURL, opt.kubeconfig) + if err != nil { + klog.Fatalf("error building kubeconfig: %v", err) + } + client, err := ctrlruntimeclient.New(cfg, ctrlruntimeclient.Options{}) if err != nil { klog.Fatalf("failed to build client: %v", err) } + // Start with assuming that current cluster will be used as worker cluster + workerClient := client + // Handing for worker client + if opt.workerClusterKubeconfig != "" { + workerClusterConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: opt.workerClusterKubeconfig}, + &clientcmd.ConfigOverrides{}).ClientConfig() + if err != nil { + klog.Fatal(err) + } + + // Build dedicated client for worker cluster + workerClient, err = ctrlruntimeclient.New(workerClusterConfig, ctrlruntimeclient.Options{}) + if err != nil { + klog.Fatalf("failed to build worker client: %v", err) + } + } + um, err := userdatamanager.New() if err != nil { klog.Fatalf("error initialising userdata plugins: %v", err) } - srv, err := admission.New(admissionListenAddress, client, um, nodeFlags, useOSM, namespace) + srv, err := admission.New(opt.admissionListenAddress, client, workerClient, um, nodeFlags, opt.useOSM, opt.namespace) if err != nil { klog.Fatalf("failed to create admission hook: %v", err) } - if err := srv.ListenAndServeTLS(admissionTLSCertPath, admissionTLSKeyPath); err != nil { + if err := srv.ListenAndServeTLS(opt.admissionTLSCertPath, opt.admissionTLSKeyPath); err != nil { klog.Fatalf("Failed to start server: %v", err) } defer func() { @@ -103,6 +125,6 @@ func main() { klog.Fatalf("Failed to shutdown server: %v", err) } }() - klog.Infof("Listening on %s", admissionListenAddress) + klog.Infof("Listening on %s", opt.admissionListenAddress) select {} } diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go index 110c6ace6..bc355dfd9 100644 --- a/pkg/admission/admission.go +++ b/pkg/admission/admission.go @@ -42,6 +42,7 @@ import ( type admissionData struct { client ctrlruntimeclient.Client + workerClient ctrlruntimeclient.Client userDataManager *userdatamanager.Manager nodeSettings machinecontroller.NodeSettings useOSM bool @@ -53,6 +54,7 @@ var jsonPatch = admissionv1.PatchTypeJSONPatch func New( listenAddress string, client ctrlruntimeclient.Client, + workerClient ctrlruntimeclient.Client, um *userdatamanager.Manager, nodeFlags *node.Flags, useOSM bool, @@ -61,6 +63,7 @@ func New( mux := http.NewServeMux() ad := &admissionData{ client: client, + workerClient: workerClient, userDataManager: um, useOSM: useOSM, namespace: namespace, diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index fc8b4c539..eeefd66ff 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -113,7 +113,7 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec } } - skg := providerconfig.NewConfigVarResolver(ctx, ad.client) + skg := providerconfig.NewConfigVarResolver(ctx, ad.workerClient) prov, err := cloudprovider.ForProvider(providerConfig.CloudProvider, skg) if err != nil { return fmt.Errorf("failed to get cloud provider %q: %v", providerConfig.CloudProvider, err) From bbe21e98d85d17e6bc9c0406678e151bbc6946eb Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Wed, 12 Jan 2022 06:32:43 +0100 Subject: [PATCH 056/489] add bootstrap kubeconfig to the osm bootstraping (#1156) Signed-off-by: Moath Qasim --- pkg/controller/machine/bootstrap.go | 37 ++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/pkg/controller/machine/bootstrap.go b/pkg/controller/machine/bootstrap.go index f1a031e6a..fc6fe3fc9 100644 --- a/pkg/controller/machine/bootstrap.go +++ b/pkg/controller/machine/bootstrap.go @@ -31,6 +31,7 @@ import ( providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" "github.com/kubermatic/machine-controller/pkg/userdata/convert" "github.com/kubermatic/machine-controller/pkg/userdata/flatcar" + "github.com/kubermatic/machine-controller/pkg/userdata/helper" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -55,14 +56,14 @@ func getOSMBootstrapUserdata(ctx context.Context, client ctrlruntimeclient.Clien // Ignition configuration is used for flatcar if useIgnition(pconfig) { - return getOSMBootstrapUserDataForIgnition(ctx, req, pconfig.SSHPublicKeys, token, secretName, clusterName) + return getOSMBootstrapUserDataForIgnition(req, pconfig.SSHPublicKeys, token, secretName, clusterName) } // cloud-init is used for all other operating systems - return getOSMBootstrapUserDataForCloudInit(ctx, req, pconfig, token, secretName, clusterName) + return getOSMBootstrapUserDataForCloudInit(req, pconfig, token, secretName, clusterName) } // getOSMBootstrapUserDataForIgnition returns the userdata for the ignition bootstrap config -func getOSMBootstrapUserDataForIgnition(ctx context.Context, req plugin.UserDataRequest, sshPublicKeys []string, token, secretName, clusterName string) (string, error) { +func getOSMBootstrapUserDataForIgnition(req plugin.UserDataRequest, sshPublicKeys []string, token, secretName, clusterName string) (string, error) { data := struct { Token string SecretName string @@ -106,7 +107,7 @@ func getOSMBootstrapUserDataForIgnition(ctx context.Context, req plugin.UserData } // getOSMBootstrapUserDataForCloudInit returns the userdata for the cloud-init bootstrap script -func getOSMBootstrapUserDataForCloudInit(ctx context.Context, req plugin.UserDataRequest, pconfig *providerconfigtypes.Config, token, secretName, clusterName string) (string, error) { +func getOSMBootstrapUserDataForCloudInit(req plugin.UserDataRequest, pconfig *providerconfigtypes.Config, token, secretName, clusterName string) (string, error) { data := struct { Token string SecretName string @@ -166,17 +167,24 @@ func getOSMBootstrapUserDataForCloudInit(ctx context.Context, req plugin.UserDat return "", fmt.Errorf("failed to parse download-binaries template: %v", err) } + bootstrapKubeconfig, err := helper.StringifyKubeconfig(req.Kubeconfig) + if err != nil { + return "", fmt.Errorf("failed to format bootstrap kubeconfig: %v", err) + } + cloudInit := &bytes.Buffer{} err = bsCloudInit.Execute(cloudInit, struct { Script string Service string plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config + ProviderSpec *providerconfigtypes.Config + BootstrapKubeconfig string }{ - Script: base64.StdEncoding.EncodeToString(script.Bytes()), - Service: base64.StdEncoding.EncodeToString([]byte(bootstrapServiceContentTemplate)), - UserDataRequest: req, - ProviderSpec: pconfig, + Script: base64.StdEncoding.EncodeToString(script.Bytes()), + Service: base64.StdEncoding.EncodeToString([]byte(bootstrapServiceContentTemplate)), + UserDataRequest: req, + ProviderSpec: pconfig, + BootstrapKubeconfig: base64.StdEncoding.EncodeToString([]byte(bootstrapKubeconfig)), }) if err != nil { return "", fmt.Errorf("failed to execute cloudInitTemplate template: %v", err) @@ -293,6 +301,11 @@ write_files: encoding: b64 content: | {{ .Script }} +- path: /etc/kubernetes/bootstrap-kubelet.conf + permissions: '0600' + encoding: b64 + content: | + {{ .BootstrapKubeconfig }} - path: /etc/systemd/system/bootstrap.service permissions: '0644' encoding: b64 @@ -331,6 +344,12 @@ reboot {{- end }} storage: files: + - path: /etc/kubernetes/bootstrap-kubelet.conf + mode: 0600 + filesystem: root + contents: + inline: | + {{ .BootstrapKubeconfig }} - path: /opt/bin/bootstrap mode: 0755 filesystem: root From af811e3fb775c3c22408213a23a1b69f7a420b7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Wed, 12 Jan 2022 08:05:45 +0100 Subject: [PATCH 057/489] Rename NUTANIX_ALLOW_INSECURE to NUTANIX_INSECURE (#1155) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- pkg/cloudprovider/provider/nutanix/provider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 8b02aa204..5884d776a 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -150,7 +150,7 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return nil, nil, nil, err } - c.AllowInsecure, err = p.configVarResolver.GetConfigVarBoolValueOrEnv(rawConfig.AllowInsecure, "NUTANIX_ALLOW_INSECURE") + c.AllowInsecure, err = p.configVarResolver.GetConfigVarBoolValueOrEnv(rawConfig.AllowInsecure, "NUTANIX_INSECURE") if err != nil { return nil, nil, nil, err } From 17474051fe67702a33a453ff1883b6008983988f Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Wed, 12 Jan 2022 21:56:39 +0100 Subject: [PATCH 058/489] fix cloud config ignition selection (#1160) Signed-off-by: Moath Qasim --- pkg/controller/machine/bootstrap.go | 56 +++++++++++++---------------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/pkg/controller/machine/bootstrap.go b/pkg/controller/machine/bootstrap.go index fc6fe3fc9..ada02c5cd 100644 --- a/pkg/controller/machine/bootstrap.go +++ b/pkg/controller/machine/bootstrap.go @@ -30,7 +30,6 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" "github.com/kubermatic/machine-controller/pkg/userdata/convert" - "github.com/kubermatic/machine-controller/pkg/userdata/flatcar" "github.com/kubermatic/machine-controller/pkg/userdata/helper" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -54,16 +53,23 @@ func getOSMBootstrapUserdata(ctx context.Context, client ctrlruntimeclient.Clien return "", fmt.Errorf("failed to get providerSpec: %v", err) } - // Ignition configuration is used for flatcar - if useIgnition(pconfig) { - return getOSMBootstrapUserDataForIgnition(req, pconfig.SSHPublicKeys, token, secretName, clusterName) + bootstrapKubeconfig, err := helper.StringifyKubeconfig(req.Kubeconfig) + if err != nil { + return "", fmt.Errorf("failed to format bootstrap kubeconfig: %v", err) + } + + // Regardless if the provisioningUtility is set to use cloud-init, we only allow using ignition to provision flatcar + // machines with osm. + if pconfig.OperatingSystem == providerconfigtypes.OperatingSystemFlatcar { + return getOSMBootstrapUserDataForIgnition(req, pconfig.SSHPublicKeys, token, secretName, clusterName, bootstrapKubeconfig) } + // cloud-init is used for all other operating systems - return getOSMBootstrapUserDataForCloudInit(req, pconfig, token, secretName, clusterName) + return getOSMBootstrapUserDataForCloudInit(req, pconfig, token, secretName, clusterName, bootstrapKubeconfig) } // getOSMBootstrapUserDataForIgnition returns the userdata for the ignition bootstrap config -func getOSMBootstrapUserDataForIgnition(req plugin.UserDataRequest, sshPublicKeys []string, token, secretName, clusterName string) (string, error) { +func getOSMBootstrapUserDataForIgnition(req plugin.UserDataRequest, sshPublicKeys []string, token, secretName, clusterName, bootstrapKfg string) (string, error) { data := struct { Token string SecretName string @@ -89,15 +95,17 @@ func getOSMBootstrapUserDataForIgnition(req plugin.UserDataRequest, sshPublicKey ignitionConfig := &bytes.Buffer{} err = bsIgnitionConfig.Execute(ignitionConfig, struct { - Script string - Service string - SSHPublicKeys []string plugin.UserDataRequest + Script string + Service string + SSHPublicKeys []string + BootstrapKubeconfig string }{ - Script: script.String(), - Service: bootstrapServiceContentTemplate, - SSHPublicKeys: sshPublicKeys, - UserDataRequest: req, + UserDataRequest: req, + Script: script.String(), + Service: bootstrapServiceContentTemplate, + SSHPublicKeys: sshPublicKeys, + BootstrapKubeconfig: bootstrapKfg, }) if err != nil { return "", fmt.Errorf("failed to execute ignitionTemplate template: %v", err) @@ -107,7 +115,7 @@ func getOSMBootstrapUserDataForIgnition(req plugin.UserDataRequest, sshPublicKey } // getOSMBootstrapUserDataForCloudInit returns the userdata for the cloud-init bootstrap script -func getOSMBootstrapUserDataForCloudInit(req plugin.UserDataRequest, pconfig *providerconfigtypes.Config, token, secretName, clusterName string) (string, error) { +func getOSMBootstrapUserDataForCloudInit(req plugin.UserDataRequest, pconfig *providerconfigtypes.Config, token, secretName, clusterName, bootstrapKfg string) (string, error) { data := struct { Token string SecretName string @@ -167,11 +175,6 @@ func getOSMBootstrapUserDataForCloudInit(req plugin.UserDataRequest, pconfig *pr return "", fmt.Errorf("failed to parse download-binaries template: %v", err) } - bootstrapKubeconfig, err := helper.StringifyKubeconfig(req.Kubeconfig) - if err != nil { - return "", fmt.Errorf("failed to format bootstrap kubeconfig: %v", err) - } - cloudInit := &bytes.Buffer{} err = bsCloudInit.Execute(cloudInit, struct { Script string @@ -184,7 +187,7 @@ func getOSMBootstrapUserDataForCloudInit(req plugin.UserDataRequest, pconfig *pr Service: base64.StdEncoding.EncodeToString([]byte(bootstrapServiceContentTemplate)), UserDataRequest: req, ProviderSpec: pconfig, - BootstrapKubeconfig: base64.StdEncoding.EncodeToString([]byte(bootstrapKubeconfig)), + BootstrapKubeconfig: base64.StdEncoding.EncodeToString([]byte(bootstrapKfg)), }) if err != nil { return "", fmt.Errorf("failed to execute cloudInitTemplate template: %v", err) @@ -202,17 +205,6 @@ func cleanupTemplateOutput(output string) (string, error) { return woBlankLines, nil } -func useIgnition(p *providerconfigtypes.Config) bool { - if p.OperatingSystem == providerconfigtypes.OperatingSystemFlatcar { - config, err := flatcar.LoadConfig(p.OperatingSystemSpec) - if err != nil { - return false - } - return config.ProvisioningUtility == flatcar.Ignition - } - return false -} - const ( bootstrapAptBinContentTemplate = `#!/bin/bash set -xeuo pipefail @@ -349,7 +341,7 @@ storage: filesystem: root contents: inline: | - {{ .BootstrapKubeconfig }} +{{ .BootstrapKubeconfig | indent 10 }} - path: /opt/bin/bootstrap mode: 0755 filesystem: root From 4744ddce7d2ee7d6ae32408d9667c103f9dcb835 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 13 Jan 2022 16:01:45 +0500 Subject: [PATCH 059/489] Fixes for bootstrapping config on OpenStack (#1159) * Fixes for openstack bootstrapping Signed-off-by: Waleed Malik * Update mutating logic for default OSP annotation in MachineDeployment Signed-off-by: Waleed Malik * Refactored code Signed-off-by: Waleed Malik --- go.mod | 3 +-- go.sum | 4 +++- pkg/admission/machinedeployments_validation.go | 7 ++----- pkg/controller/machine/bootstrap.go | 12 +++++++++++- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 185e024cd..365447137 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/Azure/azure-sdk-for-go v49.0.0+incompatible github.com/Azure/go-autorest/autorest/azure/auth v0.5.5 github.com/Azure/go-autorest/autorest/to v0.4.0 - github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/BurntSushi/toml v0.3.1 github.com/Masterminds/semver/v3 v3.1.1 github.com/Masterminds/sprig/v3 v3.2.2 @@ -43,7 +42,7 @@ require ( google.golang.org/grpc v1.38.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b - k8c.io/operating-system-manager v0.3.0 + k8c.io/operating-system-manager v0.3.6 k8s.io/api v0.22.2 k8s.io/apiextensions-apiserver v0.22.2 k8s.io/apimachinery v0.22.2 diff --git a/go.sum b/go.sum index 70df54ce8..a04aea637 100644 --- a/go.sum +++ b/go.sum @@ -857,6 +857,7 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kubermatic/machine-controller v1.23.1/go.mod h1:mXWbT7SjqpgFhzCFT3yMEHKdIlT+KkGy4KQCkNRM9Fc= github.com/kubermatic/machine-controller v1.26.0/go.mod h1:dcJ+GdDSCxCwM0poxwOK8hVO7epiOORDmNMmb2veyw4= github.com/kubermatic/machine-controller v1.36.1/go.mod h1:6BFZEvEMZi8OT8aHOsS7DXYsF6ZSpmsNxsci7OLTTn8= +github.com/kubermatic/machine-controller v1.40.1/go.mod h1:5LVcN4tCybGg+55hIHcVzCjNsBJy2PlnXG0xIzKmXGY= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -1965,8 +1966,9 @@ honnef.co/go/tools v0.1.4/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= k8c.io/kubermatic/v2 v2.16.2 h1:tjPfI+VV51pggXCvcDL/qG1r7KHDBQPSPYngPxpRtp8= k8c.io/kubermatic/v2 v2.16.2/go.mod h1:NdW+2mq4ynRtfZs9yPnvcnFWQpzmM7ngntW6GeuQicU= k8c.io/operating-system-manager v0.1.0/go.mod h1:ULyZQO1irKjsQTNjIdrHld7SZ+joHjmPnOEs5Db8G8M= -k8c.io/operating-system-manager v0.3.0 h1:xu1BA1Uj22MAeXSx9mNumfm63/6P8xjRcojel+QD5wI= k8c.io/operating-system-manager v0.3.0/go.mod h1:ME5GOCNUrHG+57igEKP1JCJKVHynaLfodT8bRiYH3MY= +k8c.io/operating-system-manager v0.3.6 h1:irFFYE/IJM2Qo+lH1zat2o3Yvgb8hUaypPWAc0qGHNM= +k8c.io/operating-system-manager v0.3.6/go.mod h1:aFyB/RH9DBAk0Kj5JVtCixhm9ugTeC8akgRGMW28lPg= k8s.io/api v0.0.0-20181018013834-843ad2d9b9ae/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20181115043458-b799cb063522/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20190725062911-6607c48751ae/go.mod h1:1O0xzX/RAtnm7l+5VEUxZ1ysO2ghatfq/OZED4zM9kA= diff --git a/pkg/admission/machinedeployments_validation.go b/pkg/admission/machinedeployments_validation.go index 7a32952fb..b7d50718c 100644 --- a/pkg/admission/machinedeployments_validation.go +++ b/pkg/admission/machinedeployments_validation.go @@ -156,13 +156,10 @@ func ensureOSPAnnotation(md *v1alpha1.MachineDeployment, providerConfig provider // Annotation not specified, populate default OSP annotation switch providerConfig.OperatingSystem { case providerconfigtypes.OperatingSystemUbuntu, providerconfigtypes.OperatingSystemCentOS, providerconfigtypes.OperatingSystemFlatcar, - providerconfigtypes.OperatingSystemRHEL, providerconfigtypes.OperatingSystemSLES: + providerconfigtypes.OperatingSystemAmazonLinux2: md.Annotations[osmresources.MachineDeploymentOSPAnnotation] = fmt.Sprintf(ospNamePattern, providerConfig.OperatingSystem) return nil - case providerconfigtypes.OperatingSystemAmazonLinux2: - // This is a special case where the OS name suffix in OSP is different then the actual OS name - md.Annotations[osmresources.MachineDeploymentOSPAnnotation] = fmt.Sprintf(ospNamePattern, "amazon-linux") - return nil + default: return fmt.Errorf("failed to populate OSP annotation for machinedeployment with unsupported Operating System %s", providerConfig.OperatingSystem) } diff --git a/pkg/controller/machine/bootstrap.go b/pkg/controller/machine/bootstrap.go index ada02c5cd..02666d281 100644 --- a/pkg/controller/machine/bootstrap.go +++ b/pkg/controller/machine/bootstrap.go @@ -208,6 +208,8 @@ func cleanupTemplateOutput(output string) (string, error) { const ( bootstrapAptBinContentTemplate = `#!/bin/bash set -xeuo pipefail + +export DEBIAN_FRONTEND=noninteractive apt update && apt install -y curl jq curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-config"]' -r| base64 -d > /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg cloud-init clean @@ -296,8 +298,16 @@ write_files: - path: /etc/kubernetes/bootstrap-kubelet.conf permissions: '0600' encoding: b64 - content: | + content: | {{ .BootstrapKubeconfig }} +{{- if and (eq .ProviderSpec.CloudProvider "openstack") (eq .ProviderSpec.OperatingSystem "centos") }} +{{- /* The normal way of setting it via cloud-init is broken, see */}} +{{- /* https://bugs.launchpad.net/cloud-init/+bug/1662542 */}} +- path: /etc/hostname + permissions: '0600' + content: | + {{ .MachineSpec.Name }} +{{ end }} - path: /etc/systemd/system/bootstrap.service permissions: '0644' encoding: b64 From bcc8bd294b382dc1b921709e1f69a84517a5bbde Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 13 Jan 2022 17:16:44 +0500 Subject: [PATCH 060/489] Support for kubernetes v1.23.0 in e2e tests (#1158) * Support for v1.23.0 and remove v1.19.15 from tests Signed-off-by: Waleed Malik * Minor nit * Update README.md --- README.md | 2 +- examples/alibaba-machinedeployment.yaml | 2 +- examples/anexia-machinedeployment.yaml | 2 +- examples/aws-machinedeployment.yaml | 2 +- examples/azure-machinedeployment.yaml | 2 +- examples/digitalocean-machinedeployment.yaml | 2 +- examples/equinixmetal-machinedeployment.yaml | 2 +- examples/gce-machinedeployment.yaml | 2 +- examples/hetzner-machinedeployment.yaml | 2 +- examples/kubevirt-machinedeployment.yaml | 2 +- examples/linode-machinedeployment.yaml | 2 +- examples/openstack-machinedeployment.yaml | 2 +- examples/scaleway-machinedeployment.yaml | 2 +- ...e-datastore-cluster-machinedeployment.yaml | 2 +- examples/vsphere-machinedeployment.yaml | 2 +- hack/update-fixtures.sh | 2 +- pkg/userdata/amzn2/provider_test.go | 34 +++++++++--------- .../containerd-kubelet-v1.20-aws.yaml | 2 +- .../amzn2/testdata/kubelet-v1.20-aws.yaml | 2 +- .../testdata/kubelet-v1.21-aws-external.yaml | 2 +- .../amzn2/testdata/kubelet-v1.21-aws.yaml | 2 +- .../kubelet-v1.21-vsphere-mirrors.yaml | 2 +- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 2 +- .../amzn2/testdata/kubelet-v1.21-vsphere.yaml | 2 +- .../amzn2/testdata/kubelet-v1.22-aws.yaml | 2 +- ...-v1.19-aws.yaml => kubelet-v1.23-aws.yaml} | 4 +-- pkg/userdata/centos/provider_test.go | 34 +++++++++--------- .../kubelet-containerd-v1.20-aws.yaml | 2 +- .../centos/testdata/kubelet-v1.20-aws.yaml | 2 +- .../testdata/kubelet-v1.21-aws-external.yaml | 2 +- .../centos/testdata/kubelet-v1.21-aws.yaml | 2 +- .../kubelet-v1.21-vsphere-mirrors.yaml | 2 +- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 2 +- .../testdata/kubelet-v1.21-vsphere.yaml | 2 +- .../centos/testdata/kubelet-v1.22-aws.yaml | 2 +- ...-v1.19-aws.yaml => kubelet-v1.23-aws.yaml} | 4 +-- pkg/userdata/flatcar/provider_test.go | 34 +++++++++--------- ...v1.19.15.yaml => cloud-init_v1.20.14.yaml} | 2 +- ...t_v1.21.5.yaml => cloud-init_v1.21.8.yaml} | 2 +- ...t_v1.22.2.yaml => cloud-init_v1.22.5.yaml} | 2 +- ..._v1.20.11.yaml => cloud-init_v1.23.0.yaml} | 4 +-- pkg/userdata/flatcar/testdata/containerd.yaml | 2 +- ...n_v1.19.15.json => ignition_v1.20.14.json} | 2 +- ...ion_v1.21.5.json => ignition_v1.21.8.json} | 2 +- ...ion_v1.22.2.json => ignition_v1.22.5.json} | 2 +- ...on_v1.20.11.json => ignition_v1.23.0.json} | 2 +- pkg/userdata/helper/common_test.go | 8 ++--- .../helper/download_binaries_script_test.go | 4 +-- pkg/userdata/helper/kubelet_test.go | 8 ++--- ...lden => download_binaries_v1.20.14.golden} | 2 +- ...olden => download_binaries_v1.21.8.golden} | 2 +- ...olden => download_binaries_v1.22.5.golden} | 2 +- ...olden => download_binaries_v1.23.0.golden} | 2 +- ...emd_unit_version-v1.20.14-external.golden} | 0 ...blet_systemd_unit_version-v1.20.14.golden} | 0 ...temd_unit_version-v1.21.8-external.golden} | 0 ...ublet_systemd_unit_version-v1.21.8.golden} | 0 ...temd_unit_version-v1.22.5-external.golden} | 0 ...ublet_systemd_unit_version-v1.22.5.golden} | 0 ...temd_unit_version-v1.23.0-external.golden} | 2 -- ...ublet_systemd_unit_version-v1.23.0.golden} | 2 -- ... => safe_download_binaries_v1.22.5.golden} | 2 +- pkg/userdata/rhel/provider_test.go | 36 +++++++++---------- ...yaml => kubelet-containerd-v1.20-aws.yaml} | 2 +- .../rhel/testdata/kubelet-v1.20-aws.yaml | 2 +- .../rhel/testdata/kubelet-v1.21-aws.yaml | 2 +- .../rhel/testdata/kubelet-v1.22-aws.yaml | 2 +- ...l.yaml => kubelet-v1.23-aws-external.yaml} | 4 +-- ...-v1.19-aws.yaml => kubelet-v1.23-aws.yaml} | 4 +-- ...aml => kubelet-v1.23-vsphere-mirrors.yaml} | 4 +-- ....yaml => kubelet-v1.23-vsphere-proxy.yaml} | 4 +-- ...sphere.yaml => kubelet-v1.23-vsphere.yaml} | 4 +-- pkg/userdata/sles/provider_test.go | 10 +++--- .../sles/testdata/dist-upgrade-on-boot.yaml | 2 +- .../kubelet-version-without-v-prefix.yaml | 2 +- .../sles/testdata/multiple-dns-servers.yaml | 2 +- .../sles/testdata/multiple-ssh-keys.yaml | 2 +- .../openstack-overwrite-cloud-config.yaml | 2 +- pkg/userdata/sles/testdata/openstack.yaml | 2 +- ...sion-1.19.15.yaml => version-1.20.14.yaml} | 2 +- ...ersion-1.22.2.yaml => version-1.21.8.yaml} | 2 +- ...ersion-1.21.5.yaml => version-1.22.5.yaml} | 2 +- ...rsion-1.20.11.yaml => version-1.23.0.yaml} | 4 +-- .../sles/testdata/vsphere-mirrors.yaml | 2 +- pkg/userdata/sles/testdata/vsphere-proxy.yaml | 2 +- pkg/userdata/sles/testdata/vsphere.yaml | 2 +- pkg/userdata/ubuntu/provider_test.go | 22 ++++++------ pkg/userdata/ubuntu/testdata/containerd.yaml | 2 +- .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 2 +- .../kubelet-version-without-v-prefix.yaml | 2 +- .../ubuntu/testdata/multiple-dns-servers.yaml | 2 +- .../ubuntu/testdata/multiple-ssh-keys.yaml | 2 +- .../openstack-overwrite-cloud-config.yaml | 2 +- pkg/userdata/ubuntu/testdata/openstack.yaml | 2 +- ...sion-1.19.15.yaml => version-1.20.14.yaml} | 2 +- ...ersion-1.21.5.yaml => version-1.21.8.yaml} | 2 +- ...ersion-1.22.2.yaml => version-1.22.5.yaml} | 2 +- ...rsion-1.20.11.yaml => version-1.23.0.yaml} | 4 +-- .../ubuntu/testdata/vsphere-mirrors.yaml | 2 +- .../ubuntu/testdata/vsphere-proxy.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere.yaml | 2 +- test/e2e/provisioning/all_e2e_test.go | 16 ++++----- test/e2e/provisioning/helper.go | 8 ++--- ...hinedeployment-azure-redhat-satellite.yaml | 2 +- test/tools/integration/Makefile | 2 +- .../integration/master_install_script.sh | 2 +- test/tools/integration/versions.tf | 4 +-- 107 files changed, 196 insertions(+), 220 deletions(-) rename pkg/userdata/amzn2/testdata/{kubelet-v1.19-aws.yaml => kubelet-v1.23-aws.yaml} (98%) rename pkg/userdata/centos/testdata/{kubelet-v1.19-aws.yaml => kubelet-v1.23-aws.yaml} (98%) rename pkg/userdata/flatcar/testdata/{cloud-init_v1.19.15.yaml => cloud-init_v1.20.14.yaml} (99%) rename pkg/userdata/flatcar/testdata/{cloud-init_v1.21.5.yaml => cloud-init_v1.21.8.yaml} (99%) rename pkg/userdata/flatcar/testdata/{cloud-init_v1.22.2.yaml => cloud-init_v1.22.5.yaml} (99%) rename pkg/userdata/flatcar/testdata/{cloud-init_v1.20.11.yaml => cloud-init_v1.23.0.yaml} (98%) rename pkg/userdata/flatcar/testdata/{ignition_v1.19.15.json => ignition_v1.20.14.json} (99%) rename pkg/userdata/flatcar/testdata/{ignition_v1.21.5.json => ignition_v1.21.8.json} (99%) rename pkg/userdata/flatcar/testdata/{ignition_v1.22.2.json => ignition_v1.22.5.json} (99%) rename pkg/userdata/flatcar/testdata/{ignition_v1.20.11.json => ignition_v1.23.0.json} (72%) rename pkg/userdata/helper/testdata/{download_binaries_v1.19.15.golden => download_binaries_v1.20.14.golden} (92%) rename pkg/userdata/helper/testdata/{download_binaries_v1.21.5.golden => download_binaries_v1.21.8.golden} (92%) rename pkg/userdata/helper/testdata/{download_binaries_v1.22.2.golden => download_binaries_v1.22.5.golden} (92%) rename pkg/userdata/helper/testdata/{download_binaries_v1.20.11.golden => download_binaries_v1.23.0.golden} (92%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.19.15-external.golden => kublet_systemd_unit_version-v1.20.14-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.19.15.golden => kublet_systemd_unit_version-v1.20.14.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.20.11-external.golden => kublet_systemd_unit_version-v1.21.8-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.20.11.golden => kublet_systemd_unit_version-v1.21.8.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.21.5-external.golden => kublet_systemd_unit_version-v1.22.5-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.21.5.golden => kublet_systemd_unit_version-v1.22.5.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.22.2-external.golden => kublet_systemd_unit_version-v1.23.0-external.golden} (89%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.22.2.golden => kublet_systemd_unit_version-v1.23.0.golden} (89%) rename pkg/userdata/helper/testdata/{safe_download_binaries_v1.20.1.golden => safe_download_binaries_v1.22.5.golden} (98%) rename pkg/userdata/rhel/testdata/{kubelet-containerd-v1.19-aws.yaml => kubelet-containerd-v1.20-aws.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.22-aws-external.yaml => kubelet-v1.23-aws-external.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.19-aws.yaml => kubelet-v1.23-aws.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.22-vsphere-mirrors.yaml => kubelet-v1.23-vsphere-mirrors.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.22-vsphere-proxy.yaml => kubelet-v1.23-vsphere-proxy.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.22-vsphere.yaml => kubelet-v1.23-vsphere.yaml} (99%) rename pkg/userdata/sles/testdata/{version-1.19.15.yaml => version-1.20.14.yaml} (99%) rename pkg/userdata/sles/testdata/{version-1.22.2.yaml => version-1.21.8.yaml} (99%) rename pkg/userdata/sles/testdata/{version-1.21.5.yaml => version-1.22.5.yaml} (99%) rename pkg/userdata/sles/testdata/{version-1.20.11.yaml => version-1.23.0.yaml} (98%) rename pkg/userdata/ubuntu/testdata/{version-1.19.15.yaml => version-1.20.14.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.21.5.yaml => version-1.21.8.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.22.2.yaml => version-1.22.5.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.20.11.yaml => version-1.23.0.yaml} (98%) diff --git a/README.md b/README.md index 010e9626a..7e07d201c 100644 --- a/README.md +++ b/README.md @@ -28,10 +28,10 @@ machine-controller tries to follow the Kubernetes version Currently supported K8S versions are: +- 1.23 - 1.22 - 1.21 - 1.20 -- 1.19 ## What does not work - Master creation (Not planned at the moment) diff --git a/examples/alibaba-machinedeployment.yaml b/examples/alibaba-machinedeployment.yaml index 1736b4730..df5c0ccdb 100644 --- a/examples/alibaba-machinedeployment.yaml +++ b/examples/alibaba-machinedeployment.yaml @@ -62,4 +62,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.13.1 + kubelet: 1.22.5 diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index 04df5c66a..156cfd21b 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -45,4 +45,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: "1.19.1" + kubelet: 1.22.5 diff --git a/examples/aws-machinedeployment.yaml b/examples/aws-machinedeployment.yaml index 5d0060f21..6404fff68 100644 --- a/examples/aws-machinedeployment.yaml +++ b/examples/aws-machinedeployment.yaml @@ -80,4 +80,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.9.6 + kubelet: 1.22.5 diff --git a/examples/azure-machinedeployment.yaml b/examples/azure-machinedeployment.yaml index 20793921c..abbf780e5 100644 --- a/examples/azure-machinedeployment.yaml +++ b/examples/azure-machinedeployment.yaml @@ -92,4 +92,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.9.6 + kubelet: 1.22.5 diff --git a/examples/digitalocean-machinedeployment.yaml b/examples/digitalocean-machinedeployment.yaml index ebde3cf31..5887aaf50 100644 --- a/examples/digitalocean-machinedeployment.yaml +++ b/examples/digitalocean-machinedeployment.yaml @@ -57,4 +57,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.9.6 + kubelet: 1.22.5 diff --git a/examples/equinixmetal-machinedeployment.yaml b/examples/equinixmetal-machinedeployment.yaml index 3300a951b..c119de9d9 100644 --- a/examples/equinixmetal-machinedeployment.yaml +++ b/examples/equinixmetal-machinedeployment.yaml @@ -51,4 +51,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.13.1 + kubelet: 1.22.5 diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index 315099270..6df8cdb06 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -75,4 +75,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.13.5 + kubelet: 1.22.5 diff --git a/examples/hetzner-machinedeployment.yaml b/examples/hetzner-machinedeployment.yaml index 51b7c1cf4..53c85b533 100644 --- a/examples/hetzner-machinedeployment.yaml +++ b/examples/hetzner-machinedeployment.yaml @@ -61,4 +61,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.13.1 + kubelet: 1.22.5 diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index 8686c9acc..f8bc5c295 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -47,4 +47,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: "1.12.2" + kubelet: 1.22.5 diff --git a/examples/linode-machinedeployment.yaml b/examples/linode-machinedeployment.yaml index 88a510ebe..7caec78e0 100644 --- a/examples/linode-machinedeployment.yaml +++ b/examples/linode-machinedeployment.yaml @@ -53,4 +53,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.9.6 + kubelet: 1.22.5 diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index cf8ea8267..5b7442984 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -164,4 +164,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.9.6 + kubelet: 1.22.5 diff --git a/examples/scaleway-machinedeployment.yaml b/examples/scaleway-machinedeployment.yaml index 4b79fed20..0cea3c70c 100644 --- a/examples/scaleway-machinedeployment.yaml +++ b/examples/scaleway-machinedeployment.yaml @@ -61,4 +61,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.21.5 + kubelet: 1.22.5 diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml index 066d7e59e..18bc56084 100644 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ b/examples/vsphere-datastore-cluster-machinedeployment.yaml @@ -75,4 +75,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.9.6 + kubelet: 1.22.5 diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index 060f483d7..740fe2515 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -75,4 +75,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.9.6 + kubelet: 1.22.5 diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 8d2b2b552..3826851ec 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -14,6 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -go test ./... -update || go test ./... +go test ./... -v -update || go test ./... if [[ $? -eq 0 ]]; then echo "Successfully updated fixtures"; else "Failed to update fixtures"; fi diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index 6d77d9149..2bc45fc5f 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -99,21 +99,12 @@ func TestUserDataGeneration(t *testing.T) { t.Parallel() tests := []userDataTestCase{ - { - name: "kubelet-v1.19-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.19.15", - }, - }, - }, { name: "kubelet-v1.20-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.20.11", + Kubelet: "1.20.14", }, }, }, @@ -122,7 +113,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.20.11", + Kubelet: "1.20.14", }, }, containerruntime: "containerd", @@ -132,7 +123,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.21.8", }, }, }, @@ -141,7 +132,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.21.8", }, }, externalCloudProvider: true, @@ -151,7 +142,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.21.8", }, }, cloudProviderName: stringPtr("vsphere"), @@ -161,7 +152,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.21.8", }, }, cloudProviderName: stringPtr("vsphere"), @@ -175,7 +166,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.21.8", }, }, cloudProviderName: stringPtr("vsphere"), @@ -189,7 +180,16 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.2", + Kubelet: "1.22.5", + }, + }, + }, + { + name: "kubelet-v1.23-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.23.0", }, }, }, diff --git a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml index d010fef17..a0cd36bf1 100644 --- a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml @@ -143,7 +143,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.20.11}" + KUBE_VERSION="${KUBE_VERSION:-v1.20.14}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml index 54ccf66c0..4aef3134b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml @@ -140,7 +140,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.20.11}" + KUBE_VERSION="${KUBE_VERSION:-v1.20.14}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml index 2cc89d9f1..1589d659e 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml @@ -140,7 +140,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml index feccb64d3..d12483f32 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml @@ -140,7 +140,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 14718cb55..0bb089e0b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -153,7 +153,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml index 2ff8bba55..9d8770ec9 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -153,7 +153,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml index d8dc13a75..e8888cd18 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml @@ -145,7 +145,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml index 77ab23022..90cf27ffd 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml @@ -140,7 +140,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml similarity index 98% rename from pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml index 55814f194..86bd8353e 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml @@ -140,7 +140,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.19.15}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -213,8 +213,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index 55660717b..e9ac91ae6 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -99,21 +99,12 @@ func TestUserDataGeneration(t *testing.T) { t.Parallel() tests := []userDataTestCase{ - { - name: "kubelet-v1.19-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.19.15", - }, - }, - }, { name: "kubelet-v1.20-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.20.11", + Kubelet: "1.20.14", }, }, }, @@ -122,7 +113,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.20.11", + Kubelet: "1.20.14", }, }, containerruntime: "containerd", @@ -132,7 +123,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.21.8", }, }, }, @@ -141,7 +132,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.21.8", }, }, externalCloudProvider: true, @@ -151,7 +142,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.21.8", }, }, cloudProviderName: stringPtr("vsphere"), @@ -161,7 +152,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.21.8", }, }, cloudProviderName: stringPtr("vsphere"), @@ -175,7 +166,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.21.8", }, }, cloudProviderName: stringPtr("vsphere"), @@ -189,7 +180,16 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.2", + Kubelet: "1.22.5", + }, + }, + }, + { + name: "kubelet-v1.23-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.23.0", }, }, }, diff --git a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml index b5f903ed5..dc062671c 100644 --- a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.20.11}" + KUBE_VERSION="${KUBE_VERSION:-v1.20.14}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml index b5351c639..609bdc495 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml @@ -145,7 +145,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.20.11}" + KUBE_VERSION="${KUBE_VERSION:-v1.20.14}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml index 445eb27cb..4e417e4fc 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml @@ -145,7 +145,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml index ed16be8d0..f2eaa17c2 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml @@ -145,7 +145,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml index ba70e2b71..096bd7459 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -158,7 +158,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml index d9708de7a..def43e8c0 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -158,7 +158,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml index 5024529a9..214252946 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index 0f57fc439..19f34f449 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -145,7 +145,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml similarity index 98% rename from pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index 7ec997fd6..b5015b267 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -145,7 +145,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.19.15}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -218,8 +218,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index 9c9509c69..5a1cf0fa3 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -122,7 +122,7 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "ignition_v1.19.15", + name: "ignition_v1.20.14", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -138,7 +138,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.19.15", + Kubelet: "v1.20.14", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -153,7 +153,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.20.11", + name: "ignition_v1.21.8", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -169,7 +169,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.20.11", + Kubelet: "v1.21.8", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -184,7 +184,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.21.5", + name: "ignition_v1.22.5", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -200,7 +200,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.21.5", + Kubelet: "v1.22.5", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -215,7 +215,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.22.2", + name: "ignition_v1.23.0", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -231,7 +231,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.22.2", + Kubelet: "v1.23.0", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -246,7 +246,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.19.15", + name: "cloud-init_v1.20.14", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -262,7 +262,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.19.15", + Kubelet: "v1.20.14", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -277,7 +277,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.20.11", + name: "cloud-init_v1.21.8", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -293,7 +293,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.20.11", + Kubelet: "v1.21.8", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -308,7 +308,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.21.5", + name: "cloud-init_v1.22.5", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -324,7 +324,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.21.5", + Kubelet: "v1.22.5", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -339,7 +339,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.22.2", + name: "cloud-init_v1.23.0", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -355,7 +355,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.22.2", + Kubelet: "v1.23.0", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -381,7 +381,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.20.11", + Kubelet: "v1.21.8", }, }, ccProvider: &fakeCloudConfigProvider{}, diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.19.15.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml similarity index 99% rename from pkg/userdata/flatcar/testdata/cloud-init_v1.19.15.yaml rename to pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml index 140e35117..a4fcbc840 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.19.15.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml @@ -405,7 +405,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.19.15}" + KUBE_VERSION="${KUBE_VERSION:-v1.20.14}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml similarity index 99% rename from pkg/userdata/flatcar/testdata/cloud-init_v1.21.5.yaml rename to pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml index 933d1b18d..10b2beef3 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.5.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml @@ -405,7 +405,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.2.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml similarity index 99% rename from pkg/userdata/flatcar/testdata/cloud-init_v1.22.2.yaml rename to pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml index 1f2634ddd..e5158b13a 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.2.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml @@ -405,7 +405,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.11.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml similarity index 98% rename from pkg/userdata/flatcar/testdata/cloud-init_v1.20.11.yaml rename to pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml index b4461d4ed..70214dcec 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.11.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml @@ -124,8 +124,6 @@ coreos: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} [Install] @@ -405,7 +403,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.20.11}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index 63505cd17..f6525236d 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -388,7 +388,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.20.11}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.19.15.json b/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json similarity index 99% rename from pkg/userdata/flatcar/testdata/ignition_v1.19.15.json rename to pkg/userdata/flatcar/testdata/ignition_v1.20.14.json index fc862002c..b5072e336 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.19.15.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.19.15%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.14%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.21.5.json b/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json similarity index 99% rename from pkg/userdata/flatcar/testdata/ignition_v1.21.5.json rename to pkg/userdata/flatcar/testdata/ignition_v1.21.8.json index f47d95b64..7679a2f17 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.21.5.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.8%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.22.2.json b/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json similarity index 99% rename from pkg/userdata/flatcar/testdata/ignition_v1.22.2.json rename to pkg/userdata/flatcar/testdata/ignition_v1.22.5.json index cf0ced1af..245525c21 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.22.2.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.2%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.20.11.json b/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json similarity index 72% rename from pkg/userdata/flatcar/testdata/ignition_v1.20.11.json rename to pkg/userdata/flatcar/testdata/ignition_v1.23.0.json index b9a3d8142..baf23fc1e 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.20.11.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.11%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/common_test.go b/pkg/userdata/helper/common_test.go index b9e6b559e..2b8a79edd 100644 --- a/pkg/userdata/helper/common_test.go +++ b/pkg/userdata/helper/common_test.go @@ -26,9 +26,9 @@ var update = flag.Bool("update", false, "update testdata files") var ( versions = []*semver.Version{ - semver.MustParse("v1.19.15"), - semver.MustParse("v1.20.11"), - semver.MustParse("v1.21.5"), - semver.MustParse("v1.22.2"), + semver.MustParse("v1.20.14"), + semver.MustParse("v1.21.8"), + semver.MustParse("v1.22.5"), + semver.MustParse("v1.23.0"), } ) diff --git a/pkg/userdata/helper/download_binaries_script_test.go b/pkg/userdata/helper/download_binaries_script_test.go index 4e2099281..6dd60b90d 100644 --- a/pkg/userdata/helper/download_binaries_script_test.go +++ b/pkg/userdata/helper/download_binaries_script_test.go @@ -38,9 +38,9 @@ func TestDownloadBinariesScript(t *testing.T) { } func TestSafeDownloadBinariesScript(t *testing.T) { - name := "safe_download_binaries_v1.20.1" + name := "safe_download_binaries_v1.22.5" t.Run(name, func(t *testing.T) { - script, err := SafeDownloadBinariesScript("v1.20.1") + script, err := SafeDownloadBinariesScript("v1.22.5") if err != nil { t.Error(err) } diff --git a/pkg/userdata/helper/kubelet_test.go b/pkg/userdata/helper/kubelet_test.go index 813430a38..e5e1b33f7 100644 --- a/pkg/userdata/helper/kubelet_test.go +++ b/pkg/userdata/helper/kubelet_test.go @@ -63,7 +63,7 @@ func TestKubeletSystemdUnit(t *testing.T) { tests = append(tests, []kubeletFlagTestCase{ { name: "multiple-dns-servers", - version: semver.MustParse("v1.20.1"), + version: semver.MustParse("v1.20.14"), dnsIPs: []net.IP{ net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), @@ -73,14 +73,14 @@ func TestKubeletSystemdUnit(t *testing.T) { }, { name: "cloud-provider-set", - version: semver.MustParse("v1.20.1"), + version: semver.MustParse("v1.20.14"), dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, hostname: "some-test-node", cloudProvider: "aws", }, { name: "pause-image-set", - version: semver.MustParse("v1.20.1"), + version: semver.MustParse("v1.20.14"), dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, hostname: "some-test-node", cloudProvider: "aws", @@ -88,7 +88,7 @@ func TestKubeletSystemdUnit(t *testing.T) { }, { name: "taints-set", - version: semver.MustParse("v1.20.1"), + version: semver.MustParse("v1.20.14"), dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, hostname: "some-test-node", cloudProvider: "aws", diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.19.15.golden b/pkg/userdata/helper/testdata/download_binaries_v1.20.14.golden similarity index 92% rename from pkg/userdata/helper/testdata/download_binaries_v1.19.15.golden rename to pkg/userdata/helper/testdata/download_binaries_v1.20.14.golden index 66118f42a..3d8518a34 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.19.15.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.20.14.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.19.15/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.20.14/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.21.5.golden b/pkg/userdata/helper/testdata/download_binaries_v1.21.8.golden similarity index 92% rename from pkg/userdata/helper/testdata/download_binaries_v1.21.5.golden rename to pkg/userdata/helper/testdata/download_binaries_v1.21.8.golden index d5f466619..e31636457 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.21.5.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.21.8.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.21.5/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.21.8/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.22.2.golden b/pkg/userdata/helper/testdata/download_binaries_v1.22.5.golden similarity index 92% rename from pkg/userdata/helper/testdata/download_binaries_v1.22.2.golden rename to pkg/userdata/helper/testdata/download_binaries_v1.22.5.golden index 05148ac2d..5a6d5e8b8 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.22.2.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.22.5.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.22.2/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.22.5/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.20.11.golden b/pkg/userdata/helper/testdata/download_binaries_v1.23.0.golden similarity index 92% rename from pkg/userdata/helper/testdata/download_binaries_v1.20.11.golden rename to pkg/userdata/helper/testdata/download_binaries_v1.23.0.golden index 7d41cb886..c93028eb3 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.20.11.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.23.0.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.20.11/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.23.0/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.19.15-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14-external.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.19.15-external.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14-external.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.19.15.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.19.15.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.11-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8-external.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.11-external.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8-external.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.11.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.11.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.5-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5-external.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.5-external.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5-external.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.5.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.5.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.2-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0-external.golden similarity index 89% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.2-external.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0-external.golden index e595e9677..672b55ae0 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.2-external.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0-external.golden @@ -27,8 +27,6 @@ ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --hostname-override=some-test-node \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.2.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0.golden similarity index 89% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.2.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0.golden index b05f41061..c7b334881 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.2.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0.golden @@ -26,8 +26,6 @@ ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --hostname-override=some-test-node \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/helper/testdata/safe_download_binaries_v1.20.1.golden b/pkg/userdata/helper/testdata/safe_download_binaries_v1.22.5.golden similarity index 98% rename from pkg/userdata/helper/testdata/safe_download_binaries_v1.20.1.golden rename to pkg/userdata/helper/testdata/safe_download_binaries_v1.22.5.golden index c8465e026..c3ef51f69 100644 --- a/pkg/userdata/helper/testdata/safe_download_binaries_v1.20.1.golden +++ b/pkg/userdata/helper/testdata/safe_download_binaries_v1.22.5.golden @@ -39,7 +39,7 @@ tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - -KUBE_VERSION="${KUBE_VERSION:-v1.20.1}" +KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index e1170990d..5390131ca 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -100,77 +100,77 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.19-aws", + name: "kubelet-v1.20-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.19.15", + Kubelet: "1.20.14", }, }, }, { - name: "kubelet-containerd-v1.19-aws", + name: "kubelet-containerd-v1.20-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.19.15", + Kubelet: "1.20.14", }, }, containerruntime: "containerd", }, { - name: "kubelet-v1.20-aws", + name: "kubelet-v1.21-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.20.11", + Kubelet: "1.21.8", }, }, }, { - name: "kubelet-v1.21-aws", + name: "kubelet-v1.22-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.22.5", }, }, }, { - name: "kubelet-v1.22-aws", + name: "kubelet-v1.23-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.2", + Kubelet: "1.23.0", }, }, }, { - name: "kubelet-v1.22-aws-external", + name: "kubelet-v1.23-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.2", + Kubelet: "1.23.0", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.22-vsphere", + name: "kubelet-v1.23-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.2", + Kubelet: "1.23.0", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.22-vsphere-proxy", + name: "kubelet-v1.23-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.2", + Kubelet: "1.23.0", }, }, cloudProviderName: stringPtr("vsphere"), @@ -180,11 +180,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.22-vsphere-mirrors", + name: "kubelet-v1.23-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.2", + Kubelet: "1.23.0", }, }, cloudProviderName: stringPtr("vsphere"), diff --git a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.19-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-containerd-v1.19-aws.yaml rename to pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml index d978297ec..c57587acd 100644 --- a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.19-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.19.15}" + KUBE_VERSION="${KUBE_VERSION:-v1.20.14}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml index 05229a20c..f956a6cb6 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml @@ -145,7 +145,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.20.11}" + KUBE_VERSION="${KUBE_VERSION:-v1.20.14}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml index 32c376dfb..9c539c954 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml @@ -145,7 +145,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index 7b0f1e22b..d23515508 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -145,7 +145,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.22-aws-external.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index acf043be3..b33e71100 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -145,7 +145,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -218,8 +218,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index 0c8b7b939..1216b9206 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.19-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -145,7 +145,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.19.15}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -218,8 +218,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-mirrors.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 2fa84efd8..7a245c7d9 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -159,7 +159,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -236,8 +236,6 @@ write_files: --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-proxy.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index 600490f62..7d9464d77 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -159,7 +159,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -236,8 +236,6 @@ write_files: --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index 1bd8e8b76..0eb0919b9 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -227,8 +227,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/provider_test.go b/pkg/userdata/sles/provider_test.go index f4d64cfea..794ead9bd 100644 --- a/pkg/userdata/sles/provider_test.go +++ b/pkg/userdata/sles/provider_test.go @@ -93,7 +93,7 @@ kPe6XoSbiLm/kxk32T0= ) const ( - defaultVersion = "1.21.5" + defaultVersion = "1.22.5" ) type fakeCloudConfigProvider struct { @@ -126,10 +126,10 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.19.15"), - semver.MustParse("v1.20.11"), - semver.MustParse("v1.21.5"), - semver.MustParse("v1.22.2"), + semver.MustParse("v1.20.14"), + semver.MustParse("v1.21.8"), + semver.MustParse("v1.22.5"), + semver.MustParse("v1.23.0"), } var tests []userDataTestCase diff --git a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml index 4253fa908..02831730c 100644 --- a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml @@ -106,7 +106,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml index d6eff7725..f187a8d87 100644 --- a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml @@ -104,7 +104,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml index eed8e7c76..1dd38a7f2 100644 --- a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml @@ -104,7 +104,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml index a1a41665e..69919657b 100644 --- a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml @@ -106,7 +106,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml index 67221d6bd..90f47e9fd 100644 --- a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml @@ -104,7 +104,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/sles/testdata/openstack.yaml b/pkg/userdata/sles/testdata/openstack.yaml index 25629422e..408a4bbc3 100644 --- a/pkg/userdata/sles/testdata/openstack.yaml +++ b/pkg/userdata/sles/testdata/openstack.yaml @@ -104,7 +104,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/sles/testdata/version-1.19.15.yaml b/pkg/userdata/sles/testdata/version-1.20.14.yaml similarity index 99% rename from pkg/userdata/sles/testdata/version-1.19.15.yaml rename to pkg/userdata/sles/testdata/version-1.20.14.yaml index 08670aea6..4aac8a172 100644 --- a/pkg/userdata/sles/testdata/version-1.19.15.yaml +++ b/pkg/userdata/sles/testdata/version-1.20.14.yaml @@ -104,7 +104,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.19.15}" + KUBE_VERSION="${KUBE_VERSION:-v1.20.14}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/sles/testdata/version-1.22.2.yaml b/pkg/userdata/sles/testdata/version-1.21.8.yaml similarity index 99% rename from pkg/userdata/sles/testdata/version-1.22.2.yaml rename to pkg/userdata/sles/testdata/version-1.21.8.yaml index 99bb98744..e1af02656 100644 --- a/pkg/userdata/sles/testdata/version-1.22.2.yaml +++ b/pkg/userdata/sles/testdata/version-1.21.8.yaml @@ -104,7 +104,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/sles/testdata/version-1.21.5.yaml b/pkg/userdata/sles/testdata/version-1.22.5.yaml similarity index 99% rename from pkg/userdata/sles/testdata/version-1.21.5.yaml rename to pkg/userdata/sles/testdata/version-1.22.5.yaml index d6eff7725..f187a8d87 100644 --- a/pkg/userdata/sles/testdata/version-1.21.5.yaml +++ b/pkg/userdata/sles/testdata/version-1.22.5.yaml @@ -104,7 +104,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/sles/testdata/version-1.20.11.yaml b/pkg/userdata/sles/testdata/version-1.23.0.yaml similarity index 98% rename from pkg/userdata/sles/testdata/version-1.20.11.yaml rename to pkg/userdata/sles/testdata/version-1.23.0.yaml index c5be4ae4a..b8c35b6be 100644 --- a/pkg/userdata/sles/testdata/version-1.20.11.yaml +++ b/pkg/userdata/sles/testdata/version-1.23.0.yaml @@ -104,7 +104,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.20.11}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -176,8 +176,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml index 4a76ef827..0c4253402 100644 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml @@ -114,7 +114,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml index fd2c37291..78ec370d5 100644 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/sles/testdata/vsphere-proxy.yaml @@ -114,7 +114,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml index 61eed97b4..d8ed00fc8 100644 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ b/pkg/userdata/sles/testdata/vsphere.yaml @@ -105,7 +105,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 6bc781b42..7fd043e32 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -92,7 +92,7 @@ kPe6XoSbiLm/kxk32T0= ) const ( - defaultVersion = "1.21.5" + defaultVersion = "1.22.5" ) type fakeCloudConfigProvider struct { @@ -126,10 +126,10 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.19.15"), - semver.MustParse("v1.20.11"), - semver.MustParse("v1.21.5"), - semver.MustParse("v1.22.2"), + semver.MustParse("v1.20.14"), + semver.MustParse("v1.21.8"), + semver.MustParse("v1.22.5"), + semver.MustParse("v1.23.0"), } var tests []userDataTestCase @@ -232,7 +232,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.22.5", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -257,7 +257,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.22.5", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -308,7 +308,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.22.5", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -334,7 +334,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.22.5", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -360,7 +360,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.22.5", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -390,7 +390,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.22.5", }, }, ccProvider: &fakeCloudConfigProvider{ diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index 4011665c1..3d21ca6c1 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -153,7 +153,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 19e1119bf..fa0faca4e 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -153,7 +153,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index 70390d074..27d75a820 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 4fa378eec..8a2a594a8 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 8d42cb184..4845acaf2 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -153,7 +153,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index 58fb855ec..4f890536b 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index c6d961936..4e1210d21 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.19.15.yaml b/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.19.15.yaml rename to pkg/userdata/ubuntu/testdata/version-1.20.14.yaml index 86eca89b3..fca5c5cc8 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.19.15.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.19.15}" + KUBE_VERSION="${KUBE_VERSION:-v1.20.14}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.21.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.21.5.yaml rename to pkg/userdata/ubuntu/testdata/version-1.21.8.yaml index 70390d074..5194458b8 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.21.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.2.yaml b/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.22.2.yaml rename to pkg/userdata/ubuntu/testdata/version-1.22.5.yaml index e8edd1526..27d75a820 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.22.2.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.20.11.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml similarity index 98% rename from pkg/userdata/ubuntu/testdata/version-1.20.11.yaml rename to pkg/userdata/ubuntu/testdata/version-1.23.0.yaml index 11b7fb56f..f110deffb 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.20.11.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.20.11}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -222,8 +222,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index 6365262aa..682b6a924 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -161,7 +161,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 16c9b0989..97b4d6b6c 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -161,7 +161,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 1470cc9fe..5c7559782 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -152,7 +152,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 2f6a46d5d..cb38e133a 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -355,7 +355,7 @@ func TestOpenstackProjectAuthProvisioningE2E(t *testing.T) { name: "MachineDeploy with project auth vars", osName: "ubuntu", containerRuntime: "containerd", - kubernetesVersion: "1.21.2", + kubernetesVersion: "1.21.8", executor: verifyCreateAndDelete, } testScenario(t, scenario, *testRunIdentifier, params, OSManifestProjectAuth, false) @@ -425,7 +425,7 @@ func TestAWSAssumeRoleProvisioningE2E(t *testing.T) { name: "AWS with AssumeRole", osName: "ubuntu", containerRuntime: "docker", - kubernetesVersion: "1.19.9", + kubernetesVersion: "1.22.5", executor: verifyCreateAndDelete, } testScenario(t, scenario, *testRunIdentifier, params, AWSManifest, false) @@ -534,7 +534,7 @@ func TestAWSFlatcarContainerdProvisioningE2E(t *testing.T) { name: "flatcar with containerd in AWS", osName: "flatcar", containerRuntime: "containerd", - kubernetesVersion: "1.19.15", + kubernetesVersion: "1.22.5", executor: verifyCreateAndDelete, } testScenario(t, scenario, *testRunIdentifier, params, AWSManifest, false) @@ -584,7 +584,7 @@ func TestAWSEbsEncryptionEnabledProvisioningE2E(t *testing.T) { name: "AWS with ebs encryption enabled", osName: "ubuntu", containerRuntime: "containerd", - kubernetesVersion: "v1.21.5", + kubernetesVersion: "v1.21.8", executor: verifyCreateAndDelete, } testScenario(t, scenario, fmt.Sprintf("aws-%s", *testRunIdentifier), params, AWSEBSEncryptedManifest, false) @@ -667,7 +667,7 @@ func TestAzureRedhatSatelliteProvisioningE2E(t *testing.T) { name: "Azure redhat satellite server subscription", osName: "rhel", containerRuntime: "docker", - kubernetesVersion: "1.21.5", + kubernetesVersion: "1.21.8", executor: verifyCreateAndDelete, } @@ -837,7 +837,7 @@ func TestVsphereResourcePoolProvisioningE2E(t *testing.T) { name: "vSphere resource pool provisioning", osName: "flatcar", containerRuntime: "docker", - kubernetesVersion: "1.22.2", + kubernetesVersion: "1.22.5", executor: verifyCreateAndDelete, } @@ -911,7 +911,7 @@ func TestUbuntuProvisioningWithUpgradeE2E(t *testing.T) { name: "Ubuntu upgrade", osName: "ubuntu", containerRuntime: "docker", - kubernetesVersion: "1.22.2", + kubernetesVersion: "1.22.5", executor: verifyCreateAndDelete, } @@ -936,7 +936,7 @@ func TestDeploymentControllerUpgradesMachineE2E(t *testing.T) { name: "MachineDeployment upgrade", osName: "ubuntu", containerRuntime: "docker", - kubernetesVersion: "1.21.5", + kubernetesVersion: "1.21.8", executor: verifyCreateUpdateAndDelete, } testScenario(t, scenario, *testRunIdentifier, params, HZManifest, false) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index a3b709918..89b8fdc03 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,10 +33,10 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.19.15"), - semver.MustParse("v1.20.11"), - semver.MustParse("v1.21.5"), - semver.MustParse("v1.22.2"), + semver.MustParse("v1.20.14"), + semver.MustParse("v1.21.8"), + semver.MustParse("v1.22.5"), + semver.MustParse("v1.23.0"), } operatingSystems = []providerconfigtypes.OperatingSystem{ diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml index 4e20c5e33..17dd230d4 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml @@ -53,4 +53,4 @@ spec: rhelOrganizationName: "" rhelActivationKey: "" versions: - kubelet: "1.21.5" + kubelet: 1.22.5 diff --git a/test/tools/integration/Makefile b/test/tools/integration/Makefile index c35759c48..84ae9926a 100644 --- a/test/tools/integration/Makefile +++ b/test/tools/integration/Makefile @@ -27,7 +27,7 @@ endif .PHONY: terraform terraform: @if ! which terraform; then \ - curl https://releases.hashicorp.com/terraform/1.0.9/terraform_1.0.9_linux_amd64.zip \ + curl https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_amd64.zip \ --retry 5 \ -o /tmp/terraform.zip && \ unzip -n /tmp/terraform.zip terraform && \ diff --git a/test/tools/integration/master_install_script.sh b/test/tools/integration/master_install_script.sh index 5cdb88336..a507bb52b 100644 --- a/test/tools/integration/master_install_script.sh +++ b/test/tools/integration/master_install_script.sh @@ -17,7 +17,7 @@ set -euo pipefail set -x -K8S_VERSION=1.22.2 +K8S_VERSION=1.23.0 echo "$LC_E2E_SSH_PUBKEY" >> .ssh/authorized_keys # Hetzner's Ubuntu Bionic comes with swap pre-configured, so we force it off. diff --git a/test/tools/integration/versions.tf b/test/tools/integration/versions.tf index 9d0cbdbda..63d4f1c5e 100644 --- a/test/tools/integration/versions.tf +++ b/test/tools/integration/versions.tf @@ -1,9 +1,9 @@ terraform { - required_version = ">= 0.13" + required_version = ">= 1.0.0" required_providers { hcloud = { source = "hetznercloud/hcloud" - version = "~> 1.23.0" + version = "~> 1.31.0" } } } From b85947d8ed49477a40f1416a673dcb7ed9f799c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Thu, 13 Jan 2022 14:59:41 +0100 Subject: [PATCH 061/489] Differentiate CloudProviderName and KubeletCloudProviderName in templates (#1162) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Differentiate CloudProviderName and KubeletCloudProviderName Signed-off-by: Marko Mudrinić * Update templates to use CloudProviderName and KubeletCloudProviderName Signed-off-by: Marko Mudrinić --- pkg/apis/plugin/plugin.go | 29 +++++++++--------- pkg/controller/machine/machine_controller.go | 31 ++++++++++---------- pkg/userdata/amzn2/provider.go | 2 +- pkg/userdata/amzn2/provider_test.go | 21 ++++++------- pkg/userdata/centos/provider.go | 2 +- pkg/userdata/centos/provider_test.go | 21 ++++++------- pkg/userdata/flatcar/provider.go | 4 +-- pkg/userdata/flatcar/provider_test.go | 21 ++++++------- pkg/userdata/rhel/provider.go | 2 +- pkg/userdata/rhel/provider_test.go | 21 ++++++------- pkg/userdata/sles/provider.go | 2 +- pkg/userdata/sles/provider_test.go | 21 ++++++------- pkg/userdata/ubuntu/provider.go | 2 +- pkg/userdata/ubuntu/provider_test.go | 21 ++++++------- 14 files changed, 104 insertions(+), 96 deletions(-) diff --git a/pkg/apis/plugin/plugin.go b/pkg/apis/plugin/plugin.go index 5c5797e4d..ce0a75c0f 100644 --- a/pkg/apis/plugin/plugin.go +++ b/pkg/apis/plugin/plugin.go @@ -41,20 +41,21 @@ const ( // UserDataRequest requests user data with the given arguments. type UserDataRequest struct { - MachineSpec clusterv1alpha1.MachineSpec - Kubeconfig *clientcmdapi.Config - CloudProviderName string - CloudConfig string - DNSIPs []net.IP - ExternalCloudProvider bool - HTTPProxy string - NoProxy string - PauseImage string - KubeletFeatureGates map[string]bool - KubeletConfigs map[string]string - ContainerRuntime containerruntime.Config - PodCIDR string - NodePortRange string + MachineSpec clusterv1alpha1.MachineSpec + Kubeconfig *clientcmdapi.Config + CloudProviderName string + CloudConfig string + DNSIPs []net.IP + ExternalCloudProvider bool + HTTPProxy string + NoProxy string + PauseImage string + KubeletCloudProviderName string + KubeletFeatureGates map[string]bool + KubeletConfigs map[string]string + ContainerRuntime containerruntime.Config + PodCIDR string + NodePortRange string } // UserDataResponse contains the responded user data. diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index b014f3c9c..5a150663b 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -702,7 +702,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( return nil, fmt.Errorf("failed to create bootstrap kubeconfig: %v", err) } - cloudConfig, cloudProviderName, err := prov.GetCloudConfig(machine.Spec) + cloudConfig, kubeletCloudProviderName, err := prov.GetCloudConfig(machine.Spec) if err != nil { return nil, fmt.Errorf("failed to render cloud config: %v", err) } @@ -746,20 +746,21 @@ func (r *Reconciler) ensureInstanceExistsForMachine( crRuntime.RegistryCredentials = registryCredentials req := plugin.UserDataRequest{ - MachineSpec: machine.Spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: cloudProviderName, - ExternalCloudProvider: externalCloudProvider, - DNSIPs: r.nodeSettings.ClusterDNSIPs, - PauseImage: r.nodeSettings.PauseImage, - KubeletFeatureGates: kubeletFeatureGates, - KubeletConfigs: KubeletConfigs, - NoProxy: r.nodeSettings.NoProxy, - HTTPProxy: r.nodeSettings.HTTPProxy, - ContainerRuntime: crRuntime, - PodCIDR: r.podCIDR, - NodePortRange: r.nodePortRange, + MachineSpec: machine.Spec, + Kubeconfig: kubeconfig, + CloudConfig: cloudConfig, + CloudProviderName: string(providerConfig.CloudProvider), + ExternalCloudProvider: externalCloudProvider, + DNSIPs: r.nodeSettings.ClusterDNSIPs, + PauseImage: r.nodeSettings.PauseImage, + KubeletCloudProviderName: kubeletCloudProviderName, + KubeletFeatureGates: kubeletFeatureGates, + KubeletConfigs: KubeletConfigs, + NoProxy: r.nodeSettings.NoProxy, + HTTPProxy: r.nodeSettings.HTTPProxy, + ContainerRuntime: crRuntime, + PodCIDR: r.podCIDR, + NodePortRange: r.nodePortRange, } // Here we do stuff! diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index 05da14cc5..a6ac2cc26 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -246,7 +246,7 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index 2bc45fc5f..c235b8fba 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -241,16 +241,17 @@ func TestUserDataGeneration(t *testing.T) { } req := plugin.UserDataRequest{ - MachineSpec: test.spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: cloudProviderName, - DNSIPs: test.clusterDNSIPs, - ExternalCloudProvider: test.externalCloudProvider, - HTTPProxy: test.httpProxy, - NoProxy: test.noProxy, - PauseImage: test.pauseImage, - KubeletFeatureGates: kubeletFeatureGates, + MachineSpec: test.spec, + Kubeconfig: kubeconfig, + CloudConfig: cloudConfig, + CloudProviderName: cloudProviderName, + KubeletCloudProviderName: cloudProviderName, + DNSIPs: test.clusterDNSIPs, + ExternalCloudProvider: test.externalCloudProvider, + HTTPProxy: test.httpProxy, + NoProxy: test.noProxy, + PauseImage: test.pauseImage, + KubeletFeatureGates: kubeletFeatureGates, ContainerRuntime: containerruntime.Get( test.containerruntime, containerruntime.WithInsecureRegistries(test.insecureRegistries), diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index 9bfeea643..69673693d 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -246,7 +246,7 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index e9ac91ae6..2b4cc44c7 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -241,16 +241,17 @@ func TestUserDataGeneration(t *testing.T) { } req := plugin.UserDataRequest{ - MachineSpec: test.spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: cloudProviderName, - DNSIPs: test.clusterDNSIPs, - ExternalCloudProvider: test.externalCloudProvider, - HTTPProxy: test.httpProxy, - NoProxy: test.noProxy, - PauseImage: test.pauseImage, - KubeletFeatureGates: kubeletFeatureGates, + MachineSpec: test.spec, + Kubeconfig: kubeconfig, + CloudConfig: cloudConfig, + CloudProviderName: cloudProviderName, + KubeletCloudProviderName: cloudProviderName, + DNSIPs: test.clusterDNSIPs, + ExternalCloudProvider: test.externalCloudProvider, + HTTPProxy: test.httpProxy, + NoProxy: test.noProxy, + PauseImage: test.pauseImage, + KubeletFeatureGates: kubeletFeatureGates, ContainerRuntime: containerruntime.Get( test.containerruntime, containerruntime.WithInsecureRegistries(test.insecureRegistries), diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index f54814a29..3bc8ca7b3 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -252,7 +252,7 @@ systemd: Requires=download-script.service After=download-script.service contents: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 8 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 8 }} storage: files: @@ -520,7 +520,7 @@ coreos: Requires=download-script.service After=download-script.service content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 6 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 6 }} - name: apply-sysctl-settings.service enable: true diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index 5a1cf0fa3..530e891af 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -421,16 +421,17 @@ func TestUserDataGeneration(t *testing.T) { } req := plugin.UserDataRequest{ - MachineSpec: test.spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: cloudProviderName, - DNSIPs: test.DNSIPs, - ExternalCloudProvider: test.externalCloudProvider, - HTTPProxy: test.httpProxy, - NoProxy: test.noProxy, - PauseImage: test.pauseImage, - KubeletFeatureGates: kubeletFeatureGates, + MachineSpec: test.spec, + Kubeconfig: kubeconfig, + CloudConfig: cloudConfig, + CloudProviderName: cloudProviderName, + KubeletCloudProviderName: cloudProviderName, + DNSIPs: test.DNSIPs, + ExternalCloudProvider: test.externalCloudProvider, + HTTPProxy: test.httpProxy, + NoProxy: test.noProxy, + PauseImage: test.pauseImage, + KubeletFeatureGates: kubeletFeatureGates, ContainerRuntime: containerruntime.Get( test.containerruntime, containerruntime.WithInsecureRegistries(test.insecureRegistries), diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 29fa0a540..4f53467a5 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -258,7 +258,7 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index 5390131ca..c381c13ea 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -241,16 +241,17 @@ func TestUserDataGeneration(t *testing.T) { } req := plugin.UserDataRequest{ - MachineSpec: test.spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: cloudProviderName, - DNSIPs: test.clusterDNSIPs, - ExternalCloudProvider: test.externalCloudProvider, - HTTPProxy: test.httpProxy, - NoProxy: test.noProxy, - PauseImage: test.pauseImage, - KubeletFeatureGates: kubeletFeatureGates, + MachineSpec: test.spec, + Kubeconfig: kubeconfig, + CloudConfig: cloudConfig, + CloudProviderName: cloudProviderName, + KubeletCloudProviderName: cloudProviderName, + DNSIPs: test.clusterDNSIPs, + ExternalCloudProvider: test.externalCloudProvider, + HTTPProxy: test.httpProxy, + NoProxy: test.noProxy, + PauseImage: test.pauseImage, + KubeletFeatureGates: kubeletFeatureGates, ContainerRuntime: containerruntime.Get( test.containerruntime, containerruntime.WithInsecureRegistries(test.insecureRegistries), diff --git a/pkg/userdata/sles/provider.go b/pkg/userdata/sles/provider.go index a3a5583e7..4ccdb75b6 100644 --- a/pkg/userdata/sles/provider.go +++ b/pkg/userdata/sles/provider.go @@ -202,7 +202,7 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntime.String .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntime.String .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} - path: "/etc/systemd/system/kubelet.service.d/extras.conf" content: | diff --git a/pkg/userdata/sles/provider_test.go b/pkg/userdata/sles/provider_test.go index 794ead9bd..dba3dfccf 100644 --- a/pkg/userdata/sles/provider_test.go +++ b/pkg/userdata/sles/provider_test.go @@ -438,16 +438,17 @@ func TestUserDataGeneration(t *testing.T) { } req := plugin.UserDataRequest{ - MachineSpec: test.spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: cloudProviderName, - DNSIPs: test.DNSIPs, - ExternalCloudProvider: test.externalCloudProvider, - HTTPProxy: test.httpProxy, - NoProxy: test.noProxy, - PauseImage: test.pauseImage, - KubeletFeatureGates: kubeletFeatureGates, + MachineSpec: test.spec, + Kubeconfig: kubeconfig, + CloudConfig: cloudConfig, + CloudProviderName: cloudProviderName, + KubeletCloudProviderName: cloudProviderName, + DNSIPs: test.DNSIPs, + ExternalCloudProvider: test.externalCloudProvider, + HTTPProxy: test.httpProxy, + NoProxy: test.noProxy, + PauseImage: test.pauseImage, + KubeletFeatureGates: kubeletFeatureGates, ContainerRuntime: containerruntime.Get( test.containerruntime, containerruntime.WithInsecureRegistries(test.insecureRegistries), diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index 803275db8..5d26595a1 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -247,7 +247,7 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} - path: "/etc/systemd/system/kubelet.service.d/extras.conf" content: | diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 7fd043e32..c1229bcd8 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -474,16 +474,17 @@ func TestUserDataGeneration(t *testing.T) { } req := plugin.UserDataRequest{ - MachineSpec: test.spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: cloudProviderName, - DNSIPs: test.DNSIPs, - ExternalCloudProvider: test.externalCloudProvider, - HTTPProxy: test.httpProxy, - NoProxy: test.noProxy, - PauseImage: test.pauseImage, - KubeletFeatureGates: kubeletFeatureGates, + MachineSpec: test.spec, + Kubeconfig: kubeconfig, + CloudConfig: cloudConfig, + CloudProviderName: cloudProviderName, + KubeletCloudProviderName: cloudProviderName, + DNSIPs: test.DNSIPs, + ExternalCloudProvider: test.externalCloudProvider, + HTTPProxy: test.httpProxy, + NoProxy: test.noProxy, + PauseImage: test.pauseImage, + KubeletFeatureGates: kubeletFeatureGates, ContainerRuntime: containerruntime.Get( test.containerruntime, containerruntime.WithInsecureRegistries(test.insecureRegistries), From 430239078e6b3fd61063ac39b4e74c930858f8cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Thu, 13 Jan 2022 16:26:44 +0100 Subject: [PATCH 062/489] Install and enable iscsid on Nutanix machines (#1161) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Install and enable iscsid on Nutanix machines Signed-off-by: Marko Mudrinić * Update fixtures Signed-off-by: Marko Mudrinić * Fix YAML lint error Signed-off-by: Marko Mudrinić --- pkg/userdata/centos/provider.go | 9 +- pkg/userdata/centos/provider_test.go | 10 + .../kubelet-containerd-v1.20-aws.yaml | 1 - .../centos/testdata/kubelet-v1.20-aws.yaml | 1 - .../testdata/kubelet-v1.21-aws-external.yaml | 1 - .../centos/testdata/kubelet-v1.21-aws.yaml | 1 - .../testdata/kubelet-v1.21-nutanix.yaml | 429 ++++++++++++++++ .../kubelet-v1.21-vsphere-mirrors.yaml | 1 - .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 1 - .../testdata/kubelet-v1.21-vsphere.yaml | 1 - .../centos/testdata/kubelet-v1.22-aws.yaml | 1 - .../centos/testdata/kubelet-v1.23-aws.yaml | 1 - pkg/userdata/rhel/provider.go | 8 + pkg/userdata/rhel/provider_test.go | 10 + .../rhel/testdata/kubelet-v1.22-nutanix.yaml | 465 ++++++++++++++++++ pkg/userdata/ubuntu/provider.go | 8 + pkg/userdata/ubuntu/provider_test.go | 26 + pkg/userdata/ubuntu/testdata/nutanix.yaml | 440 +++++++++++++++++ 18 files changed, 1404 insertions(+), 10 deletions(-) create mode 100644 pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml create mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml create mode 100644 pkg/userdata/ubuntu/testdata/nutanix.yaml diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index 69673693d..57973aa6c 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -220,8 +220,15 @@ write_files: {{- if eq .CloudProviderName "vsphere" }} open-vm-tools \ {{- end }} + {{- if eq .CloudProviderName "nutanix" }} + iscsi-initiator-utils \ + {{- end }} ipvsadm - + + {{- /* iscsid service is required on Nutanix machines for CSI driver to attach volumes. */}} + {{- if eq .CloudProviderName "nutanix" }} + systemctl enable --now iscsid + {{ end }} {{ .ContainerRuntimeScript | indent 4 }} {{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index 2b4cc44c7..86f30751d 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -193,6 +193,16 @@ func TestUserDataGeneration(t *testing.T) { }, }, }, + { + name: "kubelet-v1.21-nutanix", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.21.5", + }, + }, + cloudProviderName: stringPtr("nutanix"), + }, } defaultCloudProvider := &fakeCloudConfigProvider{ diff --git a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml index dc062671c..4a2bc488f 100644 --- a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml @@ -80,7 +80,6 @@ write_files: curl \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml index 609bdc495..71417c801 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml @@ -80,7 +80,6 @@ write_files: curl \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml index 4e417e4fc..eb9c0e540 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml @@ -80,7 +80,6 @@ write_files: curl \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml index f2eaa17c2..e95d9e99b 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml @@ -80,7 +80,6 @@ write_files: curl \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml new file mode 100644 index 000000000..bee7b653e --- /dev/null +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml @@ -0,0 +1,429 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + hostnamectl set-hostname node1 + + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + iscsi-initiator-utils \ + ipvsadm + systemctl enable --now iscsid + + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=nutanix \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: {} + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 096bd7459..238c7ebf6 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -93,7 +93,6 @@ write_files: open-vm-tools \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml index def43e8c0..04783408f 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -93,7 +93,6 @@ write_files: open-vm-tools \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml index 214252946..d7fd521f7 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml @@ -85,7 +85,6 @@ write_files: open-vm-tools \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index 19f34f449..ba9391fc1 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -80,7 +80,6 @@ write_files: curl \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index b5015b267..642ff989e 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -80,7 +80,6 @@ write_files: curl \ ipvsadm - yum install -y yum-utils yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 4f53467a5..411ebd32b 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -225,7 +225,15 @@ write_files: {{- if eq .CloudProviderName "vsphere" }} open-vm-tools \ {{- end }} + {{- if eq .CloudProviderName "nutanix" }} + iscsi-initiator-utils \ + {{- end }} ipvsadm + + {{- /* iscsid service is required on Nutanix machines for CSI driver to attach volumes. */}} + {{- if eq .CloudProviderName "nutanix" }} + systemctl enable --now iscsid + {{ end }} {{ .ContainerRuntimeScript | indent 4 }} {{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} # set kubelet nodeip environment variable diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index c381c13ea..08d37b427 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -193,6 +193,16 @@ func TestUserDataGeneration(t *testing.T) { registryMirrors: map[string][]string{"docker.io": {"https://registry.docker-cn.com/"}}, pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, + { + name: "kubelet-v1.22-nutanix", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.22.2", + }, + }, + cloudProviderName: stringPtr("nutanix"), + }, } defaultCloudProvider := &fakeCloudConfigProvider{ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml new file mode 100644 index 000000000..a95765679 --- /dev/null +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml @@ -0,0 +1,465 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + +hostname: node1 +fqdn: node1 + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + hostnamectl set-hostname node1 + + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + iscsi-initiator-utils \ + ipvsadm + systemctl enable --now iscsid + + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=nutanix \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: {} + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + +rh_subscription: + username: "" + password: "" + auto-attach: false + +runcmd: +- systemctl start setup.service +- systemctl start disable-nm-cloud-setup.service diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index 5d26595a1..e9bcff1e1 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -215,7 +215,15 @@ write_files: {{- if eq .CloudProviderName "vsphere" }} open-vm-tools \ {{- end }} + {{- if eq .CloudProviderName "nutanix" }} + open-iscsi \ + {{- end }} ipvsadm + + {{- /* iscsid service is required on Nutanix machines for CSI driver to attach volumes. */}} + {{- if eq .CloudProviderName "nutanix" }} + systemctl enable --now iscsid + {{ end }} # Update grub to include kernel command options to enable swap accounting. # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index c1229bcd8..0962a064d 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -444,6 +444,32 @@ func TestUserDataGeneration(t *testing.T) { DistUpgradeOnBoot: true, }, }, + { + name: "nutanix", + providerSpec: &providerconfigtypes.Config{ + CloudProvider: "nutanix", + SSHPublicKeys: []string{"ssh-rsa AAABBB"}, + OverwriteCloudConfig: stringPtr("custom\ncloud\nconfig"), + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.21.5", + }, + }, + ccProvider: &fakeCloudConfigProvider{ + name: "nutanix", + config: "{nutanix-config:true}", + err: nil, + }, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, + kubernetesCACert: "CACert", + osConfig: &Config{ + DistUpgradeOnBoot: false, + }, + }, }...) for _, test := range tests { diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml new file mode 100644 index 000000000..234501799 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -0,0 +1,440 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + open-iscsi \ + ipvsadm + systemctl enable --now iscsid + + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=nutanix \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + custom + cloud + config + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: {} + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service From 30453d23daa3bb49ae16eccc2c1217eca9da0000 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Thu, 13 Jan 2022 16:57:43 +0100 Subject: [PATCH 063/489] Actually read environment variable in GetConfigVarBoolValueOrEnv (#1154) * Make GetConfigVarBoolValueOrEnv actually read environment variable Signed-off-by: Marvin Beckers * Use Valid attribute to differentiate between 'false' and unset value Signed-off-by: Marvin Beckers * Make CI happy Signed-off-by: Marvin Beckers * Convert ConfigVarBool.Value to pointer Signed-off-by: Marvin Beckers * Handle errors Signed-off-by: Marvin Beckers * Use json.Unmarshal to parse *bool Signed-off-by: Marvin Beckers * Do not trigger linode job Signed-off-by: Marvin Beckers --- .prow.yaml | 1 - pkg/cloudprovider/provider/aws/provider.go | 4 +- pkg/cloudprovider/provider/azure/provider.go | 2 +- .../provider/digitalocean/provider.go | 8 +-- pkg/cloudprovider/provider/gce/config.go | 8 +-- pkg/cloudprovider/provider/linode/provider.go | 4 +- .../provider/openstack/provider.go | 2 +- .../provider/scaleway/provider.go | 2 +- pkg/providerconfig/types.go | 69 ++++++++++++------- pkg/providerconfig/types/types.go | 26 ++++--- pkg/providerconfig/types/types_test.go | 15 ++-- 11 files changed, 87 insertions(+), 54 deletions(-) diff --git a/.prow.yaml b/.prow.yaml index 98975b3a6..450502805 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -475,7 +475,6 @@ presubmits: - name: pull-machine-controller-e2e-linode always_run: false - run_if_changed: pkg\/cloudprovider\/provider\/linode\/.* optional: true decorate: true error_on_eviction: true diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index ceda33dac..1381be5fe 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -436,7 +436,7 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt c.DiskIops = rawConfig.DiskIops } - c.EBSVolumeEncrypted, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.EBSVolumeEncrypted) + c.EBSVolumeEncrypted, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.EBSVolumeEncrypted) if err != nil { return nil, nil, nil, fmt.Errorf("failed to get ebsVolumeEncrypted value: %v", err) } @@ -450,7 +450,7 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt } c.SpotMaxPrice = pointer.StringPtr(maxPrice) - persistentRequest, err := p.configVarResolver.GetConfigVarBoolValue(rawConfig.SpotInstanceConfig.PersistentRequest) + persistentRequest, _, err := p.configVarResolver.GetConfigVarBoolValue(rawConfig.SpotInstanceConfig.PersistentRequest) if err != nil { return nil, nil, nil, err } diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 558d42f99..a4c707b14 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -271,7 +271,7 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*config, *providerconfigt return nil, nil, fmt.Errorf("failed to get the value of \"routeTableName\" field, error = %v", err) } - c.AssignPublicIP, err = p.configVarResolver.GetConfigVarBoolValue(rawCfg.AssignPublicIP) + c.AssignPublicIP, _, err = p.configVarResolver.GetConfigVarBoolValue(rawCfg.AssignPublicIP) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"assignPublicIP\" field, error = %v", err) } diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index 61cf5fef1..81e99d99b 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -133,19 +133,19 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt if err != nil { return nil, nil, err } - c.Backups, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.Backups) + c.Backups, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.Backups) if err != nil { return nil, nil, err } - c.IPv6, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.IPv6) + c.IPv6, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.IPv6) if err != nil { return nil, nil, err } - c.PrivateNetworking, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.PrivateNetworking) + c.PrivateNetworking, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.PrivateNetworking) if err != nil { return nil, nil, err } - c.Monitoring, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.Monitoring) + c.Monitoring, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.Monitoring) if err != nil { return nil, nil, err } diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index 1c70e4007..092e2ff1d 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -161,7 +161,7 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide return nil, fmt.Errorf("cannot retrieve subnetwork: %v", err) } - cfg.preemptible, err = resolver.GetConfigVarBoolValue(cpSpec.Preemptible) + cfg.preemptible, _, err = resolver.GetConfigVarBoolValue(cpSpec.Preemptible) if err != nil { return nil, fmt.Errorf("cannot retrieve preemptible: %v", err) } @@ -170,18 +170,18 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide cfg.assignPublicIPAddress = true if cpSpec.AssignPublicIPAddress != nil { - cfg.assignPublicIPAddress, err = resolver.GetConfigVarBoolValue(*cpSpec.AssignPublicIPAddress) + cfg.assignPublicIPAddress, _, err = resolver.GetConfigVarBoolValue(*cpSpec.AssignPublicIPAddress) if err != nil { return nil, fmt.Errorf("failed to retrieve assignPublicIPAddress: %v", err) } } - cfg.multizone, err = resolver.GetConfigVarBoolValue(cpSpec.MultiZone) + cfg.multizone, _, err = resolver.GetConfigVarBoolValue(cpSpec.MultiZone) if err != nil { return nil, fmt.Errorf("failed to retrieve multizone: %v", err) } - cfg.regional, err = resolver.GetConfigVarBoolValue(cpSpec.Regional) + cfg.regional, _, err = resolver.GetConfigVarBoolValue(cpSpec.Regional) if err != nil { return nil, fmt.Errorf("failed to retrieve regional: %v", err) } diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index 1692757da..dbcd01c0d 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -140,11 +140,11 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt if err != nil { return nil, nil, err } - c.Backups, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.Backups) + c.Backups, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.Backups) if err != nil { return nil, nil, err } - c.PrivateNetworking, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.PrivateNetworking) + c.PrivateNetworking, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.PrivateNetworking) if err != nil { return nil, nil, err } diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 8f60f6d69..972e57b39 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -255,7 +255,7 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt if err != nil { return nil, nil, nil, err } - c.TrustDevicePath, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.TrustDevicePath) + c.TrustDevicePath, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.TrustDevicePath) if err != nil { return nil, nil, nil, err } diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go index 02d428ad8..1f1738293 100644 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ b/pkg/cloudprovider/provider/scaleway/provider.go @@ -127,7 +127,7 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt if err != nil { return nil, nil, err } - c.IPv6, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.IPv6) + c.IPv6, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.IPv6) if err != nil { return nil, nil, err } diff --git a/pkg/providerconfig/types.go b/pkg/providerconfig/types.go index a1380b850..a984e21c1 100644 --- a/pkg/providerconfig/types.go +++ b/pkg/providerconfig/types.go @@ -116,37 +116,60 @@ func (cvr *ConfigVarResolver) GetConfigVarStringValueOrEnv(configVar providercon return envVal, nil } -func (cvr *ConfigVarResolver) GetConfigVarBoolValue(configVar providerconfigtypes.ConfigVarBool) (bool, error) { - cvs := providerconfigtypes.ConfigVarString{Value: strconv.FormatBool(configVar.Value), SecretKeyRef: configVar.SecretKeyRef} - stringVal, err := cvr.GetConfigVarStringValue(cvs) - if err != nil { - return false, err +// GetConfigVarBoolValue returns a boolean from a ConfigVarBool. If there is no valid source for the boolean, +// the second bool returned will be false (to be able to differentiate between "false" and "unset") +func (cvr *ConfigVarResolver) GetConfigVarBoolValue(configVar providerconfigtypes.ConfigVarBool) (bool, bool, error) { + // We need all three of these to fetch and use a secret + if configVar.SecretKeyRef.Name != "" && configVar.SecretKeyRef.Namespace != "" && configVar.SecretKeyRef.Key != "" { + secret := &corev1.Secret{} + name := types.NamespacedName{Namespace: configVar.SecretKeyRef.Namespace, Name: configVar.SecretKeyRef.Name} + if err := cvr.client.Get(cvr.ctx, name, secret); err != nil { + return false, false, fmt.Errorf("error retrieving secret '%s' from namespace '%s': '%v'", configVar.SecretKeyRef.Name, configVar.SecretKeyRef.Namespace, err) + } + if val, ok := secret.Data[configVar.SecretKeyRef.Key]; ok { + boolVal, err := strconv.ParseBool(string(val)) + return boolVal, (err == nil), err + } + return false, false, fmt.Errorf("secret '%s' in namespace '%s' has no key '%s'", configVar.SecretKeyRef.Name, configVar.SecretKeyRef.Namespace, configVar.SecretKeyRef.Key) } - boolVal, err := strconv.ParseBool(stringVal) - if err != nil { - return false, err + + // We need all three of these to fetch and use a configmap + if configVar.ConfigMapKeyRef.Name != "" && configVar.ConfigMapKeyRef.Namespace != "" && configVar.ConfigMapKeyRef.Key != "" { + configMap := &corev1.ConfigMap{} + name := types.NamespacedName{Namespace: configVar.ConfigMapKeyRef.Namespace, Name: configVar.ConfigMapKeyRef.Name} + if err := cvr.client.Get(cvr.ctx, name, configMap); err != nil { + return false, false, fmt.Errorf("error retrieving configmap '%s' from namespace '%s': '%v'", configVar.ConfigMapKeyRef.Name, configVar.ConfigMapKeyRef.Namespace, err) + } + if val, ok := configMap.Data[configVar.ConfigMapKeyRef.Key]; ok { + boolVal, err := strconv.ParseBool(val) + return boolVal, (err == nil), err + } + return false, false, fmt.Errorf("configmap '%s' in namespace '%s' has no key '%s'", configVar.ConfigMapKeyRef.Name, configVar.ConfigMapKeyRef.Namespace, configVar.ConfigMapKeyRef.Key) + } + + if configVar.Value == nil { + return false, false, nil } - return boolVal, nil + + return configVar.Value != nil && *configVar.Value, true, nil } func (cvr *ConfigVarResolver) GetConfigVarBoolValueOrEnv(configVar providerconfigtypes.ConfigVarBool, envVarName string) (bool, error) { - cvs := providerconfigtypes.ConfigVarString{Value: strconv.FormatBool(configVar.Value), SecretKeyRef: configVar.SecretKeyRef} - stringVal, err := cvr.GetConfigVarStringValue(cvs) - if err != nil { - return false, err + boolVal, valid, err := cvr.GetConfigVarBoolValue(configVar) + if valid && err == nil { + return boolVal, nil } - if stringVal == "" { - envVal, envValFound := os.LookupEnv(envVarName) - if !envValFound { - return false, fmt.Errorf("all mechanisms(value, secret, configMap) of getting the value failed, including reading from environment variable = %s which was not set", envVarName) + + envVal, envValFound := os.LookupEnv(envVarName) + if envValFound { + envValBool, err := strconv.ParseBool(envVal) + if err != nil { + return false, err } - stringVal = envVal + return envValBool, nil } - boolVal, err := strconv.ParseBool(stringVal) - if err != nil { - return false, err - } - return boolVal, nil + + return false, nil } func NewConfigVarResolver(ctx context.Context, client ctrlruntimeclient.Client) *ConfigVarResolver { diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 340e0a72b..186dc8262 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -21,7 +21,6 @@ import ( "encoding/json" "errors" "fmt" - "strconv" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -221,7 +220,7 @@ func (configVarString *ConfigVarString) UnmarshalJSON(b []byte) error { } type ConfigVarBool struct { - Value bool `json:"value,omitempty"` + Value *bool `json:"value,omitempty"` SecretKeyRef GlobalSecretKeySelector `json:"secretKeyRef,omitempty"` ConfigMapKeyRef GlobalConfigMapKeySelector `json:"configMapKeyRef,omitempty"` } @@ -247,7 +246,11 @@ func (configVarBool ConfigVarBool) MarshalJSON() ([]byte, error) { } if secretKeyRefEmpty && configMapKeyRefEmpty { - return []byte(fmt.Sprintf("%v", configVarBool.Value)), nil + jsonVal, err := json.Marshal(configVarBool.Value) + if err != nil { + return []byte{}, err + } + return []byte(fmt.Sprintf("%v", string(jsonVal))), nil } buffer := bytes.NewBufferString("{") @@ -271,20 +274,27 @@ func (configVarBool ConfigVarBool) MarshalJSON() ([]byte, error) { buffer.WriteString(fmt.Sprintf(`%s"configMapKeyRef":%s`, leadingComma, jsonVal)) } - buffer.WriteString(fmt.Sprintf(`,"value":%v}`, configVarBool.Value)) + jsonVal, err := json.Marshal(configVarBool.Value) + if err != nil { + return []byte{}, err + } + + buffer.WriteString(fmt.Sprintf(`,"value":%v}`, string(jsonVal))) return buffer.Bytes(), nil } func (configVarBool *ConfigVarBool) UnmarshalJSON(b []byte) error { if !bytes.HasPrefix(b, []byte("{")) { - value, err := strconv.ParseBool(string(b)) - if err != nil { - return fmt.Errorf("Error converting string to bool: '%v'", err) + var val *bool + if err := json.Unmarshal(b, &val); err != nil { + return fmt.Errorf("Error parsing value: '%v'", err) } - configVarBool.Value = value + configVarBool.Value = val + return nil } + var cvbDummy configVarBoolWithoutUnmarshaller err := json.Unmarshal(b, &cvbDummy) if err != nil { diff --git a/pkg/providerconfig/types/types_test.go b/pkg/providerconfig/types/types_test.go index dad0b6559..2e0ac8971 100644 --- a/pkg/providerconfig/types/types_test.go +++ b/pkg/providerconfig/types/types_test.go @@ -22,6 +22,7 @@ import ( "testing" "k8s.io/api/core/v1" + "k8s.io/utils/pointer" ) func TestConfigVarStringUnmarshalling(t *testing.T) { @@ -48,7 +49,7 @@ func TestConfigVarBoolUnmarshalling(t *testing.T) { jsonBool := []byte("true") jsonMapBool := []byte(`{"value":true}`) - expectedResult := ConfigVarBool{Value: true} + expectedResult := ConfigVarBool{Value: pointer.Bool(true)} var jsonBoolTarget ConfigVarBool var jsonMapBoolTarget ConfigVarBool @@ -97,12 +98,12 @@ func TestConfigVarBoolMarshalling(t *testing.T) { expected string }{ { - cvb: ConfigVarBool{Value: true}, + cvb: ConfigVarBool{Value: pointer.Bool(true)}, expected: `true`, }, { cvb: ConfigVarBool{SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, - expected: `{"secretKeyRef":{"namespace":"ns","name":"name","key":"key"},"value":false}`, + expected: `{"secretKeyRef":{"namespace":"ns","name":"name","key":"key"},"value":null}`, }, } @@ -155,17 +156,17 @@ func TestConfigVarStringMarshallingAndUnmarshalling(t *testing.T) { func TestConfigVarBoolMarshallingAndUnmarshalling(t *testing.T) { testCases := []ConfigVarBool{ - {Value: true}, + {Value: pointer.Bool(true)}, {SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, - {Value: true, SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, + {Value: pointer.Bool(true), SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, {ConfigMapKeyRef: GlobalConfigMapKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, - {Value: true, ConfigMapKeyRef: GlobalConfigMapKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, + {Value: pointer.Bool(true), ConfigMapKeyRef: GlobalConfigMapKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, { ConfigMapKeyRef: GlobalConfigMapKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, }, { - Value: true, + Value: pointer.Bool(true), ConfigMapKeyRef: GlobalConfigMapKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, }, From eae402697b99befc0d356ee0b5c47f48222a63c0 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Thu, 13 Jan 2022 19:13:43 +0100 Subject: [PATCH 064/489] Omit null value from marshalled JSON and add more tests (#1165) Signed-off-by: Marvin Beckers --- pkg/providerconfig/types/types.go | 14 +++-- pkg/providerconfig/types/types_test.go | 73 ++++++++++++++++++++------ 2 files changed, 67 insertions(+), 20 deletions(-) diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 186dc8262..6bf7063c5 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -250,7 +250,7 @@ func (configVarBool ConfigVarBool) MarshalJSON() ([]byte, error) { if err != nil { return []byte{}, err } - return []byte(fmt.Sprintf("%v", string(jsonVal))), nil + return jsonVal, nil } buffer := bytes.NewBufferString("{") @@ -274,12 +274,16 @@ func (configVarBool ConfigVarBool) MarshalJSON() ([]byte, error) { buffer.WriteString(fmt.Sprintf(`%s"configMapKeyRef":%s`, leadingComma, jsonVal)) } - jsonVal, err := json.Marshal(configVarBool.Value) - if err != nil { - return []byte{}, err + if configVarBool.Value != nil { + jsonVal, err := json.Marshal(configVarBool.Value) + if err != nil { + return []byte{}, err + } + + buffer.WriteString(fmt.Sprintf(`,"value":%v`, string(jsonVal))) } - buffer.WriteString(fmt.Sprintf(`,"value":%v}`, string(jsonVal))) + buffer.WriteString("}") return buffer.Bytes(), nil } diff --git a/pkg/providerconfig/types/types_test.go b/pkg/providerconfig/types/types_test.go index 2e0ac8971..1ee107966 100644 --- a/pkg/providerconfig/types/types_test.go +++ b/pkg/providerconfig/types/types_test.go @@ -46,21 +46,50 @@ func TestConfigVarStringUnmarshalling(t *testing.T) { } func TestConfigVarBoolUnmarshalling(t *testing.T) { - jsonBool := []byte("true") - jsonMapBool := []byte(`{"value":true}`) - - expectedResult := ConfigVarBool{Value: pointer.Bool(true)} - - var jsonBoolTarget ConfigVarBool - var jsonMapBoolTarget ConfigVarBool - - err := json.Unmarshal(jsonBool, &jsonBoolTarget) - if err != nil || !reflect.DeepEqual(expectedResult, jsonBoolTarget) { - t.Fatalf("Decoding raw bool into configVarBool failed! Error: '%v'", err) + testCases := []struct { + jsonString string + expected ConfigVarBool + }{ + { + jsonString: "true", + expected: ConfigVarBool{Value: pointer.Bool(true)}, + }, + { + jsonString: `{"value":true}`, + expected: ConfigVarBool{Value: pointer.Bool(true)}, + }, + { + jsonString: "null", + expected: ConfigVarBool{}, + }, + { + jsonString: `{"value":null}`, + expected: ConfigVarBool{}, + }, + { + jsonString: `{"secretKeyRef":{"namespace":"ns","name":"name","key":"key"}}`, + expected: ConfigVarBool{Value: nil, SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, + }, + { + jsonString: `{"value": null, "secretKeyRef":{"namespace":"ns","name":"name","key":"key"}}`, + expected: ConfigVarBool{Value: nil, SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, + }, + { + jsonString: `{"value":false, "secretKeyRef":{"namespace":"ns","name":"name","key":"key"}}`, + expected: ConfigVarBool{Value: pointer.Bool(false), SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, + }, + { + jsonString: `{"value":true, "secretKeyRef":{"namespace":"ns","name":"name","key":"key"}}`, + expected: ConfigVarBool{Value: pointer.Bool(true), SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, + }, } - err = json.Unmarshal(jsonMapBool, &jsonMapBoolTarget) - if err != nil || !reflect.DeepEqual(expectedResult, jsonMapBoolTarget) { - t.Fatalf("Decoding map bool into configVarBool failed! Error: '%v'", err) + + for _, testCase := range testCases { + var cvb ConfigVarBool + err := json.Unmarshal([]byte(testCase.jsonString), &cvb) + if err != nil || !reflect.DeepEqual(testCase.expected, cvb) { + t.Fatalf("Decoding '%s' into configVarBool failed! Error: '%v'", testCase.jsonString, err) + } } } @@ -97,13 +126,25 @@ func TestConfigVarBoolMarshalling(t *testing.T) { cvb ConfigVarBool expected string }{ + { + cvb: ConfigVarBool{}, + expected: `null`, + }, { cvb: ConfigVarBool{Value: pointer.Bool(true)}, expected: `true`, }, { cvb: ConfigVarBool{SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, - expected: `{"secretKeyRef":{"namespace":"ns","name":"name","key":"key"},"value":null}`, + expected: `{"secretKeyRef":{"namespace":"ns","name":"name","key":"key"}}`, + }, + { + cvb: ConfigVarBool{SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, Value: pointer.Bool(true)}, + expected: `{"secretKeyRef":{"namespace":"ns","name":"name","key":"key"},"value":true}`, + }, + { + cvb: ConfigVarBool{SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, Value: pointer.Bool(false)}, + expected: `{"secretKeyRef":{"namespace":"ns","name":"name","key":"key"},"value":false}`, }, } @@ -156,6 +197,8 @@ func TestConfigVarStringMarshallingAndUnmarshalling(t *testing.T) { func TestConfigVarBoolMarshallingAndUnmarshalling(t *testing.T) { testCases := []ConfigVarBool{ + {}, + {Value: pointer.Bool(false)}, {Value: pointer.Bool(true)}, {SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, {Value: pointer.Bool(true), SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, From fe12bf526fd9a1b511e66e4e96a0e1a2923456cb Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Thu, 13 Jan 2022 21:48:39 +0100 Subject: [PATCH 065/489] set the machine_controller_workers metric to an actual value (#1163) --- pkg/controller/machine/machine_controller.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 5a150663b..0f2469951 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -216,6 +216,8 @@ func Add( return err } + metrics.Workers.Set(float64(numWorkers)) + return c.Watch( &source.Kind{Type: &corev1.Node{}}, handler.EnqueueRequestsFromMapFunc(func(node client.Object) (result []reconcile.Request) { From 4e40ef0c1cf10dafa4cd7ae41c077c6bef8ffc9d Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 18 Jan 2022 16:39:49 +0100 Subject: [PATCH 066/489] bump osm to 0.3.9 (#1168) Signed-off-by: Moath Qasim --- go.mod | 2 +- go.sum | 502 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 503 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 365447137..c6806a3f1 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( google.golang.org/grpc v1.38.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b - k8c.io/operating-system-manager v0.3.6 + k8c.io/operating-system-manager v0.3.9 k8s.io/api v0.22.2 k8s.io/apiextensions-apiserver v0.22.2 k8s.io/apimachinery v0.22.2 diff --git a/go.sum b/go.sum index a04aea637..45af0b18d 100644 --- a/go.sum +++ b/go.sum @@ -26,9 +26,12 @@ cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNF cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.1.0 h1:9x7Bx0A9R5/M9jibeJeZWqjeVEIxYW9fZYqB9a70/bY= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/logging v1.0.0/go.mod h1:V1cc3ogwobYzQq5f2R7DS/GvRIrI4FKj01Gs5glwAls= cloud.google.com/go/logging v1.1.2 h1:KNALX0NZn8UJhqKnqoHxhMqyoZfBZoh5wF7CQJZ5XrU= @@ -36,6 +39,7 @@ cloud.google.com/go/logging v1.1.2/go.mod h1:KrljuAHIw631j9+QXsnq9vDwsrwmdxfGpiv cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1 h1:ukjixP1wl0LpnZ6LWtZJ0mX5tBmjp1f8Sqer8Z2OMUU= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= @@ -43,14 +47,21 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +code.cloudfoundry.org/go-pubsub v0.0.0-20180503211407-becd51dc37cb h1:N9iWwP+UaQWj0GB53fIEZTD/qK/0EoJjfIS+YUiRU5I= code.cloudfoundry.org/go-pubsub v0.0.0-20180503211407-becd51dc37cb/go.mod h1:2mohpzdn59JWHT85lXjjglNpGLF51tk6hHqfxpc0utk= +contrib.go.opencensus.io/exporter/ocagent v0.4.12 h1:jGFvw3l57ViIVEPKKEUXPcLYIXJmQxLUh6ey1eJhwyc= contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= +contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg= contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= +contrib.go.opencensus.io/exporter/stackdriver v0.12.8 h1:iXI5hr7pUwMx0IwMphpKz5Q3If/G5JiWFVZ5MPPxP9E= contrib.go.opencensus.io/exporter/stackdriver v0.12.8/go.mod h1:XyyafDnFOsqoxHJgTFycKZMrRUrPThLh2iYTJF6uoO0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +git.apache.org/thrift.git v0.12.0 h1:CMxsZlAmxKs+VAZMlDDL0wXciMblJcutQbEe3A9CYUM= git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= +github.com/Azure/azure-pipeline-go v0.1.9 h1:u7JFb9fFTE6Y/j8ae2VK33ePrRqJqoCM/IWkQdAZ+rg= github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-sdk-for-go v28.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v35.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -58,9 +69,11 @@ github.com/Azure/azure-sdk-for-go v38.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9mo github.com/Azure/azure-sdk-for-go v46.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v49.0.0+incompatible h1:rvYYNgKNBwoxUaBFmd/+TpW3qrd805EHBBvUp5FmFso= github.com/Azure/azure-sdk-for-go v49.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804 h1:QjGHsWFbJyl312t0BtgkmZy2TTYA++FF0UakGbr3ZhQ= github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -116,13 +129,20 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher v0.0.0-20191203181535-308b93ad1f39 h1:Pjo3SOZigEnIGevhFqcbFndnqyCH8WimcREd3hRM9vU= github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher v0.0.0-20191203181535-308b93ad1f39/go.mod h1:yfGmCjKuUzk9WzubMlW2zwjhCraIc/J+M40cufdemRM= +github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534 h1:N7lSsF+R7wSulUADi36SInSQA3RvfO/XclHQfedr0qk= github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14= +github.com/GoogleCloudPlatform/testgrid v0.0.1-alpha.4 h1:TxlW95CqGjvJmW6trWh1XgKZP8GOIgkJEKAUBX/DQ4Q= github.com/GoogleCloudPlatform/testgrid v0.0.1-alpha.4/go.mod h1:f96W2HYy3tiBNV5zbbRc+NczwYHgG1PHXMQfoEWv680= +github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -136,12 +156,16 @@ github.com/Masterminds/sprig/v3 v3.1.0/go.mod h1:ONGMf7UfYGAbMXCZmQLy8x3lCDIPrEZ github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.7/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= @@ -149,20 +173,29 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/sarama v1.23.1 h1:XxJBCZEoWJtoWjf/xRbmGUpAmTZGnuuF0ON0EvxxBrs= github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= +github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/Venafi/vcert/v4 v4.11.0 h1:37gfyjS9v5YvZcIABwNPo1fAC31lIZT7glVK1vfUxk4= github.com/Venafi/vcert/v4 v4.11.0/go.mod h1:OE+UZ0cj8qqVUuk0u7R4GIk4ZB6JMSf/WySqnBPNwws= +github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= +github.com/a8m/expect v1.0.0 h1:o0PXeXn7zLB77ajwOyT1s1HcPJ4hbV6jAvCWUwvFBUM= github.com/a8m/expect v1.0.0/go.mod h1:4IwSCMumY49ScypDnjNbYEjgVeqy1/U2cEs3Lat96eA= +github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 h1:rFw4nCn9iMW+Vajsk51NtYIcwSTkXr+JGrMd36kTDJw= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/agnivade/levenshtein v1.0.1 h1:3oJU7J3FGFmyhn8KHjmVaZCN5hxTr7GxgRue+sxIXdQ= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 h1:4SPQljF/GJ8Q+QlCWMWxRBepub4DresnOm4eI2ebFGc= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd h1:NlKlOv3aVJ5ODMC0JWPvddw05KENkL3cZttIuu8kJRo= github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd/go.mod h1:idhzw68Q7v4j+rQ2AGyq3OlZW2Jij9mdmGA4/Sk6J0E= +github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -171,28 +204,44 @@ github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRY github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.61.751 h1:PX0jCn9kBBgaybsFltpmQ8F7O74hQXY/3yNyCjInDag= github.com/aliyun/alibaba-cloud-sdk-go v1.61.751/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/andygrunwald/go-gerrit v0.0.0-20190120104749-174420ebee6c h1:uUuUZipfD5nPl2L/i0I3N4iRKJcoO2CPjktaH/kP9gQ= github.com/andygrunwald/go-gerrit v0.0.0-20190120104749-174420ebee6c/go.mod h1:0iuRQp6WJ44ts+iihy5E/WlPqfg5RNeQxOmzRkxCdtk= github.com/anexia-it/go-anxcloud v0.3.8 h1:+ZOVqUHwINTm9Q68GPVh+Q/c794Fe+2GahIVagNLjDg= github.com/anexia-it/go-anxcloud v0.3.8/go.mod h1:cevqezsbOJ4GBlAWaztfLKl9w4VzxJBt4ipgHORi3gw= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apache/thrift v0.13.0 h1:5hryIiq9gtn+MiLVn0wP37kb/uTeRZgN08WoCsAhIhI= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/apoydence/onpar v0.0.0-20200406201722-06f95a1c68e8 h1:+4DSd7k17/TWmO2a1TTTTh9aCHuN30pEUuPB7bXcNkw= github.com/apoydence/onpar v0.0.0-20200406201722-06f95a1c68e8/go.mod h1:maauOJD0kdDqIz4xmkunipFVbBoTM6pFSy0kkWBcIUY= +github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30 h1:Kn3rqvbUFqSepE2OqVu0Pn1CbDw9IuMlONapol0zuwk= github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30/go.mod h1:4AJxUpXUhv4N+ziTvIcWWXgeorXpxPZOfk9HdEVr96M= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 h1:G1bPvciwNyF7IUmKXNt9Ak3m6u9DE1rF+RmtIkBpVdA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a h1:pv34s756C4pEXnjgPfGYgdhg/ZdajGhyOvzx8k+23nw= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= +github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-k8s-tester v0.0.0-20190114231546-b411acf57dfe h1:B7prfUXk9GHnMc/a7NrEY29IJRgI4/2cCbHDqQJGgLs= github.com/aws/aws-k8s-tester v0.0.0-20190114231546-b411acf57dfe/go.mod h1:1ADF5tAtU1/mVtfMcHAYSm2fPw71DA7fFk0yed64/0I= +github.com/aws/aws-lambda-go v1.13.3 h1:SuCy7H3NLyp+1Mrfp+m80jcbi9KYWAs9/BXwppwRDzY= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.16.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -204,7 +253,9 @@ github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/ github.com/aws/aws-sdk-go v1.34.30/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.36.2 h1:UAeFPct+jHqWM+tgiqDrC9/sfbWj6wkcvpsJ+zdcsvA= github.com/aws/aws-sdk-go v1.36.2/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go-v2 v0.18.0 h1:qZ+woO4SamnH/eEbjM2IDLhRNwIwND/RQyVlBLp3Jqg= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/bazelbuild/buildtools v0.0.0-20190917191645-69366ca98f89 h1:3B/ZE1a6eEJ/4Jf/M6RM2KBouN8yKCUcMmXzSyWqa3g= github.com/bazelbuild/buildtools v0.0.0-20190917191645-69366ca98f89/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -213,58 +264,90 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c h1:+0HFd5KSZ/mm3JmhmrDukiId5iR6w4+BdFtfSy4yWIc= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d h1:DMb8SuAL9+demT8equqMMzD8C/uxqWmj4cgV7ufrpQo= github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d/go.mod h1:IyUJYN1gvWjtLF5ZuygmxbnsAyP3aJS6cHzIuZY50B0= +github.com/briandowns/spinner v1.8.0 h1:SeidJ8ASAayR4Wxl5Of54LHqgi8s6sBvAHg4kxKxia4= github.com/briandowns/spinner v1.8.0/go.mod h1://Zf9tMcxfRUA36V23M6YGEAv+kECGfvpnLTnb8n4XQ= +github.com/bwmarrin/snowflake v0.0.0 h1:dRbqXFjM10uA3wdrVZ8Kh19uhciRMOroUYJ7qAqDLhY= github.com/bwmarrin/snowflake v0.0.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= +github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY= github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8= +github.com/casbin/casbin/v2 v2.1.2 h1:bTwon/ECRx9dwBy2ewRVr5OiqjeXSGiTUY74sDPQi/g= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 h1:uH66TXeswKn5PW5zdZ39xEwfS9an067BirqA+P4QaLI= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= +github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/clarketm/json v1.13.4 h1:0JketcMdLC16WGnRGJiNmTXuQznDEQaiknxSPRBxg+k= github.com/clarketm/json v1.13.4/go.mod h1:ynr2LRfb0fQU34l07csRNBTcivjySLLiY1YzQqKVfdo= +github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec h1:EdRZT3IeKQmfCSrgo8SZ8V3MEnskuJP0wCYNpe+aiXo= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= +github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudevents/sdk-go v0.0.0-20190509003705-56931988abe3 h1:DNM19kh6j6qGBx/FI7OmHKBL2vCW1eN28ESYK1+O5DY= github.com/cloudevents/sdk-go v0.0.0-20190509003705-56931988abe3/go.mod h1:j1nZWMLGg3om8SswStBoY6/SHvcLM19MuZqwDtMtmzs= +github.com/cloudflare/cloudflare-go v0.13.2 h1:bhMGoNhAg21DuqJjU9jQepRRft6vYfo6pejT3NN4V6A= github.com/cloudflare/cloudflare-go v0.13.2/go.mod h1:27kfc1apuifUmJhp069y0+hwlKDg4bd8LWlu7oKeZvM= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 h1:xD/lrqdvwsc+O2bjSSi3YqY73Ke3LAiSCx49aCesA0E= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= +github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.1 h1:pASeJT3R3YyVn+94qEPk0SnU1OQ20Jd/T+SPKy9xehY= github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containernetworking/cni v0.7.1 h1:fE3r16wpSEyaqY4Z4oFrLMmIGfBYIKpPrHK31EJ9FzE= github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY= github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/container-linux-config-transpiler v0.9.0 h1:UBGpT8qWqzi48hNLrzMAgAUNJsR0LW8Gk5/dR/caI8U= github.com/coreos/container-linux-config-transpiler v0.9.0/go.mod h1:SlcxXZQ2c42knj8pezMiQsM1f+ADxFMjGetuMKR/YSQ= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.17+incompatible h1:f/Z3EoDSx1yjaIjLQGo1diYUlQYSBrrAQ5vP8NjwXwo= github.com/coreos/etcd v3.3.17+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible h1:bXhRBIXoTm9BYHS3gE0TtQuyNZyeEMux2sDi4oo5YOo= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v0.0.0-20180117170138-065b426bd416/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.0.0-20180108230905-e214231b295a/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -275,62 +358,90 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/ignition v0.35.0 h1:UFodoYq1mOPrbEjtxIsZbThcDyQwAI1owczRDqWmKkQ= github.com/coreos/ignition v0.35.0/go.mod h1:WJQapxzEn9DE0ryxsGvm8QnBajm/XsS/PkrDqSpz+bA= +github.com/coreos/locksmith v0.6.2 h1:yd/7pCRpnmXd1EA9AVbNtMIo5wCxSlRCPzP8wzZEW9Q= github.com/coreos/locksmith v0.6.2/go.mod h1:mSLRr7SVSEAIugjic7+TXif/+ZQQq0zCks1vptuj2fs= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/prometheus-operator v0.35.0 h1:kd7mysk8mCdwquBcPLyuRoRFNJCpgezXu8yUvIYE2nc= github.com/coreos/prometheus-operator v0.35.0/go.mod h1:XHYZUStZWcwd1yk/1DjZv/fywqKIyAJ6pSwvIr+v9BQ= +github.com/cpu/goacmedns v0.0.3 h1:QOeMpIEsIdm1LSASSswjaTf8CXmzcrgy5OeCfHjppA4= github.com/cpu/goacmedns v0.0.3/go.mod h1:4MipLkI+qScwqtVxcNO6okBhbgRrr7/tKXUSgSL0teQ= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cristim/ec2-instances-info v0.0.0-20201110114654-2dfcc09f67d4 h1:uPdJfcX6oBDV/n7KYnXipTvZr0Mll06CnH0FYsY5vYY= github.com/cristim/ec2-instances-info v0.0.0-20201110114654-2dfcc09f67d4/go.mod h1:0yCjO4zBzlwWSGh/zGfW2Zq1NX605qCYVBHD1fPXKNs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892 h1:qg9VbHo1TlL0KDM0vYvBG9EY0X0Yku5WYIPoFWt8f6o= github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE= +github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd h1:uVsMphB1eRx7xB1njzL3fuMdWRN8HtVzoUOItHMwv5c= github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= github.com/denisenkom/go-mssqldb v0.0.0-20190111225525-2fea367d496d/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= +github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0 h1:epsH3lb7KVbXHYk7LYGN5EiE0MxcevHU85CKITJ0wUY= github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/digitalocean/godo v1.44.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= github.com/digitalocean/godo v1.54.0 h1:KP0Nv87pgViR8k/7De3VrmflCL5pJqXbNnkcw0bwG10= github.com/digitalocean/godo v1.54.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg= github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8= github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017 h1:2HQmlpI3yI9deH18Q6xiSOIjXD4sLI55Y/gfpa8/558= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible h1:SiUATuP//KecDjpOK2tvZJgeScYAklvyjfK8JZlU6fo= github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c h1:ZfSZ3P3BedhKGUhzj7BQlPSU4OvT6tfOKe3DVHzOA7s= github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/embik/nutanix-client-go v0.0.0-20220106131900-50b8f27e5f60 h1:0FVKOkpksULFs6F7Kfd8ClBXVTvtiIKl07uV3HinOHk= github.com/embik/nutanix-client-go v0.0.0-20220106131900-50b8f27e5f60/go.mod h1:gkKNSxfEt3QtYG3S/wKiN8OmrJ4fpU7JbTlbnrMDOL8= @@ -339,6 +450,7 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT github.com/emicklei/go-restful v2.10.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.11.2+incompatible h1:Z4Z0K2AuOw+QtgwkkJnwpT165MBr12qS8rnBwjP/Pzs= github.com/emicklei/go-restful v2.11.2+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -346,9 +458,13 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d h1:QyzYnTnPE15SQyUeqU6qLbWxMkwyAyu+vGksa0b7j00= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= +github.com/etcd-io/gofail v0.0.0-20190801230047-ad7f989257ca h1:Y2I0lxOttdUKz+hNaIdG3FtjuQrTmwXun1opRV65IZc= github.com/etcd-io/gofail v0.0.0-20190801230047-ad7f989257ca/go.mod h1:49H/RkXP8pKaZy4h0d+NW16rSLhyVBt4o6VLJbmOqDE= github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= @@ -359,47 +475,70 @@ github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= +github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db h1:gb2Z18BhTPJPpLQWj4T+rfKHYCHxRHCtRxhKKjRidVw= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8 h1:a9ENSRDFBUPkJ5lCgVZh26+ZbGyoVJG7yb5SSzF5H54= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsouza/fake-gcs-server v0.0.0-20180612165233-e85be23bdaa8 h1:3iml5UHzQtk3cpnYfqW16Ia+1xSuu9tc4BElZu5470M= github.com/fsouza/fake-gcs-server v0.0.0-20180612165233-e85be23bdaa8/go.mod h1:1/HufuJ+eaDf4KTnYdS6HJMGvMRU8d4cYTuu/1QaBbI= +github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-bindata/go-bindata v3.1.2+incompatible h1:5vjJMVhowQdPzjE1LdxyFF7YFTXg5IgGVW4gBr5IbvE= github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.62.0 h1:7VJT/ZXjzqSrvtraFp4ONq80hTcRQth1c9ZnQ3uNQvU= github.com/go-ini/ini v1.62.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/log v0.1.0 h1:DGJh0Sm43HbOeYDNnVZFl8BvcYVvjD5bqYJvp0REbwQ= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-ldap/ldap v3.0.2+incompatible h1:kD5HQcAzlQ7yrhfn+h+MSABeAy/jAJhvIJ/QDllP44g= github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -421,6 +560,7 @@ github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9sn github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ= +github.com/go-openapi/analysis v0.19.16 h1:Ub9e++M8sDwtHD+S587TYi+6ANBG1NRYGZDihqk0SaY= github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk= github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.17.2/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= @@ -430,7 +570,9 @@ github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/errors v0.19.9 h1:9SnKdGhiPZHF3ttwFMiCBEb8jQ4IDdrK+5+a0oTygA4= github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= +github.com/go-openapi/inflect v0.19.0 h1:9jCH9scKIbHeV9m12SmPilScz6krDxKRasNNSNPXu/4= github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= @@ -456,6 +598,7 @@ github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5 github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY= github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= +github.com/go-openapi/loads v0.19.7 h1:6cALLpCAq4tYhaic7TMbEzjv8vq/wg+0AFivNy/Bma8= github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= github.com/go-openapi/runtime v0.17.2/go.mod h1:QO936ZXeisByFmZEO1IS1Dqhtf4QV1sYYFtIq6Ld86Q= @@ -464,6 +607,7 @@ github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29g github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo= github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98= github.com/go-openapi/runtime v0.19.20/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= +github.com/go-openapi/runtime v0.19.24 h1:TqagMVlRAOTwllE/7hNKx6rQ10O6T8ZzeJdMjSTKaD4= github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.17.2/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= @@ -482,6 +626,7 @@ github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6 github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= +github.com/go-openapi/strfmt v0.19.11 h1:0+YvbNh05rmBkgztd6zHp4OCFn7Mtu30bn46NQo2ZRw= github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= @@ -502,63 +647,91 @@ github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7 github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8= github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4= +github.com/go-openapi/validate v0.19.15 h1:oUHZO8jD7p5oRLANlXF0U8ic9ePBUkDQyRZdN0EhL6M= github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI= +github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 h1:JVrqSeQfdhYRFk24TvhTZWU0q8lfCojxZQFi3Ou7+uY= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= github.com/go-sql-driver/mysql v0.0.0-20160411075031-7ebe0a500653/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-swagger/go-swagger v0.25.0 h1:FxhyrWWV8V/A9P6GtI5szWordAdbb6Y0nqdY/y9So2w= github.com/go-swagger/go-swagger v0.25.0/go.mod h1:9639ioXrPX9E6BbnbaDklGXjNz7upAXoNBwL4Ok11Vk= +github.com/go-swagger/scan-repo-boundary v0.0.0-20180623220736-973b3573c013 h1:l9rI6sNaZgNC0LnF3MiE+qTmyBA/tZAg1rtyrGbUMK0= github.com/go-swagger/scan-repo-boundary v0.0.0-20180623220736-973b3573c013/go.mod h1:b65mBPzqzZWxOZGxSWrqs4GInLIn+u99Q9q7p+GKni0= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd h1:hSkbZ9XSyjyBirMeqSqUrK+9HboWrweVlzRNqoBi2d4= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/depgen v0.1.0 h1:31atYa/UW9V5q8vMJ+W6wd64OaaTHUrCUXER358zLM4= github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= github.com/gobuffalo/envy v1.6.5/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ= github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.1 h1:OQl5ys5MBea7OGCdvPbBJWRgnhC/fGona6QKfvFeau8= github.com/gobuffalo/envy v1.7.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w= github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= github.com/gobuffalo/flect v0.1.5/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= github.com/gobuffalo/flect v0.2.0/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= +github.com/gobuffalo/flect v0.2.2 h1:PAVD7sp0KOdfswjAw9BpLCU9hXo7wFSzgpQ+zNeks/A= github.com/gobuffalo/flect v0.2.2/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= +github.com/gobuffalo/genny v0.1.1 h1:iQ0D6SpNXIxu52WESsD+KoQ7af2e3nCfnSBoSF/hKe0= github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211 h1:mSVZ4vj4khv+oThUfS+SQU3UuFIZ5Zo6UNcvK8E8Mz8= github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= +github.com/gobuffalo/gogen v0.1.1 h1:dLg+zb+uOyd/mKeQUYIbwbNmfRsr9hd/WtYWepmayhI= github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/logger v1.0.1 h1:ZEgyRGgAm4ZAhAO45YXMs5Fp+bzGLESFewzAVBMKuTg= github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs= github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2 h1:fq9WcL1BYrm36SzK6+aAnZ8hcp+SrmnDyAxhNx8dvJk= github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packd v0.3.0 h1:eMwymTkA1uXsqxS0Tpoop3Lc0u3kTfiMBE6nKtQU4g4= github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= +github.com/gobuffalo/packr/v2 v2.7.1 h1:n3CIW5T17T8v4GGK5sWXLVWJhCz7b5aNLSxW6gYim4o= github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kETFqjFoFlOc= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754 h1:tpom+2CJmpzAWj5/VEHync2rJGi+epHNIeRSWjzGA+4= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godror/godror v0.13.3 h1:4A5GLGAJTSuELw1NThqY5bINYB+mqrln+kF5C2vuyCs= github.com/godror/godror v0.13.3/go.mod h1:2ouUT4kdhUBk7TAkHWD4SN0CdI0pgEQbo8FVHhbSKWg= +github.com/gogo/googleapis v1.1.0 h1:kFkMAZBNAn4j7K0GiZr8cRYzejq68VbheufiV3YuyFI= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -571,8 +744,11 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d h1:lBXNCxVENCipq4D1Is42JVOP4eQjlB8TQ6H69Yx5J9Q= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= +github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -582,6 +758,7 @@ github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7 h1:2hRPrmiwPrp3fQX967rNJIhQPtiGXdlQWAxKbKw3VHA= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -589,6 +766,7 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v0.0.0-20181025225059-d3de96c4c28e/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -611,14 +789,20 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450 h1:7xqw01UYS+KCI25bMrPxwNYkSns2Db1ziQPpVq99FpE= github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= +github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995 h1:f5gsjBiF9tRRVomCvrkGMMWI8W1f2OBFar2c5oakAP0= github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= +github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e h1:KhcknUwkWHKZPbFy2P7jH5LKJ3La+0ZeknkkmrSgqb0= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= +github.com/gomodule/redigo v1.7.0 h1:ZKld1VOtsGhAe37E7wMxEDgAlGM5dvFY+DiOhSkhP9Y= github.com/gomodule/redigo v1.7.0/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -632,8 +816,11 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-containerregistry v0.0.0-20200115214256-379933c9c22b h1:oGqapkPUiypdS9ch/Vu0npPe03RQ0BhVDYli+OEKNAA= github.com/google/go-containerregistry v0.0.0-20200115214256-379933c9c22b/go.mod h1:Wtl/v6YdQxv397EREtzwgd9+Ud7Q5D8XMbi3Zazgkrs= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-licenses v0.0.0-20191112164736-212ea350c932 h1:ZIb3nb+/mHAGRkyuxfPykmYdUi21mr8YTGpr/xGPJ8o= github.com/google/go-licenses v0.0.0-20191112164736-212ea350c932/go.mod h1:16wa6pRqNDUIhOtwF0GcROVqMeXHZJ7H6eGDFUh5Pfk= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -642,6 +829,7 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/licenseclassifier v0.0.0-20190926221455-842c0d70d702 h1:nVgx26pAe6l/02mYomOuZssv28XkacGw/0WeiTVorqw= github.com/google/licenseclassifier v0.0.0-20190926221455-842c0d70d702/go.mod h1:qsqn2hxC+vURpyBRygGUuinTO42MFRLcsmQ/P8v94+M= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -657,7 +845,9 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201117184057-ae444373da19 h1:iFELRewmQ9CldLrqgr0E6b6ZPfZmMvLyyz6kMsR+c4w= github.com/google/pprof v0.0.0-20201117184057-ae444373da19/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v0.0.0-20170306145142-6a5e28554805/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -685,27 +875,37 @@ github.com/gophercloud/gophercloud v0.14.0 h1:c2Byo+YMxhHlTJ3TPptjQ4dOQ1YknTHDJ/ github.com/gophercloud/gophercloud v0.14.0/go.mod h1:VX0Ibx85B60B5XOrZr6kaNwrmPUzcmMpwxvQ1WQIIWM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/csrf v1.6.2 h1:QqQ/OWwuFp4jMKgBFAzJVW3FMULdyUW7JoM4pEWuqKg= github.com/gorilla/csrf v1.6.2/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI= github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v0.0.0-20181024020800-521ea7b17d02/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.1.3 h1:uXoZdcdA5XdXF3QzuSlheVRUvjl+1rKY7zBXL68L9RU= github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc h1:f8eY6cV/x1x+HLjOp4r72s/31/V2aTUtg5oKRRPf8/Q= github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170330212424-2500245aa611/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= @@ -715,58 +915,89 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.15.2/go.mod h1:vO11I9oWA+KsxmfFQPhLnnIb1VDE24M+pdxZFiuZcA8= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/h2non/gock v1.0.9 h1:17gCehSo8ZOgEsFKpQgqHiR7VLyjxdAG3lkhVvO9QZU= github.com/h2non/gock v1.0.9/go.mod h1:CZMcB0Lg5IWnr9bF79pPMg9WeV6WumxQiUJ1UvdO1iE= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/api v1.3.0 h1:HXNYlRkkM/t+Y/Yhxtwcy02dlYwIaoxzvxPnS+cqy78= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.3.0 h1:UOxjlb4xVNF93jak1mzzoBatyFju9nrkxpVwIp/QqxQ= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= +github.com/hashicorp/go-hclog v0.8.0 h1:z3ollgGRg8RjfJH6UVBaG54R70GFd++QOkvnJH3VSBY= github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-plugin v1.0.1 h1:4OtAfUGbnKC6yS48p0CtMX2oFYtzFZVv6rok3cRWgnE= github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= +github.com/hashicorp/go-retryablehttp v0.5.4 h1:1BZvpawXoJCWX6pNtow9+rpEj+3itIlutiqnntI6jOE= github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-rootcerts v1.0.1 h1:DMo4fmknnz0E0evoNYnV48RjWndOsmd6OW+09R3cEP8= github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1 h1:sNCoNyDEvN1xa+X0baata4RdcpKwcMS6DH+xwfqPgjw= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0 h1:WhIgCr5a7AaVH6jPUwjtRuuE7/RDufnUvzIr48smyxs= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/vault/api v1.0.4 h1:j08Or/wryXT4AcHj1oCbMd7IijXcKzYUGw59LGu9onU= github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= +github.com/hashicorp/vault/sdk v0.1.13 h1:mOEPeOhT7jl0J4AMl1E705+BcmeRs1VmKNb9F0sMLy8= github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 h1:GT4RsKmHh1uZyhmTkWJTDALRjSHYQp6FRKrotf0zhAs= github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= github.com/hetznercloud/hcloud-go v1.23.1/go.mod h1:xng8lbDUg+xM1dgc0yGHX5EeqbwIq7UYlMWMTx3SQVg= github.com/hetznercloud/hcloud-go v1.25.0 h1:QAaFKtGKWRxjwjKJWBGMxGYUxVEQmIkb35j/WXrsazY= github.com/hetznercloud/hcloud-go v1.25.0/go.mod h1:2C5uMtBiMoFr3m7lBFPf7wXTdh33CevmZpQIIDPGYJI= +github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c h1:kQWxfPIHVLbgLzphqk3QUflDy9QdksZR4ygR807bpy0= github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/hudl/fargo v1.3.0 h1:0U6+BtN6LhaYuTnIJq4Wyq5cpn6O2kWrxAtcqBmYY6w= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/iancoleman/strcase v0.1.2 h1:gnomlvw9tnV3ITTAxzKSgTF+8kFWcU/f+TgttpXGz1U= github.com/iancoleman/strcase v0.1.2/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.0.0-20171009183408-7fe0c75c13ab/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -777,32 +1008,49 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/improbable-eng/thanos v0.3.2 h1:iZfU7exq+RD5Lnb8n3Eh9MNYoRLeyeGO/85AvEkLg+8= github.com/improbable-eng/thanos v0.3.2/go.mod h1:GZewVGILKuJVPNRn7L4Zw+7X96qzFOwj63b22xYGXBE= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb v0.0.0-20161215172503-049f9b42e9a5 h1:AciJ2ei/llFRundm7CtqwF6B2aOds1A7QG3sMW8QiaQ= github.com/influxdata/influxdb v0.0.0-20161215172503-049f9b42e9a5/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo= github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= +github.com/jenkins-x/go-scm v1.5.65 h1:ieH+0JSWENObn1SDWFj2K40iV5Eia4aTl6W6bDdLwI0= github.com/jenkins-x/go-scm v1.5.65/go.mod h1:MgGRkJScE/rJ30J/bXYqduN5sDPZqZFITJopsnZmTOw= +github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jetstack/cert-manager v1.1.0 h1:gEhBV9I83m+kpQShDhNO4+J8O2qfNDjvAEL27pThGmg= github.com/jetstack/cert-manager v1.1.0/go.mod h1:GULIHTGjSc2LjlgBCLhQ8u5WmQ95hk9FAiQbhjMthMk= +github.com/jinzhu/gorm v0.0.0-20170316141641-572d0a0ab1eb h1:0D5F4qAGJbRqzyCIHswU2fCwB1XGTDkBwBn9qncQYYs= github.com/jinzhu/gorm v0.0.0-20170316141641-572d0a0ab1eb/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= +github.com/jinzhu/inflection v0.0.0-20190603042836-f5c5f50e6090 h1:LIwA5USOJ9W/0hwiRH1MugeThGBHGqv+USXcDKWHIVY= github.com/jinzhu/inflection v0.0.0-20190603042836-f5c5f50e6090/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/joefitzgerald/rainbow-reporter v0.1.0 h1:AuMG652zjdzI0YCCnXAqATtRBpGXMcAnrajcaTrSeuo= github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.0.0-20141017032234-72f9bd7c4e0c/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -814,6 +1062,7 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jsonnet-bundler/jsonnet-bundler v0.1.0 h1:T/HtHFr+mYCRULrH1x/RnoB0prIs0rMkolJhFMXNC9A= github.com/jsonnet-bundler/jsonnet-bundler v0.1.0/go.mod h1:YKsSFc9VFhhLITkJS3X2PrRqWG9u2Jq99udTdDjQLfM= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= @@ -821,27 +1070,42 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20191119172530-79f836b90111 h1:Lq6HJa0JqSg5ko/mkizFWlpIrY7845g9Dzz9qeD5aXI= github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20191119172530-79f836b90111/go.mod h1:MP2HbArq3QT+oVp8pmtHNZnSnkhdkHtDnc7h6nJXmBU= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/karrick/godirwalk v1.10.3 h1:lOpSw2vJP0y5eLBW906QwKsUK/fe/QDyoqM5rnnuPDY= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kelseyhightower/envconfig v1.3.0 h1:IvRS4f2VcIQy6j4ORGIf9145T/AsUB+oY8LyvN8BXNM= github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kinvolk/container-linux-config-transpiler v0.9.1 h1:LIv3RCbjdFhXn/Fg4XHys3sBekkPHM0uxKfAzD0F2jk= github.com/kinvolk/container-linux-config-transpiler v0.9.1/go.mod h1:pjTzCvFfbXjWuMVNFjA9FdbsdmruK6+vki0hK0lDmnU= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM= github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/knative/build v0.1.2 h1:o/VYWA3HKyZlNqdU2hDE5LHpanBe8gazgPKL97XJ6bo= github.com/knative/build v0.1.2/go.mod h1:/sU74ZQkwlYA5FwYDJhYTy61i/Kn+5eWfln2jDbw3Qo= github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -850,6 +1114,7 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -858,19 +1123,28 @@ github.com/kubermatic/machine-controller v1.23.1/go.mod h1:mXWbT7SjqpgFhzCFT3yME github.com/kubermatic/machine-controller v1.26.0/go.mod h1:dcJ+GdDSCxCwM0poxwOK8hVO7epiOORDmNMmb2veyw4= github.com/kubermatic/machine-controller v1.36.1/go.mod h1:6BFZEvEMZi8OT8aHOsS7DXYsF6ZSpmsNxsci7OLTTn8= github.com/kubermatic/machine-controller v1.40.1/go.mod h1:5LVcN4tCybGg+55hIHcVzCjNsBJy2PlnXG0xIzKmXGY= +github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= +github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.1-0.20191011153232-f91d3411e481 h1:r9fnMM01mkhtfe6QfLrr/90mBVLnJHge2jGeBvApOjk= github.com/lib/pq v1.2.1-0.20191011153232-f91d3411e481/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= +github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743 h1:143Bb8f8DuGWck/xpNUOckBVYfFbBTnLevfRZ1aVVqo= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= +github.com/lightstep/lightstep-tracer-go v0.18.1 h1:vi1F1IQ8N7hNWytK9DpJsUfQhGuNSc19z330K6vl4zk= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linode/linodego v0.24.0 h1:o6hNS0T7jeikOfUHoJhUhA/e2QTCsw9MGccVmRHRLE4= github.com/linode/linodego v0.24.0/go.mod h1:GSBKPpjoQfxEfryoCRcgkuUOCuVtGHWhzI8OMdycNTE= +github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= +github.com/lyft/protoc-gen-validate v0.0.13 h1:KNt/RhmQTOLr7Aj8PsJ7mTronaFyx80mRTT9qF261dA= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -882,37 +1156,51 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/markbates/inflect v1.0.4 h1:5fh1gzTFhfae06u3hzHYO9xe3l3v3nW5Pwt3naLTP5g= github.com/markbates/inflect v1.0.4/go.mod h1:1fR9+pO2KHEO9ZRtto13gDwwZaAKstQzferVeWqbgNs= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2 h1:JgVTCPf0uBVcUSWpyXmGpgOc62nK5HWUBKAGc3Qqa5k= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a h1:+J2gw7Bw77w/fbK7wnNJJDKmw1IbWft2Ul5BzrG1Qm8= github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.6/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-oci8 v0.0.7 h1:BBXYpvzPO43QNTLDEivPFteeFZ9nKA6JQ6eifpxOmio= github.com/mattn/go-oci8 v0.0.7/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= github.com/mattn/go-runewidth v0.0.0-20181025052659-b20a3daf6a39/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.5/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v0.0.0-20160514122348-38ee283dabf1/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.12.0 h1:u/x3mp++qUxvYfulZ4HKOvVO0JWhk7HtE8lWhbGz/Do= github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-zglob v0.0.1 h1:xsEx/XUoVlI6yXjqBK062zYhRTZltCNmYPx6v+8DNaY= github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2 h1:g+4J5sZg6osfvEfkRZxJ1em0VT95/UOZgi/l7zi1/oE= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= +github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= @@ -920,22 +1208,31 @@ github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/gox v0.4.0 h1:lfGJxY7ToLJQjHHwi0EX6uYBdK78egf954SQl13PQJc= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= +github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452 h1:hOY53G+kBFhbYFpRVxHl5eS7laP6B1+Cq+Z9Dry1iMU= github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= +github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/ioprogress v0.0.0-20180201004757-6a23b12fa88e h1:Qa6dnn8DlasdXRnacluu8HzPts0S1I9zvvUPDbBnXFI= github.com/mitchellh/ioprogress v0.0.0-20180201004757-6a23b12fa88e/go.mod h1:waEya8ee1Ro/lgxpVhkJI4BVASzkm3UZqkx/cFJiYHM= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.0 h1:7ks8ZkOP5/ujthUsT07rNv+nkLXCQWKNHuwzOAesEks= github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -943,41 +1240,63 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/munnerz/crd-schema-fuzz v1.0.0 h1:8erI9yzEnOGw9K5O+a8zZdoo8N/OwrFi7c7SjBtkHAs= github.com/munnerz/crd-schema-fuzz v1.0.0/go.mod h1:4z/rcm37JxUkSsExFcLL6ZIT1SgDRdLiu7qq1evdVS0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= +github.com/nats-io/gnatsd v1.4.1 h1:RconcfDeWpKCD6QIIwiVFcvForlXpWeJP7i5/lDLy44= github.com/nats-io/gnatsd v1.4.1/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ= +github.com/nats-io/go-nats v1.7.0 h1:oQOfHcLr8hb43QG8yeVyY2jtarIaTjOv41CGdF3tTvQ= github.com/nats-io/go-nats v1.7.0/go.mod h1:+t7RHT5ApZebkrQdnn6AhQJmhJJiKAvJUio1PiiCtj0= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server/v2 v2.1.2 h1:i2Ly0B+1+rzNZHHWtD4ZwKi+OU5l+uQo1iDHZ2PmiIc= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats.go v1.9.1 h1:ik3HbLhZ0YABLto7iX80pZLPw/6dx3T+++MZJwLnMrQ= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.3 h1:6JrEfig+HzTH85yxzhSVbjHRJv9cn0p6n3IngIcM5/k= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.0/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= +github.com/nelsam/hel v0.0.0-20200611165952-2d829bae0c66 h1:C6JDvVh+cs4f8TJXRaJAOY59BC5knehTmdbMYhVfdhA= github.com/nelsam/hel v0.0.0-20200611165952-2d829bae0c66/go.mod h1:Rl/hm4V2s75ScsPmI9cNz87HLNg5MoFAMJwA90fzbkw= +github.com/nelsam/hel/v2 v2.3.2 h1:tXRsJBqRxj4ISSPCrXhbqF8sT+BXA/UaIvjhYjP5Bhk= github.com/nelsam/hel/v2 v2.3.2/go.mod h1:1ZTGfU2PFTOd5mx22i5O0Lc2GY933lQ2wb/ggy+rL3w= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/oklog v0.3.2 h1:wVfs8F+in6nTBMkA7CbRw+zZMIB7nNM825cM1wuzoTk= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= +github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -1008,75 +1327,108 @@ github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDs github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/open-policy-agent/cert-controller v0.0.0-20200921224206-24b87bbc4b6e h1:kKIQk82R8CKO8mQV4sWNl0zR+UcHLAhTwwQ8y+1o02s= github.com/open-policy-agent/cert-controller v0.0.0-20200921224206-24b87bbc4b6e/go.mod h1:/y33mmiq3Cc0N+6cickevrLI/iBbWcUwcEVjSKHA0z0= github.com/open-policy-agent/frameworks/constraint v0.0.0-20200929072634-d96896eff389/go.mod h1:Dr3QxvH+NTQcPPZWSt1ueNOsxW4VwgUltaLL7Ttnrac= +github.com/open-policy-agent/frameworks/constraint v0.0.0-20201118071520-0d37681951a4 h1:dOkENO1IWL75u2N5VPIlj773vhlpkrcJAPb6yHenjY4= github.com/open-policy-agent/frameworks/constraint v0.0.0-20201118071520-0d37681951a4/go.mod h1:vvhkBONv7Uah2fvS/bQ/N1u0rSLvxZOs2ErR6m+4QtQ= +github.com/open-policy-agent/gatekeeper v0.0.0-20201111000257-4450f08fa95e h1:CA8XSPSbDLQ096bsjQttT24tVWyfd0lbbP9eWYGOP7s= github.com/open-policy-agent/gatekeeper v0.0.0-20201111000257-4450f08fa95e/go.mod h1:IseSnWz7MX7IhEpZ4CLhA3NrMazc+T6a5rtSq9pOEc4= github.com/open-policy-agent/opa v0.19.1/go.mod h1:rrwxoT/b011T0cyj+gg2VvxqTtn6N3gp/jzmr3fjW44= github.com/open-policy-agent/opa v0.21.0/go.mod h1:cZaTfhxsj7QdIiUI0U9aBtOLLTqVNe+XE60+9kZKLHw= +github.com/open-policy-agent/opa v0.24.0 h1:fnGOIux+TTGZsC0du1bRBtV8F+KPN55Hks12uE3Fq3E= github.com/open-policy-agent/opa v0.24.0/go.mod h1:qEyD/i8j+RQettHGp4f86yjrjvv+ZYia+JHCMv2G7wA= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/openshift/api v0.0.0-20191219222812-2987a591a72c h1:WRWMmqacvmZDbUat6WYqpuCy2yEfIeDsxFD/Htgp2T0= github.com/openshift/api v0.0.0-20191219222812-2987a591a72c/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= +github.com/openshift/client-go v0.0.0-20191125132246-f6563a70e19a h1:Otk3CuCAEHiMUr4Er6b+csq4Ar6qilAs9h93tbea+qM= github.com/openshift/client-go v0.0.0-20191125132246-f6563a70e19a/go.mod h1:6rzn+JTr7+WYS2E1TExP4gByoABxMznR6y2SnUIkmxk= github.com/openshift/custom-resource-status v0.0.0-20190822192428-e62f2f3b79f3/go.mod h1:GDjWl0tX6FNIj82vIxeudWeSx2Ff6nDZ8uJn0ohUFvo= github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca h1:F1MEnOMwSrTA0YAkO0he9ip9w0JhYzI/iCB2mXmaSPg= github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca/go.mod h1:GDjWl0tX6FNIj82vIxeudWeSx2Ff6nDZ8uJn0ohUFvo= +github.com/openshift/prom-label-proxy v0.1.1-0.20191016113035-b8153a7f39f1 h1:GW8OxGwBbI2kCqjb5PQfVXRAuCJbYyX1RYs9R3ISjck= github.com/openshift/prom-label-proxy v0.1.1-0.20191016113035-b8153a7f39f1/go.mod h1:p5MuxzsYP1JPsNGwtjtcgRHHlGziCJJfztff91nNixw= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 h1:lM6RxxfUMrYL/f8bWEUqdXrANWtrL7Nndbm9iFN0DlU= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5 h1:ZCnq+JUrvXcDVhX/xRolRBZifmabN1HcS1wrPSvxhrU= github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.2.0/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.2 h1:nY8Hti+WKaP0cRsSeQ026wU03QsM762XBeCXBb9NAWI= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/otiai10/copy v1.0.2 h1:DDNipYy6RkIkjMwy+AWzgKiNTyj2RUI9yEMeETEpVyc= github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95 h1:+OLn68pqasWca0z5ryit9KGfp3sUsW4Lqg32iRMJyzs= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888 h1:ARzSqjQJcSR9IXGMEkXN1IQEZChwd2MqXWm6YNGvO5o= github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888/go.mod h1:RQHg5xR1F614BwJyepfMqrKN+32IH0i7yX+ey43rEeQ= +github.com/packethost/pkg v0.0.0-20200903155310-0433e0605550 h1:/ojL7LAVjyH1MY+db0+j6rcWU3UWWpzHksYFsHWs9vQ= github.com/packethost/pkg v0.0.0-20200903155310-0433e0605550/go.mod h1:GSv7cTtIjns4yc0pyajaM1RE/KE4djJONoblFIRDrxA= +github.com/pact-foundation/pact-go v1.0.4 h1:OYkFijGHoZAYbOIb1LWXrwKQbMMRUv1oQ89blD2Mh2Q= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pavel-v-chernykh/keystore-go v2.1.0+incompatible h1:Jd6xfriVlJ6hWPvYOE0Ni0QWcNTLRehfGPFxr3eSL80= github.com/pavel-v-chernykh/keystore-go v2.1.0+incompatible/go.mod h1:xlUlxe/2ItGlQyMTstqeDv9r3U4obH7xYd26TbDQutY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw= github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= +github.com/performancecopilot/speed v3.0.0+incompatible h1:2WnRzIquHa5QxaJKShDkLM+sc0JPuwhXzK8OYOyt3Vg= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d h1:zapSxdmZYY6vJWXFKLQ+MkI+agc+HQyfrCGowDSHiKs= github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.2.6+incompatible h1:6aCX4/YZ9v8q69hTyiR7dNLnTA3fgtKHVVW5BCd5Znw= github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.0.0-20181023235946-059132a15dd0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/profile v1.2.1 h1:F++O52m40owAmADcojzM+9gyjmMOY/T4oYJkgFDH8RE= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/poy/onpar v0.0.0-20200406201722-06f95a1c68e8/go.mod h1:nSbFQvMj97ZyhFRSJYtut+msi4sOY6zJDGCdSc+/rZU= +github.com/poy/onpar v1.0.1 h1:IzLQJa3wxHFXVU8tojF1fw5coZ3CV+9OrnDYZ7GBRy0= github.com/poy/onpar v1.0.1/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU= github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.0.0-20181025174421-f30f42803563/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -1132,38 +1484,58 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/prometheus v2.3.2+incompatible h1:EekL1S9WPoPtJL2NZvL+xo38iMpraOnyEHOiyZygMDY= github.com/prometheus/prometheus v2.3.2+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/prometheus/tsdb v0.8.0 h1:w1tAGxsBMLkuGrFMhqgcCeBkM5d1YI24udArs+aASuQ= github.com/prometheus/tsdb v0.8.0/go.mod h1:fSI0j+IUQrDd7+ZtR9WKIGtoYAYAJUKcKhYLG25tN4g= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20190706150252-9beb055b7962/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYoqz+7sG51ubIDHXeWO8DlTSrToPu6q11ziA= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= +github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.4.0 h1:LUa41nrWTQNGhzdsZ5lTnkwbNjj6rXTdazA1cSdjkOY= github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rollbar/rollbar-go v1.0.2 h1:uA3+z0jq6ka9WUUt9VX/xuiQZXZyWRoeKvkhVvLO9Jc= github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= +github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351 h1:HXr/qUllAWv9riaI4zh2eXWKmCSDqVS/XH1MRHLKRwk= github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg= +github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c h1:ht7N4d/B7Ezf58nvMNVF3OlvDlz9pp+WHVcRNS0nink= github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible h1:j1Wcmh8OrK4Q7GXY+V7SVSY8nUWQxHW5TkBe7YUl+2s= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7 h1:Do8ksLD4Nr3pA0x0hnLOLftZgkiTDvwPDShRTUxtXpE= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8= +github.com/sclevine/spec v1.2.0 h1:1Jwdf9jSfDl9NVmt8ndHqbTZ7XCCPbh1jI3hkDBHVYA= github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI= github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE= @@ -1171,10 +1543,14 @@ github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXY github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/githubv4 v0.0.0-20180925043049-51d7b505e2e9/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= github.com/shurcooL/githubv4 v0.0.0-20190718010115-4ba037080260/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= +github.com/shurcooL/githubv4 v0.0.0-20191102174205-af46314aec7b h1:Cocq9/ZZxCoiybhygOR7hX4E3/PkV8eNbd1AEcUvaHM= github.com/shurcooL/githubv4 v0.0.0-20191102174205-af46314aec7b/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= +github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM= github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/graphql v0.0.0-20180924043259-e4a3a37e6d42/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= +github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f h1:tygelZueB1EtXkPI6mQ4o9DQ0+FKW41hTbunoXZCTqk= github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= @@ -1195,11 +1571,15 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= +github.com/sony/gobreaker v0.4.1 h1:oMnRNZXX5j85zso6xCPRNPtmAycat+WcoKbklScLDgQ= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.3.2 h1:GDarE4TJQI52kYSbSAmLiId1Elfj+xgSDqrUZxFhxlU= github.com/spf13/afero v1.3.2/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= @@ -1212,8 +1592,10 @@ github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHN github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.0.1-0.20200713175500-884edc58ad08/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v0.0.0-20181024212040-082b515c9490/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -1224,15 +1606,22 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= +github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stormcat24/protodep v0.0.0-20200505140716-b02c9ba62816 h1:vG00k+DtOBlp5ug3cQdaMEsaIncIW0bzfgbhQ7qqdXg= github.com/stormcat24/protodep v0.0.0-20200505140716-b02c9ba62816/go.mod h1:mBd5PI4uI6NkqJpCyiWiYzWyTFs4QRDss/JTMC2b4kc= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 h1:WhxRHzgeVGETMlmVfqhRn8RIeeNoPr2Czh33I4Zdccw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a h1:AhmOdSHeswKHBjhsLs/7+1voOxT+LLrSk/Nxvk35fug= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -1241,31 +1630,45 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tektoncd/pipeline v0.10.1 h1:pDsYK2b70o/Ze/CE1nisELwKVVE54FxwyfLznsW1JiE= github.com/tektoncd/pipeline v0.10.1/go.mod h1:D2X0exT46zYx95BU7ByM8+erpjoN7thmUBvlKThOszU= +github.com/tektoncd/plumbing v0.0.0-20191216083742-847dcf196de9 h1:Iu6stVfs72OBV0c3srVX0oogjhLu+stqlvKHT41+pTI= github.com/tektoncd/plumbing v0.0.0-20191216083742-847dcf196de9/go.mod h1:QZHgU07PRBTRF6N57w4+ApRu8OgfYLFNqCDlfEZaD9Y= +github.com/tektoncd/plumbing/pipelinerun-logs v0.0.0-20191206114338-712d544c2c21 h1:9qeyrQsoPZbHOyOPt0OeB1TCYXfYb5swrxlFWzTIYYk= github.com/tektoncd/plumbing/pipelinerun-logs v0.0.0-20191206114338-712d544c2c21/go.mod h1:S62EUWtqmejjJgUMOGB1CCCHRp6C706laH06BoALkzU= +github.com/testcontainers/testcontainers-go v0.9.0 h1:ZyftCfROjGrKlxk3MOUn2DAzWrUtzY/mj17iAkdUIvI= github.com/testcontainers/testcontainers-go v0.9.0/go.mod h1:b22BFXhRbg4PJmeMVWh6ftqjyZHgiIl3w274e9r3C2E= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda h1:uAHwUH+06gowZMVLqQXm7jN1y3Sl+CDJHThNiKyLHus= github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda/go.mod h1:s4k7CORR0OMWd4cYwBqNBFPSJZhnSQxeKdDtMa/aspk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ= github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJMir8RTqb4ayM= github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/urfave/cli v1.18.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.1.1 h1:Qt8FeAtxE/vfdrLmR3rxR6JRE0RoVmbXu8+6kZtYU4k= github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/vdemeester/k8s-pkg-credentialprovider v0.0.0-20200107171650-7c61ffa44238/go.mod h1:JwQJCMWpUDqjZrB5jpw0f5VbN7U95zxFy1ZDpoEarGo= +github.com/vdemeester/k8s-pkg-credentialprovider v1.13.12-1 h1:IBEhRIcu5HP+Pkhzn9E9z3wV0tp3TFjDkiAQtX2FXFM= github.com/vdemeester/k8s-pkg-credentialprovider v1.13.12-1/go.mod h1:Fko0rTxEtDW2kju5Ky7yFJNS3IcNvW8IPsp4/e9oev0= +github.com/vektah/gqlparser v1.1.2 h1:ZsyLGn7/7jDNI+y4SEhI4yAxRChlv15pUHMjijT+e68= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50 h1:uxE3GYdXIOfhMv3unJKETJEhw78gvzuQqRX/rVirc2A= @@ -1273,38 +1676,58 @@ github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50/go.mod h github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/vmware/govmomi v0.23.1 h1:vU09hxnNR/I7e+4zCJvW+5vHu5dO64Aoe2Lw7Yi/KRg= github.com/vmware/govmomi v0.23.1/go.mod h1:Y+Wq4lst78L85Ge/F8+ORXIWiKYqaro1vhAulACy9Lc= +github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728 h1:sH9mEk+flyDxiUa5BuPiuhDETMbzrt9A20I2wktMvRQ= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= +github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 h1:j2hhcujLRHAg872RWAV5yaUrEjHEObwDv3aImCaNLek= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 h1:ESFSdwYZvkeru3RtdrYueztKhOBCSAAzS4Gf+k0tEow= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yashtewari/glob-intersection v0.0.0-20180916065949-5c77d914dd0b h1:vVRagRXf67ESqAb72hG2C/ZwI8NtJF2u2V76EsuOHGY= github.com/yashtewari/glob-intersection v0.0.0-20180916065949-5c77d914dd0b/go.mod h1:HptNXiXVDcJjXe9SqMd0v2FsL9f8dz4GnXgltU6q/co= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.etcd.io/bbolt v1.3.1-etcd.7/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.0.0-20181031231232-83304cfc808c/go.mod h1:weASp41xM3dk0YHg1s/W8ecdGP5G4teSTMBPpYAaUgA= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489 h1:1JFLBqwIgdyHN1ZtgjTBwO+blA6gVOmZurpiMEsETKo= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.etcd.io/etcd/api/v3 v3.5.0 h1:GsV3S+OfZEOCNXdtNkBSR7kgLobAa/SO6tCxRa0GAYw= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0 h1:2aQv6F436YnN7I4VbI8PPYrBhu+SmrTaADcf8Mi/6PU= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0 h1:ftQ0nOOHMcbMS3KIaDQ0g5Qcd6bhaBrQT6b89DfwLTs= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.etcd.io/etcd/client/v3 v3.5.0 h1:62Eh0XOro+rDwkrypAGDfgmNh5Joq+z+W9HZdlXMzek= go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= +go.etcd.io/etcd/pkg/v3 v3.5.0 h1:ntrg6vvKRW26JRmHTE0iNlDgYK6JX3hg/4cD62X0ixk= go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= +go.etcd.io/etcd/raft/v3 v3.5.0 h1:kw2TmO3yFTgE+F0mdKkG7xMxkit2duBDa2Hu6D/HMlw= go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= +go.etcd.io/etcd/server/v3 v3.5.0 h1:jk8D/lwGEDlQU9kZXUFMSANkE22Sg5+mW27ip8xcF9E= go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= +go.etcd.io/etcd/v3 v3.3.0-rc.0.0.20200728214110-6c81b20ec8de h1:RlSimOq2hFUa35bBSmFQC+Wo/diJwbux2t/T7ZNPwsw= go.etcd.io/etcd/v3 v3.3.0-rc.0.0.20200728214110-6c81b20ec8de/go.mod h1:UENlOa05tkNvLx9VnNziSerG4Ro74upGK6Apd4v6M/Y= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -1312,6 +1735,7 @@ go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qL go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= go.mongodb.org/mongo-driver v1.3.5/go.mod h1:Ual6Gkco7ZGQw8wE1t4tLnvBsf6yVSM60qW6TgOeJ5c= +go.mongodb.org/mongo-driver v1.4.3 h1:moga+uhicpVshTyaqY9L23E6QqwcHRUv1sqyOsoyOO8= go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= go.opencensus.io v0.17.0/go.mod h1:mp1VrMQxhlqqDpKvH4UcQUa4YwlzNmymAjPrDdfxNpI= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -1324,17 +1748,29 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 h1:sO4WKdPAudZGKPcpZT4MJn6JaDmpyLrMPDGGyA1SttE= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 h1:Q3C9yzW6I9jqEc8sawxzxZmY48fs9u220KXq6d5s3XU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= +go.opentelemetry.io/otel v0.20.0 h1:eaP0Fqu7SXHwvjiqDq83zImeehOHX8doTvU9AwXON8g= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= +go.opentelemetry.io/otel/metric v0.20.0 h1:4kzhXFP+btKm4jwxpjIqjs41A7MakRFUS86bqLHTIw8= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0 h1:HiITxCawalo5vQzdHfKeZurV8x7ljcqAgiWzF6Vaeaw= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= +go.opentelemetry.io/otel/sdk v0.20.0 h1:JsxtGXd06J8jrnya7fdI/U/MR6yXA5DtbZy+qoHQlr8= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk/export/metric v0.20.0 h1:c5VRjxCXdQlx1HjzwGdQHzZaVI82b5EbBgOu2ljD92g= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= +go.opentelemetry.io/otel/sdk/metric v0.20.0 h1:7ao1wpzHRVKf0OQ7GIxiQJA6X7DLX9o14gmVon7mMK8= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= +go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52lqtnbw= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzcdGg8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -1351,6 +1787,7 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+ go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -1414,8 +1851,10 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181023182221-1baf3a9d7d67/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1433,6 +1872,7 @@ golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhp golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mobile v0.0.0-20190806162312-597adff16ade h1:b373EGXtj0o+ssqkOkdVphTCZ/fVg2LwhctJn2QQbqA= golang.org/x/mobile v0.0.0-20190806162312-597adff16ade/go.mod h1:AlhUtkH4DA4asiFC5RgK7ZKmauvtkAVcy9L0epCzlWo= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= @@ -1745,8 +2185,10 @@ gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3m gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= +gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485 h1:OB/uP/Puiu5vS5QMRPrXCDWUPb+kt8f1KW8oQzFejQw= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e h1:jRyg0XfpwWlhEV8mDfdNGBeSJM2fuyh9Yjrnd8kF2Ts= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181021000519-a2651947f503/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= @@ -1881,8 +2323,11 @@ google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/l google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= +gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1891,13 +2336,18 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/cheggaaa/pb.v1 v1.0.25 h1:Ev7yu1/f6+d+b3pi5vPdRPc6nNtP1umSfcWiEfRqv6I= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/gorp.v1 v1.7.2 h1:j3DWlAyGVv8whO7AcIWznQ2Yj7yJkn34B8s63GViAAw= gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -1908,27 +2358,39 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.52.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/gokrb5.v7 v7.3.0 h1:0709Jtq/6QXEuWRfAm260XqlpcwL1vxtO1tUE2qK8Z4= gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/natefinch/lumberjack.v2 v2.0.0-20150622162204-20b71e5b60d7/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5 h1:E846t8CnR+lv5nE+VuiKTDG/v1U2stad0QzddfJC7kY= gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5/go.mod h1:hiOFpYm0ZJbusNj2ywpbrXowU3G8U6GIQzqn2mw1UIE= gopkg.in/square/go-jose.v2 v2.0.0-20180411045311-89060dee6a84/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= +gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg= gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= +gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE= gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= @@ -1950,8 +2412,10 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v0.0.0-20181223230014-1083505acf35/go.mod h1:R//lfYlUuTOTfblYI3lGoAAAebUdzjvbmQsuB7Ykd90= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1962,6 +2426,7 @@ honnef.co/go/tools v0.0.1-2019.2.2/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.4 h1:SadWOkti5uVN1FAMgxn165+Mw00fuQKyk4Gyn/inxNQ= honnef.co/go/tools v0.1.4/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= k8c.io/kubermatic/v2 v2.16.2 h1:tjPfI+VV51pggXCvcDL/qG1r7KHDBQPSPYngPxpRtp8= k8c.io/kubermatic/v2 v2.16.2/go.mod h1:NdW+2mq4ynRtfZs9yPnvcnFWQpzmM7ngntW6GeuQicU= @@ -1969,6 +2434,8 @@ k8c.io/operating-system-manager v0.1.0/go.mod h1:ULyZQO1irKjsQTNjIdrHld7SZ+joHjm k8c.io/operating-system-manager v0.3.0/go.mod h1:ME5GOCNUrHG+57igEKP1JCJKVHynaLfodT8bRiYH3MY= k8c.io/operating-system-manager v0.3.6 h1:irFFYE/IJM2Qo+lH1zat2o3Yvgb8hUaypPWAc0qGHNM= k8c.io/operating-system-manager v0.3.6/go.mod h1:aFyB/RH9DBAk0Kj5JVtCixhm9ugTeC8akgRGMW28lPg= +k8c.io/operating-system-manager v0.3.9 h1:GcZgXqh90XYKdDXRMaMenA9AE30T8PZ47fM3yrwnygc= +k8c.io/operating-system-manager v0.3.9/go.mod h1:aFyB/RH9DBAk0Kj5JVtCixhm9ugTeC8akgRGMW28lPg= k8s.io/api v0.0.0-20181018013834-843ad2d9b9ae/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20181115043458-b799cb063522/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20190725062911-6607c48751ae/go.mod h1:1O0xzX/RAtnm7l+5VEUxZ1ysO2ghatfq/OZED4zM9kA= @@ -2046,12 +2513,16 @@ k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= k8s.io/apiserver v0.19.4/go.mod h1:X8WRHCR1UGZDd7HpV0QDc1h/6VbbpAeAGyxSh8yzZXw= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.2/go.mod h1:2nKd93WyMhZx4Hp3RfgH2K5PhwyTrprrkWYnI7id7jA= +k8s.io/apiserver v0.22.2 h1:TdIfZJc6YNhu2WxeAOWq1TvukHF0Sfx0+ln4XK9qnL4= k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= +k8s.io/autoscaler v0.0.0-20190218140445-7f77136aeea4 h1:My/qvGX4p7+3wWSGZO/QQ4mZq9ly5zoNsMUaec1b/30= k8s.io/autoscaler v0.0.0-20190218140445-7f77136aeea4/go.mod h1:QEXezc9uKPT91dwqhSJq3GNI3B1HxFRQHiku9kmrsSA= k8s.io/cli-runtime v0.19.0/go.mod h1:tun9l0eUklT8IHIM0jors17KmUjcrAxn0myoBYwuNuo= +k8s.io/cli-runtime v0.19.4 h1:FPpoqFbWsFzRbZNRI+o/+iiLFmWMYTmBueIj3OaNVTI= k8s.io/cli-runtime v0.19.4/go.mod h1:m8G32dVbKOeaX1foGhleLEvNd6REvU7YnZyWn5//9rw= k8s.io/client-go v0.22.2 h1:DaSQgs02aCC1QcwUdkKZWOeaVsQjYvWv8ZazcZ6JcHc= k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= +k8s.io/cloud-provider v0.17.0 h1:BQZPD1Ja/vnTOj1GKI9/wSpd3qgIDZp9q2NAS3568Ac= k8s.io/cloud-provider v0.17.0/go.mod h1:Ze4c3w2C0bRsjkBUoHpFi+qWe3ob1wI2/7cUn+YQIDE= k8s.io/code-generator v0.0.0-20181114232248-ae218e241252/go.mod h1:IPqxl/YHk05nodzupwjke6ctMjyNRdV2zZ5/j3/F204= k8s.io/code-generator v0.0.0-20190612205613-18da4a14b22b/go.mod h1:G8bQwmHm2eafm5bgtX67XDZQ8CWKSGu9DekI+yN4Y5I= @@ -2070,6 +2541,7 @@ k8s.io/code-generator v0.19.4/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZ k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.2/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.4/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= +k8s.io/code-generator v0.22.2 h1:+bUv9lpTnAWABtPkvO4x0kfz7j/kDEchVt0P/wXU3jQ= k8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA= k8s.io/component-base v0.0.0-20190918200425-ed2f0867c778/go.mod h1:DFWQCXgXVLiWtzFaS17KxHdlUeUymP7FLxZSkmL9/jU= @@ -2088,6 +2560,7 @@ k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeY k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= k8s.io/component-base v0.22.2 h1:vNIvE0AIrLhjX8drH0BgCNJcR4QZxMXcJzBsDplDx9M= k8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug= +k8s.io/csi-translation-lib v0.17.0 h1:8hwWJDMOBCAogaWXtNWy0dYGQ2dZYzOnOzjQMiDaY+E= k8s.io/csi-translation-lib v0.17.0/go.mod h1:HEF7MEz7pOLJCnxabi45IPkhSsE/KmxPQksuCrHKWls= k8s.io/gengo v0.0.0-20181106084056-51747d6e00da/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -2099,6 +2572,7 @@ k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 h1:Uusb3oh8XcdzDF/ndlI4ToKTYVlkCSJP39SRY2mfRAw= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v0.0.0-20190306015804-8e90cee79f82/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= @@ -2114,20 +2588,25 @@ k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-aggregator v0.19.0/go.mod h1:1Ln45PQggFAG8xOqWPIYMxUq8WNtpPnYsbUJ39DpF/A= +k8s.io/kube-aggregator v0.19.4 h1:ME+z/JfCTj7IzSWzu7XVhjWHxpEGGQ3gp2FpeOS+lW0= k8s.io/kube-aggregator v0.19.4/go.mod h1:cTkvun110194d797AuThyydBBlgm+cKIFUeS2uzGJfU= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kubectl v0.19.0/go.mod h1:gPCjjsmE6unJzgaUNXIFGZGafiUp5jh0If3F/x7/rRg= +k8s.io/kubectl v0.19.4 h1:XFrHibf5fS4Ot8h3EnzdVsKrYj+pndlzKbwPkfra5hI= k8s.io/kubectl v0.19.4/go.mod h1:XPmlu4DJEYgD83pvZFeKF8+MSvGnYGqunbFSrJsqHv0= k8s.io/kubelet v0.19.4/go.mod h1:zJnPeb7nJCRvtAwxJhe9fFCtMLXL3cXbQiczPmpDrLU= k8s.io/kubelet v0.22.2 h1:7ol5AXXxcW97dUE8W/QiPjkXu1ZuGshG5VmgDmviZsc= k8s.io/kubelet v0.22.2/go.mod h1:ORIRua2/wTcx5UnEvxWosu650/8fatmzbMRC7m6WjAM= +k8s.io/legacy-cloud-providers v0.17.0 h1:ITm7sUthpxQyP96MU7K4Ra9M9M1k9eywUWv9IiTaxzc= k8s.io/legacy-cloud-providers v0.17.0/go.mod h1:DdzaepJ3RtRy+e5YhNtrCYwlgyK87j/5+Yfp0L9Syp8= k8s.io/metrics v0.19.0/go.mod h1:WykpW8B60OeAJx1imdwUgyOID2kDljr/Q+1zrPJ98Wo= +k8s.io/metrics v0.19.4 h1:adT/mgcMXbGvg/Zrj6pPO6js0rqcV7IttYFV//YWtQQ= k8s.io/metrics v0.19.4/go.mod h1:a0gvAzrxQPw2ouBqnXI7X9qlggpPkKAFgWU/Py+KZiU= k8s.io/test-infra v0.0.0-20181019233642-2e10a0bbe9b3/go.mod h1:2NzXB13Ji0nqpyublHeiPC4FZwU0TknfvyaaNfl/BTA= +k8s.io/test-infra v0.0.0-20200220102703-18fae0a00a2c h1:SWAghVxWCDXI56XuvtwQzAj3O4gGhbxIzzQ0td17lis= k8s.io/test-infra v0.0.0-20200220102703-18fae0a00a2c/go.mod h1:B9KsgNJiVixsZud99/ugFoQys8h9Tyv/A/eG5LMyrEE= k8s.io/utils v0.0.0-20181019225348-5e321f9a457c/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= @@ -2144,30 +2623,46 @@ k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +knative.dev/caching v0.0.0-20190719140829-2032732871ff h1:PrlDvOGvCASqW5Fs3ZGes0ma3P5Wr8nuzlTX+EnqfUg= knative.dev/caching v0.0.0-20190719140829-2032732871ff/go.mod h1:dHXFU6CGlLlbzaWc32g80cR92iuBSpsslDNBWI8C7eg= +knative.dev/eventing-contrib v0.6.1-0.20190723221543-5ce18048c08b h1:DTbaZGn06qcEoHuNyw3VmajapIUMuLSVjwFh6pNPews= knative.dev/eventing-contrib v0.6.1-0.20190723221543-5ce18048c08b/go.mod h1:SnXZgSGgMSMLNFTwTnpaOH7hXDzTFtw0J8OmHflNx3g= +knative.dev/pkg v0.0.0-20191111150521-6d806b998379 h1:0IbJWfv82eUhoNymvIrTjxVqrAURRK1x39+//IZV7Cc= knative.dev/pkg v0.0.0-20191111150521-6d806b998379/go.mod h1:pgODObA1dTyhNoFxPZTTjNWfx6F0aKsKzn+vaT9XO/Q= kubevirt.io/api v0.48.1 h1:C5i9h8ea7Xy3fJMoKEuzjRP74GnVMF7u2mQV8FGf2XE= kubevirt.io/api v0.48.1/go.mod h1:RoYMmFt76vWvFtw/FSiL0YUHZ2Ao6UfXlgpZAQnRswo= +kubevirt.io/client-go v0.30.0 h1:0jUvTa/Ev03lCN+Dr4mH22ipoJ9otAOkpFh6wA66b5M= kubevirt.io/client-go v0.30.0/go.mod h1:JY7hQq+SUT0aLvleXrW/+28fDfZ6BPe4E6f8FyC8jkY= +kubevirt.io/containerized-data-importer v1.10.6 h1:xkqLb48pkbdoY8gB2VDP2o+KXpO18tgQuLjcXNn0qAI= kubevirt.io/containerized-data-importer v1.10.6/go.mod h1:qF594BtRRkruyrqLwt3zbLCWdPIQNs1qWh4LR1cOzy0= kubevirt.io/containerized-data-importer-api v1.41.0 h1:VdEwYP36N+4asMnTBSadVH4SF7OVPvvraEQMtOd7Vlk= kubevirt.io/containerized-data-importer-api v1.41.0/go.mod h1:0xadDFtaMd8iy+/oD2+dYoPxACZ/YizKqay5QIrQ6cw= kubevirt.io/controller-lifecycle-operator-sdk v0.2.1 h1:I1b14fnhwrVvQLmgksMo9vgje42hmH4QN5kqyYDqbMA= kubevirt.io/controller-lifecycle-operator-sdk v0.2.1/go.mod h1:ZJhLceiY2Gl5CXFGSp5eMGt/sksOiJP0289nAZFCQf0= +modernc.org/cc v1.0.0 h1:nPibNuDEx6tvYrUAtvDTTw98rx5juGsa5zuDnKwEEQQ= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= +modernc.org/golex v1.0.0 h1:wWpDlbK8ejRfSyi0frMyhilD3JBvtcx2AdGDnU+JtsE= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/xc v1.0.0 h1:7ccXrupWZIS3twbUGrtKmHS2DXY6xegFua+6O3xgAFU= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= +mvdan.cc/xurls/v2 v2.0.0 h1:r1zSOSNS/kqtpmATyMMMvaZ4/djsesbYz5kr0+qMRWc= mvdan.cc/xurls/v2 v2.0.0/go.mod h1:2/webFPYOXN9jp/lzuj0zuAVlF+9g4KPFJANH1oJhRU= +pack.ag/amqp v0.11.0 h1:ot/IA0enDkt4/c8xfbCO7AZzjM4bHys/UffnFmnHUnU= pack.ag/amqp v0.11.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22 h1:fmRfl9WJ4ApJn7LxNuED4m0t18qivVQOxP6aAYG9J6c= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-runtime v0.3.0/go.mod h1:Cw6PkEg0Sa7dAYovGT4R0tRkGhHXpYijwNxYhAnAZZk= sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns= @@ -2181,13 +2676,16 @@ sigs.k8s.io/controller-runtime v0.10.2/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkG sigs.k8s.io/controller-tools v0.2.4/go.mod h1:m/ztfQNocGYBgTTCmFdnK94uVvgxeZeE3LtJvd/jIzA= sigs.k8s.io/controller-tools v0.2.9-0.20200414181213-645d44dca7c0/go.mod h1:YKE/iHvcKITCljdnlqHYe+kAt7ZldvtAwUzQff0k1T0= sigs.k8s.io/controller-tools v0.4.1/go.mod h1:G9rHdZMVlBDocIxGkK3jHLWqcTMNvveypYJwrvYKjWU= +sigs.k8s.io/controller-tools v0.5.0 h1:3u2RCwOlp0cjCALAigpOcbAf50pE+kHSdueUosrC/AE= sigs.k8s.io/controller-tools v0.5.0/go.mod h1:JTsstrMpxs+9BUj6eGuAaEb6SDSPTeVtUyp0jmnAM/I= +sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= sigs.k8s.io/structured-merge-diff v1.0.1 h1:LOs1LZWMsz1xs77Phr/pkB4LFaavH7IVq/3+WTN9XTA= sigs.k8s.io/structured-merge-diff v1.0.1/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= @@ -2196,12 +2694,16 @@ sigs.k8s.io/structured-merge-diff/v4 v4.1.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= +sigs.k8s.io/testing_frameworks v0.1.2 h1:vK0+tvjF0BZ/RYFeZ1E6BYBwHJJXhjuZ3TdsEKH+UQM= sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= software.sslmate.com/src/go-pkcs12 v0.0.0-20180114231543-2291e8f0f237/go.mod h1:/xvNRWUqm0+/ZMiF4EX00vrSCMsE4/NHb+Pt3freEeQ= +software.sslmate.com/src/go-pkcs12 v0.0.0-20200830195227-52f69702a001 h1:AVd6O+azYjVQYW1l55IqkbL8/JxjrLtO6q4FCmV8N5c= software.sslmate.com/src/go-pkcs12 v0.0.0-20200830195227-52f69702a001/go.mod h1:/xvNRWUqm0+/ZMiF4EX00vrSCMsE4/NHb+Pt3freEeQ= +sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 h1:ucqkfpjg9WzSUubAO62csmucvxl4/JeW3F4I4909XkM= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= +vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 h1:O69FD9pJA4WUZlEwYatBEEkRWKQ5cKodWpdKTrCS/iQ= vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= From 3040f8d673ac923782cf8718096564314398182c Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Wed, 19 Jan 2022 11:23:44 +0200 Subject: [PATCH 067/489] Configure server group of the openstack VM (#1169) Signed-off-by: Artiom Diomin --- examples/openstack-machinedeployment.yaml | 3 + go.mod | 4 +- go.sum | 512 +----------------- .../provider/openstack/provider.go | 144 +++-- .../provider/openstack/types/types.go | 1 + 5 files changed, 103 insertions(+), 561 deletions(-) diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index 5b7442984..90dc25c4c 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -128,6 +128,9 @@ spec: computeAPIVersion: "2.67" image: "Ubuntu 18.04 amd64" flavor: "m1.small" + # UUID of the server group + # used to configure affinity or anti-affinity of the VM instaces relative to hypervisor + serverGroup: "" rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" securityGroups: - configMapKeyRef: diff --git a/go.mod b/go.mod index c6806a3f1..c35691f53 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/ghodss/yaml v1.0.0 github.com/go-test/deep v1.0.7 github.com/google/uuid v1.1.2 - github.com/gophercloud/gophercloud v0.14.0 + github.com/gophercloud/gophercloud v0.24.0 github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 github.com/hetznercloud/hcloud-go v1.25.0 github.com/linode/linodego v0.24.0 @@ -35,7 +35,7 @@ require ( github.com/sethvargo/go-password v0.2.0 github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda github.com/vmware/govmomi v0.23.1 - golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 + golang.org/x/crypto v0.0.0-20211202192323-5770296d904e golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c gomodules.xyz/jsonpatch/v2 v2.2.0 google.golang.org/api v0.36.0 diff --git a/go.sum b/go.sum index 45af0b18d..b9aafb928 100644 --- a/go.sum +++ b/go.sum @@ -26,12 +26,9 @@ cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNF cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0 h1:PQcPefKFdaIzjQFbiyOgAqyx8q5djaE7x9Sqe712DPA= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0 h1:9x7Bx0A9R5/M9jibeJeZWqjeVEIxYW9fZYqB9a70/bY= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/logging v1.0.0/go.mod h1:V1cc3ogwobYzQq5f2R7DS/GvRIrI4FKj01Gs5glwAls= cloud.google.com/go/logging v1.1.2 h1:KNALX0NZn8UJhqKnqoHxhMqyoZfBZoh5wF7CQJZ5XrU= @@ -39,7 +36,6 @@ cloud.google.com/go/logging v1.1.2/go.mod h1:KrljuAHIw631j9+QXsnq9vDwsrwmdxfGpiv cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1 h1:ukjixP1wl0LpnZ6LWtZJ0mX5tBmjp1f8Sqer8Z2OMUU= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= @@ -47,21 +43,14 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -code.cloudfoundry.org/go-pubsub v0.0.0-20180503211407-becd51dc37cb h1:N9iWwP+UaQWj0GB53fIEZTD/qK/0EoJjfIS+YUiRU5I= code.cloudfoundry.org/go-pubsub v0.0.0-20180503211407-becd51dc37cb/go.mod h1:2mohpzdn59JWHT85lXjjglNpGLF51tk6hHqfxpc0utk= -contrib.go.opencensus.io/exporter/ocagent v0.4.12 h1:jGFvw3l57ViIVEPKKEUXPcLYIXJmQxLUh6ey1eJhwyc= contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= -contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg= contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= -contrib.go.opencensus.io/exporter/stackdriver v0.12.8 h1:iXI5hr7pUwMx0IwMphpKz5Q3If/G5JiWFVZ5MPPxP9E= contrib.go.opencensus.io/exporter/stackdriver v0.12.8/go.mod h1:XyyafDnFOsqoxHJgTFycKZMrRUrPThLh2iYTJF6uoO0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUuKccphEpTJnWhS2jkQyMt6Y7pJCD7fY= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -git.apache.org/thrift.git v0.12.0 h1:CMxsZlAmxKs+VAZMlDDL0wXciMblJcutQbEe3A9CYUM= git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= -github.com/Azure/azure-pipeline-go v0.1.9 h1:u7JFb9fFTE6Y/j8ae2VK33ePrRqJqoCM/IWkQdAZ+rg= github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-sdk-for-go v28.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v35.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= @@ -69,11 +58,9 @@ github.com/Azure/azure-sdk-for-go v38.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9mo github.com/Azure/azure-sdk-for-go v46.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v49.0.0+incompatible h1:rvYYNgKNBwoxUaBFmd/+TpW3qrd805EHBBvUp5FmFso= github.com/Azure/azure-sdk-for-go v49.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804 h1:QjGHsWFbJyl312t0BtgkmZy2TTYA++FF0UakGbr3ZhQ= github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -129,20 +116,13 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher v0.0.0-20191203181535-308b93ad1f39 h1:Pjo3SOZigEnIGevhFqcbFndnqyCH8WimcREd3hRM9vU= github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher v0.0.0-20191203181535-308b93ad1f39/go.mod h1:yfGmCjKuUzk9WzubMlW2zwjhCraIc/J+M40cufdemRM= -github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534 h1:N7lSsF+R7wSulUADi36SInSQA3RvfO/XclHQfedr0qk= github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14= -github.com/GoogleCloudPlatform/testgrid v0.0.1-alpha.4 h1:TxlW95CqGjvJmW6trWh1XgKZP8GOIgkJEKAUBX/DQ4Q= github.com/GoogleCloudPlatform/testgrid v0.0.1-alpha.4/go.mod h1:f96W2HYy3tiBNV5zbbRc+NczwYHgG1PHXMQfoEWv680= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd h1:sjQovDkwrZp8u+gxLtPgKGjk5hCxuy2hrRejBTA9xFU= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -156,16 +136,12 @@ github.com/Masterminds/sprig/v3 v3.1.0/go.mod h1:ONGMf7UfYGAbMXCZmQLy8x3lCDIPrEZ github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= -github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA= github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OneOfOne/xxhash v1.2.7/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8= github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= @@ -173,29 +149,20 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/sarama v1.23.1 h1:XxJBCZEoWJtoWjf/xRbmGUpAmTZGnuuF0ON0EvxxBrs= github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= -github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/Venafi/vcert/v4 v4.11.0 h1:37gfyjS9v5YvZcIABwNPo1fAC31lIZT7glVK1vfUxk4= github.com/Venafi/vcert/v4 v4.11.0/go.mod h1:OE+UZ0cj8qqVUuk0u7R4GIk4ZB6JMSf/WySqnBPNwws= -github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/a8m/expect v1.0.0 h1:o0PXeXn7zLB77ajwOyT1s1HcPJ4hbV6jAvCWUwvFBUM= github.com/a8m/expect v1.0.0/go.mod h1:4IwSCMumY49ScypDnjNbYEjgVeqy1/U2cEs3Lat96eA= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5 h1:rFw4nCn9iMW+Vajsk51NtYIcwSTkXr+JGrMd36kTDJw= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/agnivade/levenshtein v1.0.1 h1:3oJU7J3FGFmyhn8KHjmVaZCN5hxTr7GxgRue+sxIXdQ= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 h1:4SPQljF/GJ8Q+QlCWMWxRBepub4DresnOm4eI2ebFGc= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd h1:NlKlOv3aVJ5ODMC0JWPvddw05KENkL3cZttIuu8kJRo= github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd/go.mod h1:idhzw68Q7v4j+rQ2AGyq3OlZW2Jij9mdmGA4/Sk6J0E= -github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -204,44 +171,28 @@ github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRY github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.61.751 h1:PX0jCn9kBBgaybsFltpmQ8F7O74hQXY/3yNyCjInDag= github.com/aliyun/alibaba-cloud-sdk-go v1.61.751/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/andygrunwald/go-gerrit v0.0.0-20190120104749-174420ebee6c h1:uUuUZipfD5nPl2L/i0I3N4iRKJcoO2CPjktaH/kP9gQ= github.com/andygrunwald/go-gerrit v0.0.0-20190120104749-174420ebee6c/go.mod h1:0iuRQp6WJ44ts+iihy5E/WlPqfg5RNeQxOmzRkxCdtk= github.com/anexia-it/go-anxcloud v0.3.8 h1:+ZOVqUHwINTm9Q68GPVh+Q/c794Fe+2GahIVagNLjDg= github.com/anexia-it/go-anxcloud v0.3.8/go.mod h1:cevqezsbOJ4GBlAWaztfLKl9w4VzxJBt4ipgHORi3gw= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0 h1:5hryIiq9gtn+MiLVn0wP37kb/uTeRZgN08WoCsAhIhI= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apoydence/onpar v0.0.0-20200406201722-06f95a1c68e8 h1:+4DSd7k17/TWmO2a1TTTTh9aCHuN30pEUuPB7bXcNkw= github.com/apoydence/onpar v0.0.0-20200406201722-06f95a1c68e8/go.mod h1:maauOJD0kdDqIz4xmkunipFVbBoTM6pFSy0kkWBcIUY= -github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30 h1:Kn3rqvbUFqSepE2OqVu0Pn1CbDw9IuMlONapol0zuwk= github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30/go.mod h1:4AJxUpXUhv4N+ziTvIcWWXgeorXpxPZOfk9HdEVr96M= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 h1:G1bPvciwNyF7IUmKXNt9Ak3m6u9DE1rF+RmtIkBpVdA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a h1:pv34s756C4pEXnjgPfGYgdhg/ZdajGhyOvzx8k+23nw= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= -github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef h1:46PFijGLmAjMPwCCCo7Jf0W6f9slllCkkv7vyc1yOSg= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-k8s-tester v0.0.0-20190114231546-b411acf57dfe h1:B7prfUXk9GHnMc/a7NrEY29IJRgI4/2cCbHDqQJGgLs= github.com/aws/aws-k8s-tester v0.0.0-20190114231546-b411acf57dfe/go.mod h1:1ADF5tAtU1/mVtfMcHAYSm2fPw71DA7fFk0yed64/0I= -github.com/aws/aws-lambda-go v1.13.3 h1:SuCy7H3NLyp+1Mrfp+m80jcbi9KYWAs9/BXwppwRDzY= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.16.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -253,9 +204,7 @@ github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/ github.com/aws/aws-sdk-go v1.34.30/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.36.2 h1:UAeFPct+jHqWM+tgiqDrC9/sfbWj6wkcvpsJ+zdcsvA= github.com/aws/aws-sdk-go v1.36.2/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go-v2 v0.18.0 h1:qZ+woO4SamnH/eEbjM2IDLhRNwIwND/RQyVlBLp3Jqg= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/bazelbuild/buildtools v0.0.0-20190917191645-69366ca98f89 h1:3B/ZE1a6eEJ/4Jf/M6RM2KBouN8yKCUcMmXzSyWqa3g= github.com/bazelbuild/buildtools v0.0.0-20190917191645-69366ca98f89/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -264,90 +213,58 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c h1:+0HFd5KSZ/mm3JmhmrDukiId5iR6w4+BdFtfSy4yWIc= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d h1:DMb8SuAL9+demT8equqMMzD8C/uxqWmj4cgV7ufrpQo= github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d/go.mod h1:IyUJYN1gvWjtLF5ZuygmxbnsAyP3aJS6cHzIuZY50B0= -github.com/briandowns/spinner v1.8.0 h1:SeidJ8ASAayR4Wxl5Of54LHqgi8s6sBvAHg4kxKxia4= github.com/briandowns/spinner v1.8.0/go.mod h1://Zf9tMcxfRUA36V23M6YGEAv+kECGfvpnLTnb8n4XQ= -github.com/bwmarrin/snowflake v0.0.0 h1:dRbqXFjM10uA3wdrVZ8Kh19uhciRMOroUYJ7qAqDLhY= github.com/bwmarrin/snowflake v0.0.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= -github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY= github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8= -github.com/casbin/casbin/v2 v2.1.2 h1:bTwon/ECRx9dwBy2ewRVr5OiqjeXSGiTUY74sDPQi/g= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 h1:uH66TXeswKn5PW5zdZ39xEwfS9an067BirqA+P4QaLI= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5 h1:7aWHqerlJ41y6FOsEUvknqgXnGmJyJSbjhAWq5pO4F8= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= -github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/clarketm/json v1.13.4 h1:0JketcMdLC16WGnRGJiNmTXuQznDEQaiknxSPRBxg+k= github.com/clarketm/json v1.13.4/go.mod h1:ynr2LRfb0fQU34l07csRNBTcivjySLLiY1YzQqKVfdo= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec h1:EdRZT3IeKQmfCSrgo8SZ8V3MEnskuJP0wCYNpe+aiXo= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= -github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudevents/sdk-go v0.0.0-20190509003705-56931988abe3 h1:DNM19kh6j6qGBx/FI7OmHKBL2vCW1eN28ESYK1+O5DY= github.com/cloudevents/sdk-go v0.0.0-20190509003705-56931988abe3/go.mod h1:j1nZWMLGg3om8SswStBoY6/SHvcLM19MuZqwDtMtmzs= -github.com/cloudflare/cloudflare-go v0.13.2 h1:bhMGoNhAg21DuqJjU9jQepRRft6vYfo6pejT3NN4V6A= github.com/cloudflare/cloudflare-go v0.13.2/go.mod h1:27kfc1apuifUmJhp069y0+hwlKDg4bd8LWlu7oKeZvM= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403 h1:cqQfy1jclcSy/FwLjemeg3SR1yaINm74aQyupQ0Bl8M= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 h1:xD/lrqdvwsc+O2bjSSi3YqY73Ke3LAiSCx49aCesA0E= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= -github.com/cockroachdb/errors v1.2.4 h1:Lap807SXTH5tri2TivECb/4abUkMZC9zRoLarvcKDqs= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.1 h1:pASeJT3R3YyVn+94qEPk0SnU1OQ20Jd/T+SPKy9xehY= github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containernetworking/cni v0.7.1 h1:fE3r16wpSEyaqY4Z4oFrLMmIGfBYIKpPrHK31EJ9FzE= github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY= github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/container-linux-config-transpiler v0.9.0 h1:UBGpT8qWqzi48hNLrzMAgAUNJsR0LW8Gk5/dR/caI8U= github.com/coreos/container-linux-config-transpiler v0.9.0/go.mod h1:SlcxXZQ2c42knj8pezMiQsM1f+ADxFMjGetuMKR/YSQ= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.17+incompatible h1:f/Z3EoDSx1yjaIjLQGo1diYUlQYSBrrAQ5vP8NjwXwo= github.com/coreos/etcd v3.3.17+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible h1:bXhRBIXoTm9BYHS3gE0TtQuyNZyeEMux2sDi4oo5YOo= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v0.0.0-20180117170138-065b426bd416/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.0.0-20180108230905-e214231b295a/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -358,90 +275,62 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/ignition v0.35.0 h1:UFodoYq1mOPrbEjtxIsZbThcDyQwAI1owczRDqWmKkQ= github.com/coreos/ignition v0.35.0/go.mod h1:WJQapxzEn9DE0ryxsGvm8QnBajm/XsS/PkrDqSpz+bA= -github.com/coreos/locksmith v0.6.2 h1:yd/7pCRpnmXd1EA9AVbNtMIo5wCxSlRCPzP8wzZEW9Q= github.com/coreos/locksmith v0.6.2/go.mod h1:mSLRr7SVSEAIugjic7+TXif/+ZQQq0zCks1vptuj2fs= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/prometheus-operator v0.35.0 h1:kd7mysk8mCdwquBcPLyuRoRFNJCpgezXu8yUvIYE2nc= github.com/coreos/prometheus-operator v0.35.0/go.mod h1:XHYZUStZWcwd1yk/1DjZv/fywqKIyAJ6pSwvIr+v9BQ= -github.com/cpu/goacmedns v0.0.3 h1:QOeMpIEsIdm1LSASSswjaTf8CXmzcrgy5OeCfHjppA4= github.com/cpu/goacmedns v0.0.3/go.mod h1:4MipLkI+qScwqtVxcNO6okBhbgRrr7/tKXUSgSL0teQ= -github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cristim/ec2-instances-info v0.0.0-20201110114654-2dfcc09f67d4 h1:uPdJfcX6oBDV/n7KYnXipTvZr0Mll06CnH0FYsY5vYY= github.com/cristim/ec2-instances-info v0.0.0-20201110114654-2dfcc09f67d4/go.mod h1:0yCjO4zBzlwWSGh/zGfW2Zq1NX605qCYVBHD1fPXKNs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892 h1:qg9VbHo1TlL0KDM0vYvBG9EY0X0Yku5WYIPoFWt8f6o= github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE= -github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd h1:uVsMphB1eRx7xB1njzL3fuMdWRN8HtVzoUOItHMwv5c= github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= github.com/denisenkom/go-mssqldb v0.0.0-20190111225525-2fea367d496d/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= -github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0 h1:epsH3lb7KVbXHYk7LYGN5EiE0MxcevHU85CKITJ0wUY= github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954 h1:RMLoZVzv4GliuWafOuPuQDKSm1SJph7uCRnnS61JAn4= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/digitalocean/godo v1.44.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= github.com/digitalocean/godo v1.54.0 h1:KP0Nv87pgViR8k/7De3VrmflCL5pJqXbNnkcw0bwG10= github.com/digitalocean/godo v1.54.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= -github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg= github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8= github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017 h1:2HQmlpI3yI9deH18Q6xiSOIjXD4sLI55Y/gfpa8/558= github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible h1:SiUATuP//KecDjpOK2tvZJgeScYAklvyjfK8JZlU6fo= github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.6.3 h1:zI2p9+1NQYdnG6sMU26EX4aVGlqbInSQxQXLvzJ4RPQ= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= -github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c h1:ZfSZ3P3BedhKGUhzj7BQlPSU4OvT6tfOKe3DVHzOA7s= github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/embik/nutanix-client-go v0.0.0-20220106131900-50b8f27e5f60 h1:0FVKOkpksULFs6F7Kfd8ClBXVTvtiIKl07uV3HinOHk= github.com/embik/nutanix-client-go v0.0.0-20220106131900-50b8f27e5f60/go.mod h1:gkKNSxfEt3QtYG3S/wKiN8OmrJ4fpU7JbTlbnrMDOL8= @@ -450,7 +339,6 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT github.com/emicklei/go-restful v2.10.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.11.2+incompatible h1:Z4Z0K2AuOw+QtgwkkJnwpT165MBr12qS8rnBwjP/Pzs= github.com/emicklei/go-restful v2.11.2+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -458,13 +346,9 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d h1:QyzYnTnPE15SQyUeqU6qLbWxMkwyAyu+vGksa0b7j00= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= -github.com/etcd-io/gofail v0.0.0-20190801230047-ad7f989257ca h1:Y2I0lxOttdUKz+hNaIdG3FtjuQrTmwXun1opRV65IZc= github.com/etcd-io/gofail v0.0.0-20190801230047-ad7f989257ca/go.mod h1:49H/RkXP8pKaZy4h0d+NW16rSLhyVBt4o6VLJbmOqDE= github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= @@ -475,70 +359,47 @@ github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwCFad8crR9dcMQWvV9Hvulu6hwUh4tWPJnM= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= -github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db h1:gb2Z18BhTPJPpLQWj4T+rfKHYCHxRHCtRxhKKjRidVw= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8 h1:a9ENSRDFBUPkJ5lCgVZh26+ZbGyoVJG7yb5SSzF5H54= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsouza/fake-gcs-server v0.0.0-20180612165233-e85be23bdaa8 h1:3iml5UHzQtk3cpnYfqW16Ia+1xSuu9tc4BElZu5470M= github.com/fsouza/fake-gcs-server v0.0.0-20180612165233-e85be23bdaa8/go.mod h1:1/HufuJ+eaDf4KTnYdS6HJMGvMRU8d4cYTuu/1QaBbI= -github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/go-bindata/go-bindata v3.1.2+incompatible h1:5vjJMVhowQdPzjE1LdxyFF7YFTXg5IgGVW4gBr5IbvE= github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1 h1:QbL/5oDUmRBzO9/Z7Seo6zf912W/a6Sr4Eu0G/3Jho0= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4 h1:WtGNWLvXpe6ZudgnXrq0barxBImvnnJoMEhXAzcbM0I= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.62.0 h1:7VJT/ZXjzqSrvtraFp4ONq80hTcRQth1c9ZnQ3uNQvU= github.com/go-ini/ini v1.62.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0 h1:dXFJfIHVvUcpSgDOV+Ne6t7jXri8Tfv2uOLHUZ2XNuo= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= -github.com/go-kit/log v0.1.0 h1:DGJh0Sm43HbOeYDNnVZFl8BvcYVvjD5bqYJvp0REbwQ= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-ldap/ldap v3.0.2+incompatible h1:kD5HQcAzlQ7yrhfn+h+MSABeAy/jAJhvIJ/QDllP44g= github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -560,7 +421,6 @@ github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9sn github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ= -github.com/go-openapi/analysis v0.19.16 h1:Ub9e++M8sDwtHD+S587TYi+6ANBG1NRYGZDihqk0SaY= github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk= github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.17.2/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= @@ -570,9 +430,7 @@ github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.19.9 h1:9SnKdGhiPZHF3ttwFMiCBEb8jQ4IDdrK+5+a0oTygA4= github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/inflect v0.19.0 h1:9jCH9scKIbHeV9m12SmPilScz6krDxKRasNNSNPXu/4= github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4= github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= @@ -598,7 +456,6 @@ github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5 github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY= github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= -github.com/go-openapi/loads v0.19.7 h1:6cALLpCAq4tYhaic7TMbEzjv8vq/wg+0AFivNy/Bma8= github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= github.com/go-openapi/runtime v0.17.2/go.mod h1:QO936ZXeisByFmZEO1IS1Dqhtf4QV1sYYFtIq6Ld86Q= @@ -607,7 +464,6 @@ github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29g github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo= github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98= github.com/go-openapi/runtime v0.19.20/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= -github.com/go-openapi/runtime v0.19.24 h1:TqagMVlRAOTwllE/7hNKx6rQ10O6T8ZzeJdMjSTKaD4= github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.17.2/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= @@ -626,7 +482,6 @@ github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6 github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= -github.com/go-openapi/strfmt v0.19.11 h1:0+YvbNh05rmBkgztd6zHp4OCFn7Mtu30bn46NQo2ZRw= github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= @@ -647,91 +502,63 @@ github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7 github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8= github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4= -github.com/go-openapi/validate v0.19.15 h1:oUHZO8jD7p5oRLANlXF0U8ic9ePBUkDQyRZdN0EhL6M= github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI= -github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 h1:JVrqSeQfdhYRFk24TvhTZWU0q8lfCojxZQFi3Ou7+uY= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= github.com/go-sql-driver/mysql v0.0.0-20160411075031-7ebe0a500653/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-swagger/go-swagger v0.25.0 h1:FxhyrWWV8V/A9P6GtI5szWordAdbb6Y0nqdY/y9So2w= github.com/go-swagger/go-swagger v0.25.0/go.mod h1:9639ioXrPX9E6BbnbaDklGXjNz7upAXoNBwL4Ok11Vk= -github.com/go-swagger/scan-repo-boundary v0.0.0-20180623220736-973b3573c013 h1:l9rI6sNaZgNC0LnF3MiE+qTmyBA/tZAg1rtyrGbUMK0= github.com/go-swagger/scan-repo-boundary v0.0.0-20180623220736-973b3573c013/go.mod h1:b65mBPzqzZWxOZGxSWrqs4GInLIn+u99Q9q7p+GKni0= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= -github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd h1:hSkbZ9XSyjyBirMeqSqUrK+9HboWrweVlzRNqoBi2d4= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0 h1:31atYa/UW9V5q8vMJ+W6wd64OaaTHUrCUXER358zLM4= github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= github.com/gobuffalo/envy v1.6.5/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ= github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.1 h1:OQl5ys5MBea7OGCdvPbBJWRgnhC/fGona6QKfvFeau8= github.com/gobuffalo/envy v1.7.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w= github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= github.com/gobuffalo/flect v0.1.5/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= github.com/gobuffalo/flect v0.2.0/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= -github.com/gobuffalo/flect v0.2.2 h1:PAVD7sp0KOdfswjAw9BpLCU9hXo7wFSzgpQ+zNeks/A= github.com/gobuffalo/flect v0.2.2/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1 h1:iQ0D6SpNXIxu52WESsD+KoQ7af2e3nCfnSBoSF/hKe0= github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211 h1:mSVZ4vj4khv+oThUfS+SQU3UuFIZ5Zo6UNcvK8E8Mz8= github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1 h1:dLg+zb+uOyd/mKeQUYIbwbNmfRsr9hd/WtYWepmayhI= github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/logger v1.0.1 h1:ZEgyRGgAm4ZAhAO45YXMs5Fp+bzGLESFewzAVBMKuTg= github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs= github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2 h1:fq9WcL1BYrm36SzK6+aAnZ8hcp+SrmnDyAxhNx8dvJk= github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.3.0 h1:eMwymTkA1uXsqxS0Tpoop3Lc0u3kTfiMBE6nKtQU4g4= github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/packr/v2 v2.7.1 h1:n3CIW5T17T8v4GGK5sWXLVWJhCz7b5aNLSxW6gYim4o= github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kETFqjFoFlOc= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754 h1:tpom+2CJmpzAWj5/VEHync2rJGi+epHNIeRSWjzGA+4= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godror/godror v0.13.3 h1:4A5GLGAJTSuELw1NThqY5bINYB+mqrln+kF5C2vuyCs= github.com/godror/godror v0.13.3/go.mod h1:2ouUT4kdhUBk7TAkHWD4SN0CdI0pgEQbo8FVHhbSKWg= -github.com/gogo/googleapis v1.1.0 h1:kFkMAZBNAn4j7K0GiZr8cRYzejq68VbheufiV3YuyFI= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -744,11 +571,8 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d h1:lBXNCxVENCipq4D1Is42JVOP4eQjlB8TQ6H69Yx5J9Q= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -758,7 +582,6 @@ github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/lint v0.0.0-20180702182130-06c8688daad7 h1:2hRPrmiwPrp3fQX967rNJIhQPtiGXdlQWAxKbKw3VHA= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -766,7 +589,6 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v0.0.0-20181025225059-d3de96c4c28e/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -789,20 +611,14 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450 h1:7xqw01UYS+KCI25bMrPxwNYkSns2Db1ziQPpVq99FpE= github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= -github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995 h1:f5gsjBiF9tRRVomCvrkGMMWI8W1f2OBFar2c5oakAP0= github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= -github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e h1:KhcknUwkWHKZPbFy2P7jH5LKJ3La+0ZeknkkmrSgqb0= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= -github.com/gomodule/redigo v1.7.0 h1:ZKld1VOtsGhAe37E7wMxEDgAlGM5dvFY+DiOhSkhP9Y= github.com/gomodule/redigo v1.7.0/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -816,11 +632,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-containerregistry v0.0.0-20200115214256-379933c9c22b h1:oGqapkPUiypdS9ch/Vu0npPe03RQ0BhVDYli+OEKNAA= github.com/google/go-containerregistry v0.0.0-20200115214256-379933c9c22b/go.mod h1:Wtl/v6YdQxv397EREtzwgd9+Ud7Q5D8XMbi3Zazgkrs= -github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-licenses v0.0.0-20191112164736-212ea350c932 h1:ZIb3nb+/mHAGRkyuxfPykmYdUi21mr8YTGpr/xGPJ8o= github.com/google/go-licenses v0.0.0-20191112164736-212ea350c932/go.mod h1:16wa6pRqNDUIhOtwF0GcROVqMeXHZJ7H6eGDFUh5Pfk= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -829,7 +642,6 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/licenseclassifier v0.0.0-20190926221455-842c0d70d702 h1:nVgx26pAe6l/02mYomOuZssv28XkacGw/0WeiTVorqw= github.com/google/licenseclassifier v0.0.0-20190926221455-842c0d70d702/go.mod h1:qsqn2hxC+vURpyBRygGUuinTO42MFRLcsmQ/P8v94+M= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -845,9 +657,7 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201117184057-ae444373da19 h1:iFELRewmQ9CldLrqgr0E6b6ZPfZmMvLyyz6kMsR+c4w= github.com/google/pprof v0.0.0-20201117184057-ae444373da19/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v0.0.0-20170306145142-6a5e28554805/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -871,41 +681,32 @@ github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2c github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gophercloud/gophercloud v0.14.0 h1:c2Byo+YMxhHlTJ3TPptjQ4dOQ1YknTHDJ/9zClDH+84= github.com/gophercloud/gophercloud v0.14.0/go.mod h1:VX0Ibx85B60B5XOrZr6kaNwrmPUzcmMpwxvQ1WQIIWM= +github.com/gophercloud/gophercloud v0.24.0 h1:jDsIMGJ1KZpAjYfQgGI2coNQj5Q83oPzuiGJRFWgMzw= +github.com/gophercloud/gophercloud v0.24.0/go.mod h1:Q8fZtyi5zZxPS/j9aj3sSxtvj41AdQMDwyo1myduD5c= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/csrf v1.6.2 h1:QqQ/OWwuFp4jMKgBFAzJVW3FMULdyUW7JoM4pEWuqKg= github.com/gorilla/csrf v1.6.2/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI= github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v0.0.0-20181024020800-521ea7b17d02/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/sessions v1.1.3 h1:uXoZdcdA5XdXF3QzuSlheVRUvjl+1rKY7zBXL68L9RU= github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc h1:f8eY6cV/x1x+HLjOp4r72s/31/V2aTUtg5oKRRPf8/Q= github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170330212424-2500245aa611/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= @@ -915,89 +716,58 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.15.2/go.mod h1:vO11I9oWA+KsxmfFQPhLnnIb1VDE24M+pdxZFiuZcA8= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/h2non/gock v1.0.9 h1:17gCehSo8ZOgEsFKpQgqHiR7VLyjxdAG3lkhVvO9QZU= github.com/h2non/gock v1.0.9/go.mod h1:CZMcB0Lg5IWnr9bF79pPMg9WeV6WumxQiUJ1UvdO1iE= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/api v1.3.0 h1:HXNYlRkkM/t+Y/Yhxtwcy02dlYwIaoxzvxPnS+cqy78= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.3.0 h1:UOxjlb4xVNF93jak1mzzoBatyFju9nrkxpVwIp/QqxQ= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= -github.com/hashicorp/go-hclog v0.8.0 h1:z3ollgGRg8RjfJH6UVBaG54R70GFd++QOkvnJH3VSBY= github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-plugin v1.0.1 h1:4OtAfUGbnKC6yS48p0CtMX2oFYtzFZVv6rok3cRWgnE= github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= -github.com/hashicorp/go-retryablehttp v0.5.4 h1:1BZvpawXoJCWX6pNtow9+rpEj+3itIlutiqnntI6jOE= github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-rootcerts v1.0.1 h1:DMo4fmknnz0E0evoNYnV48RjWndOsmd6OW+09R3cEP8= github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= -github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1 h1:sNCoNyDEvN1xa+X0baata4RdcpKwcMS6DH+xwfqPgjw= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0 h1:WhIgCr5a7AaVH6jPUwjtRuuE7/RDufnUvzIr48smyxs= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hashicorp/vault/api v1.0.4 h1:j08Or/wryXT4AcHj1oCbMd7IijXcKzYUGw59LGu9onU= github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= -github.com/hashicorp/vault/sdk v0.1.13 h1:mOEPeOhT7jl0J4AMl1E705+BcmeRs1VmKNb9F0sMLy8= github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= -github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d h1:kJCB4vdITiW1eC1vq2e6IsrXKrZit1bv/TDYFGMp4BQ= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 h1:GT4RsKmHh1uZyhmTkWJTDALRjSHYQp6FRKrotf0zhAs= github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= github.com/hetznercloud/hcloud-go v1.23.1/go.mod h1:xng8lbDUg+xM1dgc0yGHX5EeqbwIq7UYlMWMTx3SQVg= github.com/hetznercloud/hcloud-go v1.25.0 h1:QAaFKtGKWRxjwjKJWBGMxGYUxVEQmIkb35j/WXrsazY= github.com/hetznercloud/hcloud-go v1.25.0/go.mod h1:2C5uMtBiMoFr3m7lBFPf7wXTdh33CevmZpQIIDPGYJI= -github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c h1:kQWxfPIHVLbgLzphqk3QUflDy9QdksZR4ygR807bpy0= github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/hudl/fargo v1.3.0 h1:0U6+BtN6LhaYuTnIJq4Wyq5cpn6O2kWrxAtcqBmYY6w= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= -github.com/iancoleman/strcase v0.1.2 h1:gnomlvw9tnV3ITTAxzKSgTF+8kFWcU/f+TgttpXGz1U= github.com/iancoleman/strcase v0.1.2/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.0.0-20171009183408-7fe0c75c13ab/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -1008,49 +778,32 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/improbable-eng/thanos v0.3.2 h1:iZfU7exq+RD5Lnb8n3Eh9MNYoRLeyeGO/85AvEkLg+8= github.com/improbable-eng/thanos v0.3.2/go.mod h1:GZewVGILKuJVPNRn7L4Zw+7X96qzFOwj63b22xYGXBE= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb v0.0.0-20161215172503-049f9b42e9a5 h1:AciJ2ei/llFRundm7CtqwF6B2aOds1A7QG3sMW8QiaQ= github.com/influxdata/influxdb v0.0.0-20161215172503-049f9b42e9a5/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d h1:/WZQPMZNsjZ7IlCpsLGdQBINg5bxKQ1K1sh6awxLtkA= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jedib0t/go-pretty v4.3.0+incompatible h1:CGs8AVhEKg/n9YbUenWmNStRW2PHJzaeDodcfvRAbIo= github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= -github.com/jenkins-x/go-scm v1.5.65 h1:ieH+0JSWENObn1SDWFj2K40iV5Eia4aTl6W6bDdLwI0= github.com/jenkins-x/go-scm v1.5.65/go.mod h1:MgGRkJScE/rJ30J/bXYqduN5sDPZqZFITJopsnZmTOw= -github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jetstack/cert-manager v1.1.0 h1:gEhBV9I83m+kpQShDhNO4+J8O2qfNDjvAEL27pThGmg= github.com/jetstack/cert-manager v1.1.0/go.mod h1:GULIHTGjSc2LjlgBCLhQ8u5WmQ95hk9FAiQbhjMthMk= -github.com/jinzhu/gorm v0.0.0-20170316141641-572d0a0ab1eb h1:0D5F4qAGJbRqzyCIHswU2fCwB1XGTDkBwBn9qncQYYs= github.com/jinzhu/gorm v0.0.0-20170316141641-572d0a0ab1eb/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= -github.com/jinzhu/inflection v0.0.0-20190603042836-f5c5f50e6090 h1:LIwA5USOJ9W/0hwiRH1MugeThGBHGqv+USXcDKWHIVY= github.com/jinzhu/inflection v0.0.0-20190603042836-f5c5f50e6090/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.0.1 h1:HjfetcXq097iXP0uoPCdnM4Efp5/9MsM0/M+XOTeR3M= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joefitzgerald/rainbow-reporter v0.1.0 h1:AuMG652zjdzI0YCCnXAqATtRBpGXMcAnrajcaTrSeuo= github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= -github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.0.0-20141017032234-72f9bd7c4e0c/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= @@ -1062,7 +815,6 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jsonnet-bundler/jsonnet-bundler v0.1.0 h1:T/HtHFr+mYCRULrH1x/RnoB0prIs0rMkolJhFMXNC9A= github.com/jsonnet-bundler/jsonnet-bundler v0.1.0/go.mod h1:YKsSFc9VFhhLITkJS3X2PrRqWG9u2Jq99udTdDjQLfM= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= @@ -1070,42 +822,27 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20191119172530-79f836b90111 h1:Lq6HJa0JqSg5ko/mkizFWlpIrY7845g9Dzz9qeD5aXI= github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20191119172530-79f836b90111/go.mod h1:MP2HbArq3QT+oVp8pmtHNZnSnkhdkHtDnc7h6nJXmBU= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3 h1:lOpSw2vJP0y5eLBW906QwKsUK/fe/QDyoqM5rnnuPDY= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/kelseyhightower/envconfig v1.3.0 h1:IvRS4f2VcIQy6j4ORGIf9145T/AsUB+oY8LyvN8BXNM= github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/kinvolk/container-linux-config-transpiler v0.9.1 h1:LIv3RCbjdFhXn/Fg4XHys3sBekkPHM0uxKfAzD0F2jk= github.com/kinvolk/container-linux-config-transpiler v0.9.1/go.mod h1:pjTzCvFfbXjWuMVNFjA9FdbsdmruK6+vki0hK0lDmnU= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= -github.com/kisielk/errcheck v1.5.0 h1:e8esj/e4R+SAOwFwN+n3zr0nYeCyeweozKfO23MvHzY= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.5 h1:U+CaK85mrNNb4k8BNOfgJtJ/gr6kswUCFj6miSzVC6M= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM= github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/knative/build v0.1.2 h1:o/VYWA3HKyZlNqdU2hDE5LHpanBe8gazgPKL97XJ6bo= github.com/knative/build v0.1.2/go.mod h1:/sU74ZQkwlYA5FwYDJhYTy61i/Kn+5eWfln2jDbw3Qo= github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -1114,7 +851,6 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= -github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -1123,28 +859,19 @@ github.com/kubermatic/machine-controller v1.23.1/go.mod h1:mXWbT7SjqpgFhzCFT3yME github.com/kubermatic/machine-controller v1.26.0/go.mod h1:dcJ+GdDSCxCwM0poxwOK8hVO7epiOORDmNMmb2veyw4= github.com/kubermatic/machine-controller v1.36.1/go.mod h1:6BFZEvEMZi8OT8aHOsS7DXYsF6ZSpmsNxsci7OLTTn8= github.com/kubermatic/machine-controller v1.40.1/go.mod h1:5LVcN4tCybGg+55hIHcVzCjNsBJy2PlnXG0xIzKmXGY= -github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.1-0.20191011153232-f91d3411e481 h1:r9fnMM01mkhtfe6QfLrr/90mBVLnJHge2jGeBvApOjk= github.com/lib/pq v1.2.1-0.20191011153232-f91d3411e481/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743 h1:143Bb8f8DuGWck/xpNUOckBVYfFbBTnLevfRZ1aVVqo= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1 h1:vi1F1IQ8N7hNWytK9DpJsUfQhGuNSc19z330K6vl4zk= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linode/linodego v0.24.0 h1:o6hNS0T7jeikOfUHoJhUhA/e2QTCsw9MGccVmRHRLE4= github.com/linode/linodego v0.24.0/go.mod h1:GSBKPpjoQfxEfryoCRcgkuUOCuVtGHWhzI8OMdycNTE= -github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= -github.com/lyft/protoc-gen-validate v0.0.13 h1:KNt/RhmQTOLr7Aj8PsJ7mTronaFyx80mRTT9qF261dA= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -1156,51 +883,37 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/markbates/inflect v1.0.4 h1:5fh1gzTFhfae06u3hzHYO9xe3l3v3nW5Pwt3naLTP5g= github.com/markbates/inflect v1.0.4/go.mod h1:1fR9+pO2KHEO9ZRtto13gDwwZaAKstQzferVeWqbgNs= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2 h1:JgVTCPf0uBVcUSWpyXmGpgOc62nK5HWUBKAGc3Qqa5k= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a h1:+J2gw7Bw77w/fbK7wnNJJDKmw1IbWft2Ul5BzrG1Qm8= github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.6/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-oci8 v0.0.7 h1:BBXYpvzPO43QNTLDEivPFteeFZ9nKA6JQ6eifpxOmio= github.com/mattn/go-oci8 v0.0.7/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= github.com/mattn/go-runewidth v0.0.0-20181025052659-b20a3daf6a39/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.5/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v0.0.0-20160514122348-38ee283dabf1/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-sqlite3 v1.12.0 h1:u/x3mp++qUxvYfulZ4HKOvVO0JWhk7HtE8lWhbGz/Do= github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-zglob v0.0.1 h1:xsEx/XUoVlI6yXjqBK062zYhRTZltCNmYPx6v+8DNaY= github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2 h1:g+4J5sZg6osfvEfkRZxJ1em0VT95/UOZgi/l7zi1/oE= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.31 h1:sJFOl9BgwbYAWOGEwr61FU28pqsBNdpRBnhGXtO06Oo= github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o= github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= -github.com/mitchellh/cli v1.0.0 h1:iGBIsUe3+HZ/AD/Vd7DErOt5sU9fa8Uj7A2s1aggv1Y= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= @@ -1208,31 +921,22 @@ github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= -github.com/mitchellh/gox v0.4.0 h1:lfGJxY7ToLJQjHHwi0EX6uYBdK78egf954SQl13PQJc= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452 h1:hOY53G+kBFhbYFpRVxHl5eS7laP6B1+Cq+Z9Dry1iMU= github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= -github.com/mitchellh/iochan v1.0.0 h1:C+X3KsSTLFVBr/tK1eYN/vs4rJcvsiLU338UhYPJWeY= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/ioprogress v0.0.0-20180201004757-6a23b12fa88e h1:Qa6dnn8DlasdXRnacluu8HzPts0S1I9zvvUPDbBnXFI= github.com/mitchellh/ioprogress v0.0.0-20180201004757-6a23b12fa88e/go.mod h1:waEya8ee1Ro/lgxpVhkJI4BVASzkm3UZqkx/cFJiYHM= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.0 h1:7ks8ZkOP5/ujthUsT07rNv+nkLXCQWKNHuwzOAesEks= github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297 h1:yH0SvLzcbZxcJXho2yh7CqdENGMQe73Cw3woZBpPli0= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -1240,63 +944,41 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/munnerz/crd-schema-fuzz v1.0.0 h1:8erI9yzEnOGw9K5O+a8zZdoo8N/OwrFi7c7SjBtkHAs= github.com/munnerz/crd-schema-fuzz v1.0.0/go.mod h1:4z/rcm37JxUkSsExFcLL6ZIT1SgDRdLiu7qq1evdVS0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= -github.com/nats-io/gnatsd v1.4.1 h1:RconcfDeWpKCD6QIIwiVFcvForlXpWeJP7i5/lDLy44= github.com/nats-io/gnatsd v1.4.1/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ= -github.com/nats-io/go-nats v1.7.0 h1:oQOfHcLr8hb43QG8yeVyY2jtarIaTjOv41CGdF3tTvQ= github.com/nats-io/go-nats v1.7.0/go.mod h1:+t7RHT5ApZebkrQdnn6AhQJmhJJiKAvJUio1PiiCtj0= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2 h1:i2Ly0B+1+rzNZHHWtD4ZwKi+OU5l+uQo1iDHZ2PmiIc= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.9.1 h1:ik3HbLhZ0YABLto7iX80pZLPw/6dx3T+++MZJwLnMrQ= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3 h1:6JrEfig+HzTH85yxzhSVbjHRJv9cn0p6n3IngIcM5/k= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.0/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= -github.com/nelsam/hel v0.0.0-20200611165952-2d829bae0c66 h1:C6JDvVh+cs4f8TJXRaJAOY59BC5knehTmdbMYhVfdhA= github.com/nelsam/hel v0.0.0-20200611165952-2d829bae0c66/go.mod h1:Rl/hm4V2s75ScsPmI9cNz87HLNg5MoFAMJwA90fzbkw= -github.com/nelsam/hel/v2 v2.3.2 h1:tXRsJBqRxj4ISSPCrXhbqF8sT+BXA/UaIvjhYjP5Bhk= github.com/nelsam/hel/v2 v2.3.2/go.mod h1:1ZTGfU2PFTOd5mx22i5O0Lc2GY933lQ2wb/ggy+rL3w= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/oklog v0.3.2 h1:wVfs8F+in6nTBMkA7CbRw+zZMIB7nNM825cM1wuzoTk= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= -github.com/olekukonko/tablewriter v0.0.4 h1:vHD/YYe1Wolo78koG299f7V/VAS08c6IpCLn+Ejf/w8= github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -1327,108 +1009,75 @@ github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDs github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/open-policy-agent/cert-controller v0.0.0-20200921224206-24b87bbc4b6e h1:kKIQk82R8CKO8mQV4sWNl0zR+UcHLAhTwwQ8y+1o02s= github.com/open-policy-agent/cert-controller v0.0.0-20200921224206-24b87bbc4b6e/go.mod h1:/y33mmiq3Cc0N+6cickevrLI/iBbWcUwcEVjSKHA0z0= github.com/open-policy-agent/frameworks/constraint v0.0.0-20200929072634-d96896eff389/go.mod h1:Dr3QxvH+NTQcPPZWSt1ueNOsxW4VwgUltaLL7Ttnrac= -github.com/open-policy-agent/frameworks/constraint v0.0.0-20201118071520-0d37681951a4 h1:dOkENO1IWL75u2N5VPIlj773vhlpkrcJAPb6yHenjY4= github.com/open-policy-agent/frameworks/constraint v0.0.0-20201118071520-0d37681951a4/go.mod h1:vvhkBONv7Uah2fvS/bQ/N1u0rSLvxZOs2ErR6m+4QtQ= -github.com/open-policy-agent/gatekeeper v0.0.0-20201111000257-4450f08fa95e h1:CA8XSPSbDLQ096bsjQttT24tVWyfd0lbbP9eWYGOP7s= github.com/open-policy-agent/gatekeeper v0.0.0-20201111000257-4450f08fa95e/go.mod h1:IseSnWz7MX7IhEpZ4CLhA3NrMazc+T6a5rtSq9pOEc4= github.com/open-policy-agent/opa v0.19.1/go.mod h1:rrwxoT/b011T0cyj+gg2VvxqTtn6N3gp/jzmr3fjW44= github.com/open-policy-agent/opa v0.21.0/go.mod h1:cZaTfhxsj7QdIiUI0U9aBtOLLTqVNe+XE60+9kZKLHw= -github.com/open-policy-agent/opa v0.24.0 h1:fnGOIux+TTGZsC0du1bRBtV8F+KPN55Hks12uE3Fq3E= github.com/open-policy-agent/opa v0.24.0/go.mod h1:qEyD/i8j+RQettHGp4f86yjrjvv+ZYia+JHCMv2G7wA= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/openshift/api v0.0.0-20191219222812-2987a591a72c h1:WRWMmqacvmZDbUat6WYqpuCy2yEfIeDsxFD/Htgp2T0= github.com/openshift/api v0.0.0-20191219222812-2987a591a72c/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= -github.com/openshift/client-go v0.0.0-20191125132246-f6563a70e19a h1:Otk3CuCAEHiMUr4Er6b+csq4Ar6qilAs9h93tbea+qM= github.com/openshift/client-go v0.0.0-20191125132246-f6563a70e19a/go.mod h1:6rzn+JTr7+WYS2E1TExP4gByoABxMznR6y2SnUIkmxk= github.com/openshift/custom-resource-status v0.0.0-20190822192428-e62f2f3b79f3/go.mod h1:GDjWl0tX6FNIj82vIxeudWeSx2Ff6nDZ8uJn0ohUFvo= github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca h1:F1MEnOMwSrTA0YAkO0he9ip9w0JhYzI/iCB2mXmaSPg= github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca/go.mod h1:GDjWl0tX6FNIj82vIxeudWeSx2Ff6nDZ8uJn0ohUFvo= -github.com/openshift/prom-label-proxy v0.1.1-0.20191016113035-b8153a7f39f1 h1:GW8OxGwBbI2kCqjb5PQfVXRAuCJbYyX1RYs9R3ISjck= github.com/openshift/prom-label-proxy v0.1.1-0.20191016113035-b8153a7f39f1/go.mod h1:p5MuxzsYP1JPsNGwtjtcgRHHlGziCJJfztff91nNixw= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 h1:lM6RxxfUMrYL/f8bWEUqdXrANWtrL7Nndbm9iFN0DlU= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5 h1:ZCnq+JUrvXcDVhX/xRolRBZifmabN1HcS1wrPSvxhrU= github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.2.0/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2 h1:nY8Hti+WKaP0cRsSeQ026wU03QsM762XBeCXBb9NAWI= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/otiai10/copy v1.0.2 h1:DDNipYy6RkIkjMwy+AWzgKiNTyj2RUI9yEMeETEpVyc= github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY= -github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95 h1:+OLn68pqasWca0z5ryit9KGfp3sUsW4Lqg32iRMJyzs= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= -github.com/otiai10/mint v1.3.0 h1:Ady6MKVezQwHBkGzLFbrsywyp09Ah7rkmfjV3Bcr5uc= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888 h1:ARzSqjQJcSR9IXGMEkXN1IQEZChwd2MqXWm6YNGvO5o= github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888/go.mod h1:RQHg5xR1F614BwJyepfMqrKN+32IH0i7yX+ey43rEeQ= -github.com/packethost/pkg v0.0.0-20200903155310-0433e0605550 h1:/ojL7LAVjyH1MY+db0+j6rcWU3UWWpzHksYFsHWs9vQ= github.com/packethost/pkg v0.0.0-20200903155310-0433e0605550/go.mod h1:GSv7cTtIjns4yc0pyajaM1RE/KE4djJONoblFIRDrxA= -github.com/pact-foundation/pact-go v1.0.4 h1:OYkFijGHoZAYbOIb1LWXrwKQbMMRUv1oQ89blD2Mh2Q= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pavel-v-chernykh/keystore-go v2.1.0+incompatible h1:Jd6xfriVlJ6hWPvYOE0Ni0QWcNTLRehfGPFxr3eSL80= github.com/pavel-v-chernykh/keystore-go v2.1.0+incompatible/go.mod h1:xlUlxe/2ItGlQyMTstqeDv9r3U4obH7xYd26TbDQutY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-buffruneio v0.2.0 h1:U4t4R6YkofJ5xHm3dJzuRpPZ0mr5MMCoAWooScCR7aA= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/pelletier/go-toml v1.8.0 h1:Keo9qb7iRJs2voHvunFtuuYFsbWeOBh8/P9v/kVMFtw= github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= -github.com/performancecopilot/speed v3.0.0+incompatible h1:2WnRzIquHa5QxaJKShDkLM+sc0JPuwhXzK8OYOyt3Vg= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= -github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d h1:zapSxdmZYY6vJWXFKLQ+MkI+agc+HQyfrCGowDSHiKs= github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4 v2.2.6+incompatible h1:6aCX4/YZ9v8q69hTyiR7dNLnTA3fgtKHVVW5BCd5Znw= github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.0.0-20181023235946-059132a15dd0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1 h1:F++O52m40owAmADcojzM+9gyjmMOY/T4oYJkgFDH8RE= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= -github.com/pkg/sftp v1.10.1 h1:VasscCm72135zRysgrJDKsntdmPN+OuU3+nnHYA9wyc= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1 h1:ccV59UEOTzVDnDUEFdT95ZzHVZ+5+158q8+SJb2QV5w= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/poy/onpar v0.0.0-20200406201722-06f95a1c68e8/go.mod h1:nSbFQvMj97ZyhFRSJYtut+msi4sOY6zJDGCdSc+/rZU= -github.com/poy/onpar v1.0.1 h1:IzLQJa3wxHFXVU8tojF1fw5coZ3CV+9OrnDYZ7GBRy0= github.com/poy/onpar v1.0.1/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU= github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.0.0-20181025174421-f30f42803563/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -1484,58 +1133,38 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/prometheus v2.3.2+incompatible h1:EekL1S9WPoPtJL2NZvL+xo38iMpraOnyEHOiyZygMDY= github.com/prometheus/prometheus v2.3.2+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/prometheus/tsdb v0.8.0 h1:w1tAGxsBMLkuGrFMhqgcCeBkM5d1YI24udArs+aASuQ= github.com/prometheus/tsdb v0.8.0/go.mod h1:fSI0j+IUQrDd7+ZtR9WKIGtoYAYAJUKcKhYLG25tN4g= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20190706150252-9beb055b7962/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 h1:/NRJ5vAYoqz+7sG51ubIDHXeWO8DlTSrToPu6q11ziA= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= -github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.4.0 h1:LUa41nrWTQNGhzdsZ5lTnkwbNjj6rXTdazA1cSdjkOY= github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rollbar/rollbar-go v1.0.2 h1:uA3+z0jq6ka9WUUt9VX/xuiQZXZyWRoeKvkhVvLO9Jc= github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= -github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351 h1:HXr/qUllAWv9riaI4zh2eXWKmCSDqVS/XH1MRHLKRwk= github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg= -github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c h1:ht7N4d/B7Ezf58nvMNVF3OlvDlz9pp+WHVcRNS0nink= github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= -github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/ryanuber/columnize v2.1.0+incompatible h1:j1Wcmh8OrK4Q7GXY+V7SVSY8nUWQxHW5TkBe7YUl+2s= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7 h1:Do8ksLD4Nr3pA0x0hnLOLftZgkiTDvwPDShRTUxtXpE= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8= -github.com/sclevine/spec v1.2.0 h1:1Jwdf9jSfDl9NVmt8ndHqbTZ7XCCPbh1jI3hkDBHVYA= github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI= github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE= @@ -1543,14 +1172,10 @@ github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXY github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/githubv4 v0.0.0-20180925043049-51d7b505e2e9/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= github.com/shurcooL/githubv4 v0.0.0-20190718010115-4ba037080260/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= -github.com/shurcooL/githubv4 v0.0.0-20191102174205-af46314aec7b h1:Cocq9/ZZxCoiybhygOR7hX4E3/PkV8eNbd1AEcUvaHM= github.com/shurcooL/githubv4 v0.0.0-20191102174205-af46314aec7b/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= -github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM= github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= github.com/shurcooL/graphql v0.0.0-20180924043259-e4a3a37e6d42/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= -github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f h1:tygelZueB1EtXkPI6mQ4o9DQ0+FKW41hTbunoXZCTqk= github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= @@ -1571,15 +1196,11 @@ github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIK github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/sony/gobreaker v0.4.1 h1:oMnRNZXX5j85zso6xCPRNPtmAycat+WcoKbklScLDgQ= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.3.2 h1:GDarE4TJQI52kYSbSAmLiId1Elfj+xgSDqrUZxFhxlU= github.com/spf13/afero v1.3.2/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= @@ -1592,10 +1213,8 @@ github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHN github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.0.1-0.20200713175500-884edc58ad08/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v0.0.0-20181024212040-082b515c9490/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -1606,22 +1225,15 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= -github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= -github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/stormcat24/protodep v0.0.0-20200505140716-b02c9ba62816 h1:vG00k+DtOBlp5ug3cQdaMEsaIncIW0bzfgbhQ7qqdXg= github.com/stormcat24/protodep v0.0.0-20200505140716-b02c9ba62816/go.mod h1:mBd5PI4uI6NkqJpCyiWiYzWyTFs4QRDss/JTMC2b4kc= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 h1:WhxRHzgeVGETMlmVfqhRn8RIeeNoPr2Czh33I4Zdccw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a h1:AhmOdSHeswKHBjhsLs/7+1voOxT+LLrSk/Nxvk35fug= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -1630,45 +1242,31 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tektoncd/pipeline v0.10.1 h1:pDsYK2b70o/Ze/CE1nisELwKVVE54FxwyfLznsW1JiE= github.com/tektoncd/pipeline v0.10.1/go.mod h1:D2X0exT46zYx95BU7ByM8+erpjoN7thmUBvlKThOszU= -github.com/tektoncd/plumbing v0.0.0-20191216083742-847dcf196de9 h1:Iu6stVfs72OBV0c3srVX0oogjhLu+stqlvKHT41+pTI= github.com/tektoncd/plumbing v0.0.0-20191216083742-847dcf196de9/go.mod h1:QZHgU07PRBTRF6N57w4+ApRu8OgfYLFNqCDlfEZaD9Y= -github.com/tektoncd/plumbing/pipelinerun-logs v0.0.0-20191206114338-712d544c2c21 h1:9qeyrQsoPZbHOyOPt0OeB1TCYXfYb5swrxlFWzTIYYk= github.com/tektoncd/plumbing/pipelinerun-logs v0.0.0-20191206114338-712d544c2c21/go.mod h1:S62EUWtqmejjJgUMOGB1CCCHRp6C706laH06BoALkzU= -github.com/testcontainers/testcontainers-go v0.9.0 h1:ZyftCfROjGrKlxk3MOUn2DAzWrUtzY/mj17iAkdUIvI= github.com/testcontainers/testcontainers-go v0.9.0/go.mod h1:b22BFXhRbg4PJmeMVWh6ftqjyZHgiIl3w274e9r3C2E= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda h1:uAHwUH+06gowZMVLqQXm7jN1y3Sl+CDJHThNiKyLHus= github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda/go.mod h1:s4k7CORR0OMWd4cYwBqNBFPSJZhnSQxeKdDtMa/aspk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/toqueteos/webbrowser v1.2.0 h1:tVP/gpK69Fx+qMJKsLE7TD8LuGWPnEV71wBN9rrstGQ= github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJMir8RTqb4ayM= github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/urfave/cli v1.18.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli/v2 v2.1.1 h1:Qt8FeAtxE/vfdrLmR3rxR6JRE0RoVmbXu8+6kZtYU4k= github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/vdemeester/k8s-pkg-credentialprovider v0.0.0-20200107171650-7c61ffa44238/go.mod h1:JwQJCMWpUDqjZrB5jpw0f5VbN7U95zxFy1ZDpoEarGo= -github.com/vdemeester/k8s-pkg-credentialprovider v1.13.12-1 h1:IBEhRIcu5HP+Pkhzn9E9z3wV0tp3TFjDkiAQtX2FXFM= github.com/vdemeester/k8s-pkg-credentialprovider v1.13.12-1/go.mod h1:Fko0rTxEtDW2kju5Ky7yFJNS3IcNvW8IPsp4/e9oev0= -github.com/vektah/gqlparser v1.1.2 h1:ZsyLGn7/7jDNI+y4SEhI4yAxRChlv15pUHMjijT+e68= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50 h1:uxE3GYdXIOfhMv3unJKETJEhw78gvzuQqRX/rVirc2A= @@ -1676,58 +1274,38 @@ github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50/go.mod h github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/vmware/govmomi v0.23.1 h1:vU09hxnNR/I7e+4zCJvW+5vHu5dO64Aoe2Lw7Yi/KRg= github.com/vmware/govmomi v0.23.1/go.mod h1:Y+Wq4lst78L85Ge/F8+ORXIWiKYqaro1vhAulACy9Lc= -github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728 h1:sH9mEk+flyDxiUa5BuPiuhDETMbzrt9A20I2wktMvRQ= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= -github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 h1:j2hhcujLRHAg872RWAV5yaUrEjHEObwDv3aImCaNLek= github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 h1:ESFSdwYZvkeru3RtdrYueztKhOBCSAAzS4Gf+k0tEow= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yashtewari/glob-intersection v0.0.0-20180916065949-5c77d914dd0b h1:vVRagRXf67ESqAb72hG2C/ZwI8NtJF2u2V76EsuOHGY= github.com/yashtewari/glob-intersection v0.0.0-20180916065949-5c77d914dd0b/go.mod h1:HptNXiXVDcJjXe9SqMd0v2FsL9f8dz4GnXgltU6q/co= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5 h1:dPmz1Snjq0kmkz159iL7S6WzdahUTHnHB5M56WFVifs= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.etcd.io/bbolt v1.3.1-etcd.7/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.0.0-20181031231232-83304cfc808c/go.mod h1:weASp41xM3dk0YHg1s/W8ecdGP5G4teSTMBPpYAaUgA= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489 h1:1JFLBqwIgdyHN1ZtgjTBwO+blA6gVOmZurpiMEsETKo= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= -go.etcd.io/etcd/api/v3 v3.5.0 h1:GsV3S+OfZEOCNXdtNkBSR7kgLobAa/SO6tCxRa0GAYw= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0 h1:2aQv6F436YnN7I4VbI8PPYrBhu+SmrTaADcf8Mi/6PU= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0 h1:ftQ0nOOHMcbMS3KIaDQ0g5Qcd6bhaBrQT6b89DfwLTs= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.etcd.io/etcd/client/v3 v3.5.0 h1:62Eh0XOro+rDwkrypAGDfgmNh5Joq+z+W9HZdlXMzek= go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= -go.etcd.io/etcd/pkg/v3 v3.5.0 h1:ntrg6vvKRW26JRmHTE0iNlDgYK6JX3hg/4cD62X0ixk= go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= -go.etcd.io/etcd/raft/v3 v3.5.0 h1:kw2TmO3yFTgE+F0mdKkG7xMxkit2duBDa2Hu6D/HMlw= go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= -go.etcd.io/etcd/server/v3 v3.5.0 h1:jk8D/lwGEDlQU9kZXUFMSANkE22Sg5+mW27ip8xcF9E= go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= -go.etcd.io/etcd/v3 v3.3.0-rc.0.0.20200728214110-6c81b20ec8de h1:RlSimOq2hFUa35bBSmFQC+Wo/diJwbux2t/T7ZNPwsw= go.etcd.io/etcd/v3 v3.3.0-rc.0.0.20200728214110-6c81b20ec8de/go.mod h1:UENlOa05tkNvLx9VnNziSerG4Ro74upGK6Apd4v6M/Y= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -1735,7 +1313,6 @@ go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qL go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= go.mongodb.org/mongo-driver v1.3.5/go.mod h1:Ual6Gkco7ZGQw8wE1t4tLnvBsf6yVSM60qW6TgOeJ5c= -go.mongodb.org/mongo-driver v1.4.3 h1:moga+uhicpVshTyaqY9L23E6QqwcHRUv1sqyOsoyOO8= go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= go.opencensus.io v0.17.0/go.mod h1:mp1VrMQxhlqqDpKvH4UcQUa4YwlzNmymAjPrDdfxNpI= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -1748,29 +1325,17 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opentelemetry.io/contrib v0.20.0 h1:ubFQUn0VCZ0gPwIoJfBJVpeBlyRMxu8Mm/huKWYd9p0= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 h1:sO4WKdPAudZGKPcpZT4MJn6JaDmpyLrMPDGGyA1SttE= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 h1:Q3C9yzW6I9jqEc8sawxzxZmY48fs9u220KXq6d5s3XU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= -go.opentelemetry.io/otel v0.20.0 h1:eaP0Fqu7SXHwvjiqDq83zImeehOHX8doTvU9AwXON8g= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg= go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= -go.opentelemetry.io/otel/metric v0.20.0 h1:4kzhXFP+btKm4jwxpjIqjs41A7MakRFUS86bqLHTIw8= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/oteltest v0.20.0 h1:HiITxCawalo5vQzdHfKeZurV8x7ljcqAgiWzF6Vaeaw= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0 h1:JsxtGXd06J8jrnya7fdI/U/MR6yXA5DtbZy+qoHQlr8= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0 h1:c5VRjxCXdQlx1HjzwGdQHzZaVI82b5EbBgOu2ljD92g= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= -go.opentelemetry.io/otel/sdk/metric v0.20.0 h1:7ao1wpzHRVKf0OQ7GIxiQJA6X7DLX9o14gmVon7mMK8= go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/trace v0.20.0 h1:1DL6EXUdcg95gukhuRRvLDO/4X5THh/5dIV52lqtnbw= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/proto/otlp v0.7.0 h1:rwOQPCuKAKmwGKq2aVNnYIibI6wnV7EvzgfTCzcdGg8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -1787,7 +1352,6 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+ go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -1836,8 +1400,9 @@ golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8= +golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1851,10 +1416,8 @@ golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 h1:QE6XYQK6naiK1EPAe1g/ILLxN5RBoH5xkJk3CqlMI/Y= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181023182221-1baf3a9d7d67/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1872,7 +1435,6 @@ golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhp golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20190806162312-597adff16ade h1:b373EGXtj0o+ssqkOkdVphTCZ/fVg2LwhctJn2QQbqA= golang.org/x/mobile v0.0.0-20190806162312-597adff16ade/go.mod h1:AlhUtkH4DA4asiFC5RgK7ZKmauvtkAVcy9L0epCzlWo= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= @@ -1942,8 +1504,9 @@ golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2060,6 +1623,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo= golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2185,10 +1749,8 @@ gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3m gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= -gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485 h1:OB/uP/Puiu5vS5QMRPrXCDWUPb+kt8f1KW8oQzFejQw= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e h1:jRyg0XfpwWlhEV8mDfdNGBeSJM2fuyh9Yjrnd8kF2Ts= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181021000519-a2651947f503/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= @@ -2323,11 +1885,8 @@ google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/l google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= -gopkg.in/airbrake/gobrake.v2 v2.0.9 h1:7z2uVWwn7oVeeugY1DtlPAy5H+KYgB1KeKTnqjNatLo= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -2336,18 +1895,13 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/cheggaaa/pb.v1 v1.0.25 h1:Ev7yu1/f6+d+b3pi5vPdRPc6nNtP1umSfcWiEfRqv6I= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= -gopkg.in/errgo.v2 v2.1.0 h1:0vLT13EuvQ0hNvakwLuFZ/jYrLp5F3kcWHXdRggjCE8= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 h1:OAj3g0cR6Dx/R07QgQe8wkA9RNjB2u4i700xBkIT4e0= gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= -gopkg.in/gorp.v1 v1.7.2 h1:j3DWlAyGVv8whO7AcIWznQ2Yj7yJkn34B8s63GViAAw= gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= @@ -2358,39 +1912,27 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.52.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/gokrb5.v7 v7.3.0 h1:0709Jtq/6QXEuWRfAm260XqlpcwL1vxtO1tUE2qK8Z4= gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/natefinch/lumberjack.v2 v2.0.0-20150622162204-20b71e5b60d7/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5 h1:E846t8CnR+lv5nE+VuiKTDG/v1U2stad0QzddfJC7kY= gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5/go.mod h1:hiOFpYm0ZJbusNj2ywpbrXowU3G8U6GIQzqn2mw1UIE= gopkg.in/square/go-jose.v2 v2.0.0-20180411045311-89060dee6a84/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg= gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= -gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 h1:ivZFOIltbce2Mo8IjzUHAFoq/IylO9WHhNOAJK+LsJg= gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= -gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE= gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU= gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= @@ -2412,10 +1954,8 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v0.0.0-20181223230014-1083505acf35/go.mod h1:R//lfYlUuTOTfblYI3lGoAAAebUdzjvbmQsuB7Ykd90= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -2426,14 +1966,11 @@ honnef.co/go/tools v0.0.1-2019.2.2/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.1.4 h1:SadWOkti5uVN1FAMgxn165+Mw00fuQKyk4Gyn/inxNQ= honnef.co/go/tools v0.1.4/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= k8c.io/kubermatic/v2 v2.16.2 h1:tjPfI+VV51pggXCvcDL/qG1r7KHDBQPSPYngPxpRtp8= k8c.io/kubermatic/v2 v2.16.2/go.mod h1:NdW+2mq4ynRtfZs9yPnvcnFWQpzmM7ngntW6GeuQicU= k8c.io/operating-system-manager v0.1.0/go.mod h1:ULyZQO1irKjsQTNjIdrHld7SZ+joHjmPnOEs5Db8G8M= k8c.io/operating-system-manager v0.3.0/go.mod h1:ME5GOCNUrHG+57igEKP1JCJKVHynaLfodT8bRiYH3MY= -k8c.io/operating-system-manager v0.3.6 h1:irFFYE/IJM2Qo+lH1zat2o3Yvgb8hUaypPWAc0qGHNM= -k8c.io/operating-system-manager v0.3.6/go.mod h1:aFyB/RH9DBAk0Kj5JVtCixhm9ugTeC8akgRGMW28lPg= k8c.io/operating-system-manager v0.3.9 h1:GcZgXqh90XYKdDXRMaMenA9AE30T8PZ47fM3yrwnygc= k8c.io/operating-system-manager v0.3.9/go.mod h1:aFyB/RH9DBAk0Kj5JVtCixhm9ugTeC8akgRGMW28lPg= k8s.io/api v0.0.0-20181018013834-843ad2d9b9ae/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= @@ -2513,16 +2050,12 @@ k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= k8s.io/apiserver v0.19.4/go.mod h1:X8WRHCR1UGZDd7HpV0QDc1h/6VbbpAeAGyxSh8yzZXw= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.2/go.mod h1:2nKd93WyMhZx4Hp3RfgH2K5PhwyTrprrkWYnI7id7jA= -k8s.io/apiserver v0.22.2 h1:TdIfZJc6YNhu2WxeAOWq1TvukHF0Sfx0+ln4XK9qnL4= k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= -k8s.io/autoscaler v0.0.0-20190218140445-7f77136aeea4 h1:My/qvGX4p7+3wWSGZO/QQ4mZq9ly5zoNsMUaec1b/30= k8s.io/autoscaler v0.0.0-20190218140445-7f77136aeea4/go.mod h1:QEXezc9uKPT91dwqhSJq3GNI3B1HxFRQHiku9kmrsSA= k8s.io/cli-runtime v0.19.0/go.mod h1:tun9l0eUklT8IHIM0jors17KmUjcrAxn0myoBYwuNuo= -k8s.io/cli-runtime v0.19.4 h1:FPpoqFbWsFzRbZNRI+o/+iiLFmWMYTmBueIj3OaNVTI= k8s.io/cli-runtime v0.19.4/go.mod h1:m8G32dVbKOeaX1foGhleLEvNd6REvU7YnZyWn5//9rw= k8s.io/client-go v0.22.2 h1:DaSQgs02aCC1QcwUdkKZWOeaVsQjYvWv8ZazcZ6JcHc= k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= -k8s.io/cloud-provider v0.17.0 h1:BQZPD1Ja/vnTOj1GKI9/wSpd3qgIDZp9q2NAS3568Ac= k8s.io/cloud-provider v0.17.0/go.mod h1:Ze4c3w2C0bRsjkBUoHpFi+qWe3ob1wI2/7cUn+YQIDE= k8s.io/code-generator v0.0.0-20181114232248-ae218e241252/go.mod h1:IPqxl/YHk05nodzupwjke6ctMjyNRdV2zZ5/j3/F204= k8s.io/code-generator v0.0.0-20190612205613-18da4a14b22b/go.mod h1:G8bQwmHm2eafm5bgtX67XDZQ8CWKSGu9DekI+yN4Y5I= @@ -2541,7 +2074,6 @@ k8s.io/code-generator v0.19.4/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZ k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.2/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.4/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= -k8s.io/code-generator v0.22.2 h1:+bUv9lpTnAWABtPkvO4x0kfz7j/kDEchVt0P/wXU3jQ= k8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA= k8s.io/component-base v0.0.0-20190918200425-ed2f0867c778/go.mod h1:DFWQCXgXVLiWtzFaS17KxHdlUeUymP7FLxZSkmL9/jU= @@ -2560,7 +2092,6 @@ k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeY k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= k8s.io/component-base v0.22.2 h1:vNIvE0AIrLhjX8drH0BgCNJcR4QZxMXcJzBsDplDx9M= k8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug= -k8s.io/csi-translation-lib v0.17.0 h1:8hwWJDMOBCAogaWXtNWy0dYGQ2dZYzOnOzjQMiDaY+E= k8s.io/csi-translation-lib v0.17.0/go.mod h1:HEF7MEz7pOLJCnxabi45IPkhSsE/KmxPQksuCrHKWls= k8s.io/gengo v0.0.0-20181106084056-51747d6e00da/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -2572,7 +2103,6 @@ k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027 h1:Uusb3oh8XcdzDF/ndlI4ToKTYVlkCSJP39SRY2mfRAw= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v0.0.0-20190306015804-8e90cee79f82/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= @@ -2588,25 +2118,20 @@ k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-aggregator v0.19.0/go.mod h1:1Ln45PQggFAG8xOqWPIYMxUq8WNtpPnYsbUJ39DpF/A= -k8s.io/kube-aggregator v0.19.4 h1:ME+z/JfCTj7IzSWzu7XVhjWHxpEGGQ3gp2FpeOS+lW0= k8s.io/kube-aggregator v0.19.4/go.mod h1:cTkvun110194d797AuThyydBBlgm+cKIFUeS2uzGJfU= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kubectl v0.19.0/go.mod h1:gPCjjsmE6unJzgaUNXIFGZGafiUp5jh0If3F/x7/rRg= -k8s.io/kubectl v0.19.4 h1:XFrHibf5fS4Ot8h3EnzdVsKrYj+pndlzKbwPkfra5hI= k8s.io/kubectl v0.19.4/go.mod h1:XPmlu4DJEYgD83pvZFeKF8+MSvGnYGqunbFSrJsqHv0= k8s.io/kubelet v0.19.4/go.mod h1:zJnPeb7nJCRvtAwxJhe9fFCtMLXL3cXbQiczPmpDrLU= k8s.io/kubelet v0.22.2 h1:7ol5AXXxcW97dUE8W/QiPjkXu1ZuGshG5VmgDmviZsc= k8s.io/kubelet v0.22.2/go.mod h1:ORIRua2/wTcx5UnEvxWosu650/8fatmzbMRC7m6WjAM= -k8s.io/legacy-cloud-providers v0.17.0 h1:ITm7sUthpxQyP96MU7K4Ra9M9M1k9eywUWv9IiTaxzc= k8s.io/legacy-cloud-providers v0.17.0/go.mod h1:DdzaepJ3RtRy+e5YhNtrCYwlgyK87j/5+Yfp0L9Syp8= k8s.io/metrics v0.19.0/go.mod h1:WykpW8B60OeAJx1imdwUgyOID2kDljr/Q+1zrPJ98Wo= -k8s.io/metrics v0.19.4 h1:adT/mgcMXbGvg/Zrj6pPO6js0rqcV7IttYFV//YWtQQ= k8s.io/metrics v0.19.4/go.mod h1:a0gvAzrxQPw2ouBqnXI7X9qlggpPkKAFgWU/Py+KZiU= k8s.io/test-infra v0.0.0-20181019233642-2e10a0bbe9b3/go.mod h1:2NzXB13Ji0nqpyublHeiPC4FZwU0TknfvyaaNfl/BTA= -k8s.io/test-infra v0.0.0-20200220102703-18fae0a00a2c h1:SWAghVxWCDXI56XuvtwQzAj3O4gGhbxIzzQ0td17lis= k8s.io/test-infra v0.0.0-20200220102703-18fae0a00a2c/go.mod h1:B9KsgNJiVixsZud99/ugFoQys8h9Tyv/A/eG5LMyrEE= k8s.io/utils v0.0.0-20181019225348-5e321f9a457c/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= @@ -2623,46 +2148,30 @@ k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -knative.dev/caching v0.0.0-20190719140829-2032732871ff h1:PrlDvOGvCASqW5Fs3ZGes0ma3P5Wr8nuzlTX+EnqfUg= knative.dev/caching v0.0.0-20190719140829-2032732871ff/go.mod h1:dHXFU6CGlLlbzaWc32g80cR92iuBSpsslDNBWI8C7eg= -knative.dev/eventing-contrib v0.6.1-0.20190723221543-5ce18048c08b h1:DTbaZGn06qcEoHuNyw3VmajapIUMuLSVjwFh6pNPews= knative.dev/eventing-contrib v0.6.1-0.20190723221543-5ce18048c08b/go.mod h1:SnXZgSGgMSMLNFTwTnpaOH7hXDzTFtw0J8OmHflNx3g= -knative.dev/pkg v0.0.0-20191111150521-6d806b998379 h1:0IbJWfv82eUhoNymvIrTjxVqrAURRK1x39+//IZV7Cc= knative.dev/pkg v0.0.0-20191111150521-6d806b998379/go.mod h1:pgODObA1dTyhNoFxPZTTjNWfx6F0aKsKzn+vaT9XO/Q= kubevirt.io/api v0.48.1 h1:C5i9h8ea7Xy3fJMoKEuzjRP74GnVMF7u2mQV8FGf2XE= kubevirt.io/api v0.48.1/go.mod h1:RoYMmFt76vWvFtw/FSiL0YUHZ2Ao6UfXlgpZAQnRswo= -kubevirt.io/client-go v0.30.0 h1:0jUvTa/Ev03lCN+Dr4mH22ipoJ9otAOkpFh6wA66b5M= kubevirt.io/client-go v0.30.0/go.mod h1:JY7hQq+SUT0aLvleXrW/+28fDfZ6BPe4E6f8FyC8jkY= -kubevirt.io/containerized-data-importer v1.10.6 h1:xkqLb48pkbdoY8gB2VDP2o+KXpO18tgQuLjcXNn0qAI= kubevirt.io/containerized-data-importer v1.10.6/go.mod h1:qF594BtRRkruyrqLwt3zbLCWdPIQNs1qWh4LR1cOzy0= kubevirt.io/containerized-data-importer-api v1.41.0 h1:VdEwYP36N+4asMnTBSadVH4SF7OVPvvraEQMtOd7Vlk= kubevirt.io/containerized-data-importer-api v1.41.0/go.mod h1:0xadDFtaMd8iy+/oD2+dYoPxACZ/YizKqay5QIrQ6cw= kubevirt.io/controller-lifecycle-operator-sdk v0.2.1 h1:I1b14fnhwrVvQLmgksMo9vgje42hmH4QN5kqyYDqbMA= kubevirt.io/controller-lifecycle-operator-sdk v0.2.1/go.mod h1:ZJhLceiY2Gl5CXFGSp5eMGt/sksOiJP0289nAZFCQf0= -modernc.org/cc v1.0.0 h1:nPibNuDEx6tvYrUAtvDTTw98rx5juGsa5zuDnKwEEQQ= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= -modernc.org/golex v1.0.0 h1:wWpDlbK8ejRfSyi0frMyhilD3JBvtcx2AdGDnU+JtsE= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= -modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= -modernc.org/strutil v1.0.0 h1:XVFtQwFVwc02Wk+0L/Z/zDDXO81r5Lhe6iMKmGX3KhE= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -modernc.org/xc v1.0.0 h1:7ccXrupWZIS3twbUGrtKmHS2DXY6xegFua+6O3xgAFU= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= -mvdan.cc/xurls/v2 v2.0.0 h1:r1zSOSNS/kqtpmATyMMMvaZ4/djsesbYz5kr0+qMRWc= mvdan.cc/xurls/v2 v2.0.0/go.mod h1:2/webFPYOXN9jp/lzuj0zuAVlF+9g4KPFJANH1oJhRU= -pack.ag/amqp v0.11.0 h1:ot/IA0enDkt4/c8xfbCO7AZzjM4bHys/UffnFmnHUnU= pack.ag/amqp v0.11.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= -rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0 h1:9JKUTTIUgS6kzR9mK1YuGKv6Nl+DijDNIc0ghT58FaY= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22 h1:fmRfl9WJ4ApJn7LxNuED4m0t18qivVQOxP6aAYG9J6c= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-runtime v0.3.0/go.mod h1:Cw6PkEg0Sa7dAYovGT4R0tRkGhHXpYijwNxYhAnAZZk= sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns= @@ -2676,16 +2185,13 @@ sigs.k8s.io/controller-runtime v0.10.2/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkG sigs.k8s.io/controller-tools v0.2.4/go.mod h1:m/ztfQNocGYBgTTCmFdnK94uVvgxeZeE3LtJvd/jIzA= sigs.k8s.io/controller-tools v0.2.9-0.20200414181213-645d44dca7c0/go.mod h1:YKE/iHvcKITCljdnlqHYe+kAt7ZldvtAwUzQff0k1T0= sigs.k8s.io/controller-tools v0.4.1/go.mod h1:G9rHdZMVlBDocIxGkK3jHLWqcTMNvveypYJwrvYKjWU= -sigs.k8s.io/controller-tools v0.5.0 h1:3u2RCwOlp0cjCALAigpOcbAf50pE+kHSdueUosrC/AE= sigs.k8s.io/controller-tools v0.5.0/go.mod h1:JTsstrMpxs+9BUj6eGuAaEb6SDSPTeVtUyp0jmnAM/I= -sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= sigs.k8s.io/structured-merge-diff v1.0.1 h1:LOs1LZWMsz1xs77Phr/pkB4LFaavH7IVq/3+WTN9XTA= sigs.k8s.io/structured-merge-diff v1.0.1/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= @@ -2694,16 +2200,12 @@ sigs.k8s.io/structured-merge-diff/v4 v4.1.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= -sigs.k8s.io/testing_frameworks v0.1.2 h1:vK0+tvjF0BZ/RYFeZ1E6BYBwHJJXhjuZ3TdsEKH+UQM= sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= software.sslmate.com/src/go-pkcs12 v0.0.0-20180114231543-2291e8f0f237/go.mod h1:/xvNRWUqm0+/ZMiF4EX00vrSCMsE4/NHb+Pt3freEeQ= -software.sslmate.com/src/go-pkcs12 v0.0.0-20200830195227-52f69702a001 h1:AVd6O+azYjVQYW1l55IqkbL8/JxjrLtO6q4FCmV8N5c= software.sslmate.com/src/go-pkcs12 v0.0.0-20200830195227-52f69702a001/go.mod h1:/xvNRWUqm0+/ZMiF4EX00vrSCMsE4/NHb+Pt3freEeQ= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 h1:ucqkfpjg9WzSUubAO62csmucvxl4/JeW3F4I4909XkM= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= -vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787 h1:O69FD9pJA4WUZlEwYatBEEkRWKQ5cKodWpdKTrCS/iQ= vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 972e57b39..0940b6d95 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -28,6 +28,7 @@ import ( goopenstack "github.com/gophercloud/gophercloud/openstack" "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/bootfromvolume" osextendedstatus "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/extendedstatus" + "github.com/gophercloud/gophercloud/openstack/compute/v2/extensions/schedulerhints" osservers "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" osfloatingips "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" osnetworks "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" @@ -43,7 +44,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -103,6 +104,7 @@ type Config struct { RootDiskSizeGB *int RootDiskVolumeType string NodeVolumeAttachLimit *uint + ServerGroup string InstanceReadyCheckPeriod time.Duration InstanceReadyCheckTimeout time.Duration @@ -173,15 +175,17 @@ func (p *provider) getConfigAuth(c *Config, rawConfig *openstacktypes.RawConfig) return nil } -func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *openstacktypes.RawConfig, error) { - if s.Value == nil { +func (p *provider) getConfig(spec v1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *openstacktypes.RawConfig, error) { + if spec.Value == nil { return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } + pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pconfig) + err := json.Unmarshal(spec.Value.Raw, &pconfig) if err != nil { return nil, nil, nil, err } + var rawConfig openstacktypes.RawConfig err = json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawConfig) if err != nil { @@ -192,89 +196,107 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return nil, nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } - c := Config{} - c.IdentityEndpoint, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.IdentityEndpoint, "OS_AUTH_URL") + cfg := Config{} + cfg.IdentityEndpoint, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.IdentityEndpoint, "OS_AUTH_URL") if err != nil { return nil, nil, nil, fmt.Errorf("failed to get the value of \"identityEndpoint\" field, error = %v", err) } + // Retrieve authentication config, username/password or application credentials - err = p.getConfigAuth(&c, &rawConfig) + err = p.getConfigAuth(&cfg, &rawConfig) if err != nil { return nil, nil, nil, fmt.Errorf("failed to retrieve authentication credentials, error = %v", err) } + // Ignore Region not found as Region might not be found and we can default it later - c.Region, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Region, "OS_REGION_NAME") + cfg.Region, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Region, "OS_REGION_NAME") if err != nil { klog.V(6).Infof("Region from configuration or environment variable not found") } - c.InstanceReadyCheckPeriod, err = p.configVarResolver.GetConfigVarDurationValueOrDefault(rawConfig.InstanceReadyCheckPeriod, 5*time.Second) + cfg.InstanceReadyCheckPeriod, err = p.configVarResolver.GetConfigVarDurationValueOrDefault(rawConfig.InstanceReadyCheckPeriod, 5*time.Second) if err != nil { return nil, nil, nil, fmt.Errorf(`failed to get the value of "InstanceReadyCheckPeriod" field, error = %v`, err) } - c.InstanceReadyCheckTimeout, err = p.configVarResolver.GetConfigVarDurationValueOrDefault(rawConfig.InstanceReadyCheckTimeout, 10*time.Second) + cfg.InstanceReadyCheckTimeout, err = p.configVarResolver.GetConfigVarDurationValueOrDefault(rawConfig.InstanceReadyCheckTimeout, 10*time.Second) if err != nil { return nil, nil, nil, fmt.Errorf(`failed to get the value of "InstanceReadyCheckTimeout" field, error = %v`, err) } // We ignore errors here because the OS domain is only required when using Identity API V3 - c.DomainName, _ = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.DomainName, "OS_DOMAIN_NAME") - c.TokenID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.TokenID) + cfg.DomainName, _ = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.DomainName, "OS_DOMAIN_NAME") + cfg.TokenID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.TokenID) if err != nil { return nil, nil, nil, err } - c.Image, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Image) + + cfg.Image, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Image) if err != nil { return nil, nil, nil, err } - c.Flavor, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Flavor) + + cfg.Flavor, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Flavor) if err != nil { return nil, nil, nil, err } + for _, securityGroup := range rawConfig.SecurityGroups { securityGroupValue, err := p.configVarResolver.GetConfigVarStringValue(securityGroup) if err != nil { return nil, nil, nil, err } - c.SecurityGroups = append(c.SecurityGroups, securityGroupValue) + cfg.SecurityGroups = append(cfg.SecurityGroups, securityGroupValue) } - c.Network, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Network) + + cfg.Network, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Network) if err != nil { return nil, nil, nil, err } - c.Subnet, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Subnet) + + cfg.Subnet, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Subnet) if err != nil { return nil, nil, nil, err } - c.FloatingIPPool, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.FloatingIPPool) + + cfg.FloatingIPPool, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.FloatingIPPool) if err != nil { return nil, nil, nil, err } - c.AvailabilityZone, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.AvailabilityZone) + + cfg.AvailabilityZone, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.AvailabilityZone) if err != nil { return nil, nil, nil, err } - c.TrustDevicePath, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.TrustDevicePath) + + cfg.TrustDevicePath, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.TrustDevicePath) if err != nil { return nil, nil, nil, err } - c.ComputeAPIVersion, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ComputeAPIVersion) + + cfg.ComputeAPIVersion, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ComputeAPIVersion) if err != nil { return nil, nil, nil, err } - c.RootDiskSizeGB = rawConfig.RootDiskSizeGB - c.RootDiskVolumeType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.RootDiskVolumeType) + + cfg.RootDiskSizeGB = rawConfig.RootDiskSizeGB + cfg.RootDiskVolumeType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.RootDiskVolumeType) if err != nil { return nil, nil, nil, err } - c.NodeVolumeAttachLimit = rawConfig.NodeVolumeAttachLimit - c.Tags = rawConfig.Tags - if c.Tags == nil { - c.Tags = map[string]string{} + + cfg.NodeVolumeAttachLimit = rawConfig.NodeVolumeAttachLimit + cfg.Tags = rawConfig.Tags + if cfg.Tags == nil { + cfg.Tags = map[string]string{} } - return &c, &pconfig, &rawConfig, err + cfg.ServerGroup, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ServerGroup) + if err != nil { + return nil, nil, nil, err + } + + return &cfg, &pconfig, &rawConfig, err } func setProviderSpec(rawConfig openstacktypes.RawConfig, s v1alpha1.ProviderSpec) (*runtime.RawExtension, error) { @@ -533,7 +555,7 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { } func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) + cfg, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ Reason: common.InvalidConfigurationMachineError, @@ -541,46 +563,46 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr } } - client, err := p.clientGetter(c) + client, err := p.clientGetter(cfg) if err != nil { return nil, osErrorToTerminalError(err, "failed to get a openstack client") } - computeClient, err := getNewComputeV2(client, c) + computeClient, err := getNewComputeV2(client, cfg) if err != nil { return nil, osErrorToTerminalError(err, "failed to get a openstack client") } - flavor, err := getFlavor(computeClient, c) + flavor, err := getFlavor(computeClient, cfg) if err != nil { - return nil, osErrorToTerminalError(err, fmt.Sprintf("failed to get flavor %s", c.Flavor)) + return nil, osErrorToTerminalError(err, fmt.Sprintf("failed to get flavor %s", cfg.Flavor)) } // Get OS Image Client - imageClient, err := goopenstack.NewImageServiceV2(client, gophercloud.EndpointOpts{Region: c.Region}) + imageClient, err := goopenstack.NewImageServiceV2(client, gophercloud.EndpointOpts{Region: cfg.Region}) if err != nil { return nil, osErrorToTerminalError(err, "failed to get a image client") } - image, err := getImageByName(imageClient, c) + image, err := getImageByName(imageClient, cfg) if err != nil { - return nil, osErrorToTerminalError(err, fmt.Sprintf("failed to get image %s", c.Image)) + return nil, osErrorToTerminalError(err, fmt.Sprintf("failed to get image %s", cfg.Image)) } - netClient, err := goopenstack.NewNetworkV2(client, gophercloud.EndpointOpts{Region: c.Region}) + netClient, err := goopenstack.NewNetworkV2(client, gophercloud.EndpointOpts{Region: cfg.Region}) if err != nil { return nil, err } - network, err := getNetwork(netClient, c.Network) + network, err := getNetwork(netClient, cfg.Network) if err != nil { - return nil, osErrorToTerminalError(err, fmt.Sprintf("failed to get network %s", c.Network)) + return nil, osErrorToTerminalError(err, fmt.Sprintf("failed to get network %s", cfg.Network)) } - securityGroups := c.SecurityGroups + securityGroups := cfg.SecurityGroups if len(securityGroups) == 0 { klog.V(2).Infof("creating security group %s for worker nodes", securityGroupName) - err = ensureKubernetesSecurityGroupExist(client, c.Region, securityGroupName) + err = ensureKubernetesSecurityGroupExist(client, cfg.Region, securityGroupName) if err != nil { return nil, fmt.Errorf("Error occurred creating security groups: %v", err) } @@ -588,7 +610,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr } // we check against reserved tags in Validation method - allTags := c.Tags + allTags := cfg.Tags allTags[machineUIDMetaKey] = string(machine.UID) serverOpts := osservers.CreateOpts{ @@ -596,13 +618,24 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr FlavorRef: flavor.ID, UserData: []byte(userdata), SecurityGroups: securityGroups, - AvailabilityZone: c.AvailabilityZone, + AvailabilityZone: cfg.AvailabilityZone, Networks: []osservers.Network{{UUID: network.ID}}, Metadata: allTags, } + var createOpts osservers.CreateOptsBuilder = &serverOpts + + if cfg.ServerGroup != "" { + createOpts = schedulerhints.CreateOptsExt{ + CreateOptsBuilder: createOpts, + SchedulerHints: schedulerhints.SchedulerHints{ + Group: cfg.ServerGroup, + }, + } + } + var server serverWithExt - if c.RootDiskSizeGB != nil { + if cfg.RootDiskSizeGB != nil { blockDevices := []bootfromvolume.BlockDevice{ { BootIndex: 0, @@ -610,14 +643,16 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr DestinationType: bootfromvolume.DestinationVolume, SourceType: bootfromvolume.SourceImage, UUID: image.ID, - VolumeSize: *c.RootDiskSizeGB, - VolumeType: c.RootDiskVolumeType, + VolumeSize: *cfg.RootDiskSizeGB, + VolumeType: cfg.RootDiskVolumeType, }, } - createOpts := bootfromvolume.CreateOptsExt{ - CreateOptsBuilder: serverOpts, + + createOpts = bootfromvolume.CreateOptsExt{ + CreateOptsBuilder: createOpts, BlockDevice: blockDevices, } + if err := bootfromvolume.Create(computeClient, createOpts).ExtractInto(&server); err != nil { return nil, osErrorToTerminalError(err, "failed to create server with volume") } @@ -626,18 +661,19 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr // mapping is not used. Otherwish an error may occur with some // OpenStack providers/versions .e.g. OpenTelekom Cloud serverOpts.ImageRef = image.ID - if err := osservers.Create(computeClient, serverOpts).ExtractInto(&server); err != nil { + + if err := osservers.Create(computeClient, createOpts).ExtractInto(&server); err != nil { return nil, osErrorToTerminalError(err, "failed to create server") } } - if c.FloatingIPPool != "" { - if err := p.portReadinessWaiter(netClient, server.ID, network.ID, c.InstanceReadyCheckPeriod, c.InstanceReadyCheckTimeout); err != nil { + if cfg.FloatingIPPool != "" { + if err := p.portReadinessWaiter(netClient, server.ID, network.ID, cfg.InstanceReadyCheckPeriod, cfg.InstanceReadyCheckTimeout); err != nil { klog.V(2).Infof("port for instance %q did not became active due to: %v", server.ID, err) } // Find a free FloatingIP or allocate a new one - if err := assignFloatingIPToInstance(data.Update, machine, netClient, server.ID, c.FloatingIPPool, c.Region, network); err != nil { + if err := assignFloatingIPToInstance(data.Update, machine, netClient, server.ID, cfg.FloatingIPPool, cfg.Region, network); err != nil { defer deleteInstanceDueToFatalLogged(computeClient, server.ID) return nil, fmt.Errorf("failed to assign a floating ip to instance %s: %w", server.ID, err) } @@ -895,8 +931,8 @@ func (d *osInstance) ID() string { return d.server.ID } -func (d *osInstance) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{} +func (d *osInstance) Addresses() map[string]corev1.NodeAddressType { + addresses := map[string]corev1.NodeAddressType{} for _, networkAddresses := range d.server.Addresses { for _, element := range networkAddresses.([]interface{}) { address := element.(map[string]interface{}) diff --git a/pkg/cloudprovider/provider/openstack/types/types.go b/pkg/cloudprovider/provider/openstack/types/types.go index a345dd014..a4ebb27f9 100644 --- a/pkg/cloudprovider/provider/openstack/types/types.go +++ b/pkg/cloudprovider/provider/openstack/types/types.go @@ -50,6 +50,7 @@ type RawConfig struct { RootDiskSizeGB *int `json:"rootDiskSizeGB"` RootDiskVolumeType providerconfigtypes.ConfigVarString `json:"rootDiskVolumeType,omitempty"` NodeVolumeAttachLimit *uint `json:"nodeVolumeAttachLimit"` + ServerGroup providerconfigtypes.ConfigVarString `json:"serverGroup"` // This tag is related to server metadata, not compute server's tag Tags map[string]string `json:"tags,omitempty"` } From 18dddc8089955be53a6d77ef9aa488fac9481946 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Fri, 21 Jan 2022 15:05:50 +0100 Subject: [PATCH 068/489] disable vSphere tests (#1172) Signed-off-by: Moath Qasim --- .prow.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.prow.yaml b/.prow.yaml index 450502805..3c2bf5822 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -543,7 +543,7 @@ presubmits: cpu: 500m - name: pull-machine-controller-e2e-vsphere - always_run: true + always_run: false decorate: true error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" @@ -788,7 +788,7 @@ presubmits: cpu: 500m - name: pull-machine-controller-e2e-vsphere-resource-pool - always_run: true + always_run: false decorate: true error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" From cb15dbc6daeecad1a641ebe98699f0728d8ab245 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Fri, 21 Jan 2022 21:58:49 +0100 Subject: [PATCH 069/489] update machine controller openstack rhel image (#1171) Signed-off-by: Moath Qasim --- test/e2e/provisioning/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 89b8fdc03..44ee994d8 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -51,7 +51,7 @@ var ( openStackImages = map[string]string{ string(providerconfigtypes.OperatingSystemUbuntu): "machine-controller-e2e-ubuntu-20-04", string(providerconfigtypes.OperatingSystemCentOS): "machine-controller-e2e-centos", - string(providerconfigtypes.OperatingSystemRHEL): "machine-controller-e2e-rhel", + string(providerconfigtypes.OperatingSystemRHEL): "machine-controller-e2e-rhel-8-5", string(providerconfigtypes.OperatingSystemFlatcar): "machine-controller-e2e-flatcar-stable-2983", } ) From ed424d935c061993ff70bbb0c3298653e7e68b1a Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 25 Jan 2022 20:09:56 +0500 Subject: [PATCH 070/489] Improve generation for container runtime configuration (#1174) * Improve handling for container runtime configuration Signed-off-by: Waleed Malik * Update fixtures Signed-off-by: Waleed Malik * Fix linting errors * Handle PR feedback Signed-off-by: Waleed Malik --- cmd/machine-controller/main.go | 50 ++------- .../cluster/v1alpha1/zz_generated.deepcopy.go | 1 + pkg/containerruntime/config.go | 105 ++++++++++++++++++ pkg/containerruntime/containerruntime.go | 14 +-- .../containerruntime/flags.go | 10 +- pkg/controller/machine/machine_controller.go | 20 +--- .../v1alpha1/zz_generated.deepcopy.go | 1 + pkg/userdata/amzn2/provider_test.go | 24 ++-- pkg/userdata/centos/provider_test.go | 24 ++-- pkg/userdata/flatcar/provider_test.go | 20 ++-- pkg/userdata/rhel/provider_test.go | 24 ++-- pkg/userdata/sles/provider_test.go | 24 ++-- pkg/userdata/ubuntu/provider_test.go | 58 +++++----- 13 files changed, 237 insertions(+), 138 deletions(-) create mode 100644 pkg/containerruntime/config.go rename cmd/machine-controller/custom_flags.go => pkg/containerruntime/flags.go (84%) diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index e111dc3ce..ebb821e5b 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -23,7 +23,6 @@ import ( "net" "net/http" "net/http/pprof" - "net/url" "strings" "time" @@ -84,7 +83,7 @@ var ( podCidr string nodePortRange string nodeRegistryCredentialsSecret string - nodeContainerdRegistryMirrors = registryMirrorsFlags{} + nodeContainerdRegistryMirrors = containerruntime.RegistryMirrorsFlags{} ) const ( @@ -237,37 +236,17 @@ func main() { ctrlMetrics := machinecontroller.NewMachineControllerMetrics() ctrlMetrics.MustRegister(metrics.Registry) - var insecureRegistries []string - for _, registry := range strings.Split(nodeInsecureRegistries, ",") { - if trimmedRegistry := strings.TrimSpace(registry); trimmedRegistry != "" { - insecureRegistries = append(insecureRegistries, trimmedRegistry) - } - } - - var registryMirrors []string - for _, mirror := range strings.Split(nodeRegistryMirrors, ",") { - if trimmedMirror := strings.TrimSpace(mirror); trimmedMirror != "" { - if !strings.HasPrefix(mirror, "http") { - trimmedMirror = "https://" + mirror - } - - _, err := url.Parse(trimmedMirror) - if err != nil { - klog.Fatalf("incorrect mirror provided: %v", err) - } - - registryMirrors = append(registryMirrors, trimmedMirror) - } + containerRuntimeOpts := containerruntime.Opts{ + ContainerRuntime: nodeContainerRuntime, + ContainerdRegistryMirrors: nodeContainerdRegistryMirrors, + InsecureRegistries: nodeInsecureRegistries, + PauseImage: nodePauseImage, + RegistryMirrors: nodeRegistryMirrors, + RegistryCredentialsSecret: nodeRegistryCredentialsSecret, } - - if len(registryMirrors) > 0 { - nodeContainerdRegistryMirrors["docker.io"] = registryMirrors - } - - if nodeRegistryCredentialsSecret != "" { - if secRef := strings.Split(nodeRegistryCredentialsSecret, "/"); len(secRef) != 2 { - klog.Fatalf("-node-registry-credentials-secret is in incorrect format %q, should be in 'namespace/secretname'", nodeRegistryCredentialsSecret) - } + containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) + if err != nil { + klog.Fatalf("failed to generate container runtime config: %v", err) } runOptions := controllerRunOptions{ @@ -285,12 +264,7 @@ func main() { NoProxy: nodeNoProxy, PauseImage: nodePauseImage, RegistryCredentialsSecretRef: nodeRegistryCredentialsSecret, - ContainerRuntime: containerruntime.Get( - nodeContainerRuntime, - containerruntime.WithInsecureRegistries(insecureRegistries), - containerruntime.WithRegistryMirrors(nodeContainerdRegistryMirrors), - containerruntime.WithSandboxImage(nodePauseImage), - ), + ContainerRuntime: containerRuntimeConfig, }, useOSM: useOSM, podCidr: podCidr, diff --git a/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go index 630c681a9..e213274a1 100644 --- a/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/pkg/containerruntime/config.go b/pkg/containerruntime/config.go new file mode 100644 index 000000000..13149b705 --- /dev/null +++ b/pkg/containerruntime/config.go @@ -0,0 +1,105 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package containerruntime + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Opts struct { + ContainerRuntime string + InsecureRegistries string + RegistryMirrors string + RegistryCredentialsSecret string + PauseImage string + ContainerdRegistryMirrors RegistryMirrorsFlags +} + +func BuildConfig(opts Opts) (Config, error) { + var insecureRegistries []string + for _, registry := range strings.Split(opts.InsecureRegistries, ",") { + if trimmedRegistry := strings.TrimSpace(registry); trimmedRegistry != "" { + insecureRegistries = append(insecureRegistries, trimmedRegistry) + } + } + + var registryMirrors []string + for _, mirror := range strings.Split(opts.RegistryMirrors, ",") { + if trimmedMirror := strings.TrimSpace(mirror); trimmedMirror != "" { + if !strings.HasPrefix(mirror, "http") { + trimmedMirror = "https://" + mirror + } + + _, err := url.Parse(trimmedMirror) + if err != nil { + return Config{}, fmt.Errorf("incorrect mirror provided: %v", err) + } + + registryMirrors = append(registryMirrors, trimmedMirror) + } + } + + if len(registryMirrors) > 0 { + if opts.ContainerdRegistryMirrors == nil { + opts.ContainerdRegistryMirrors = make(RegistryMirrorsFlags) + } + opts.ContainerdRegistryMirrors["docker.io"] = registryMirrors + } + + // Only validate registry credential here + if opts.RegistryCredentialsSecret != "" { + if secRef := strings.Split(opts.RegistryCredentialsSecret, "/"); len(secRef) != 2 { + return Config{}, fmt.Errorf("-node-registry-credentials-secret is in incorrect format %q, should be in 'namespace/secretname'", opts.RegistryCredentialsSecret) + } + } + + return get( + opts.ContainerRuntime, + withInsecureRegistries(insecureRegistries), + withRegistryMirrors(opts.ContainerdRegistryMirrors), + withSandboxImage(opts.PauseImage), + ), nil +} + +func GetContainerdAuthConfig(ctx context.Context, client ctrlruntimeclient.Client, registryCredentialsSecret string) (map[string]AuthConfig, error) { + registryCredentials := map[string]AuthConfig{} + + if secRef := strings.SplitN(registryCredentialsSecret, "/", 2); len(secRef) == 2 { + var credsSecret corev1.Secret + err := client.Get(ctx, types.NamespacedName{Namespace: secRef[0], Name: secRef[1]}, &credsSecret) + if err != nil { + return nil, fmt.Errorf("failed to retrieve registry credentials secret object: %w", err) + } + + for registry, data := range credsSecret.Data { + var regCred AuthConfig + if err := json.Unmarshal(data, ®Cred); err != nil { + return nil, fmt.Errorf("failed to unmarshal registry credentials: %w", err) + } + registryCredentials[registry] = regCred + } + } + return registryCredentials, nil +} diff --git a/pkg/containerruntime/containerruntime.go b/pkg/containerruntime/containerruntime.go index 05edcfee3..c4e2852c7 100644 --- a/pkg/containerruntime/containerruntime.go +++ b/pkg/containerruntime/containerruntime.go @@ -37,31 +37,25 @@ type Engine interface { type Opt func(*Config) -func WithInsecureRegistries(registries []string) Opt { +func withInsecureRegistries(registries []string) Opt { return func(cfg *Config) { cfg.InsecureRegistries = registries } } -func WithRegistryMirrors(mirrors map[string][]string) Opt { +func withRegistryMirrors(mirrors map[string][]string) Opt { return func(cfg *Config) { cfg.RegistryMirrors = mirrors } } -func WithRegistryCredentials(auth map[string]AuthConfig) Opt { - return func(cfg *Config) { - cfg.RegistryCredentials = auth - } -} - -func WithSandboxImage(image string) Opt { +func withSandboxImage(image string) Opt { return func(cfg *Config) { cfg.SandboxImage = image } } -func Get(containerRuntimeName string, opts ...Opt) Config { +func get(containerRuntimeName string, opts ...Opt) Config { cfg := Config{} switch containerRuntimeName { diff --git a/cmd/machine-controller/custom_flags.go b/pkg/containerruntime/flags.go similarity index 84% rename from cmd/machine-controller/custom_flags.go rename to pkg/containerruntime/flags.go index 7198766f5..eb734fef0 100644 --- a/cmd/machine-controller/custom_flags.go +++ b/pkg/containerruntime/flags.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Machine Controller Authors. +Copyright 2022 The Machine Controller Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package main +package containerruntime import ( "fmt" @@ -22,9 +22,9 @@ import ( "strings" ) -type registryMirrorsFlags map[string][]string +type RegistryMirrorsFlags map[string][]string -func (fl registryMirrorsFlags) Set(val string) error { +func (fl RegistryMirrorsFlags) Set(val string) error { split := strings.SplitN(val, "=", 2) if len(split) != 2 { return fmt.Errorf("should have exactly 1 =") @@ -38,7 +38,7 @@ func (fl registryMirrorsFlags) Set(val string) error { return nil } -func (fl registryMirrorsFlags) String() string { +func (fl RegistryMirrorsFlags) String() string { var ( registryNames []string result []string diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 0f2469951..7fee0554f 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -18,7 +18,6 @@ package controller import ( "context" - "encoding/json" "errors" "fmt" "net" @@ -726,22 +725,9 @@ func (r *Reconciler) ensureInstanceExistsForMachine( externalCloudProvider, _ = strconv.ParseBool(val) } - registryCredentials := map[string]containerruntime.AuthConfig{} - - if secRef := strings.SplitN(r.nodeSettings.RegistryCredentialsSecretRef, "/", 2); len(secRef) == 2 { - var credsSecret corev1.Secret - err := r.client.Get(ctx, types.NamespacedName{Namespace: secRef[0], Name: secRef[1]}, &credsSecret) - if err != nil { - return nil, fmt.Errorf("failed to retrieve registry credentials secret object: %w", err) - } - - for registry, data := range credsSecret.Data { - var regCred containerruntime.AuthConfig - if err := json.Unmarshal(data, ®Cred); err != nil { - return nil, fmt.Errorf("failed to unmarshal registry credentials: %w", err) - } - registryCredentials[registry] = regCred - } + registryCredentials, err := containerruntime.GetContainerdAuthConfig(ctx, r.client, r.nodeSettings.RegistryCredentialsSecretRef) + if err != nil { + return nil, fmt.Errorf("failed to get containerd auth config: %v", err) } crRuntime := r.nodeSettings.ContainerRuntime diff --git a/pkg/machines/v1alpha1/zz_generated.deepcopy.go b/pkg/machines/v1alpha1/zz_generated.deepcopy.go index a11584de1..2510c81ce 100644 --- a/pkg/machines/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/machines/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index c235b8fba..03b5b172d 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -87,8 +87,8 @@ type userDataTestCase struct { externalCloudProvider bool httpProxy string noProxy string - insecureRegistries []string - registryMirrors map[string][]string + insecureRegistries string + registryMirrors string pauseImage string containerruntime string } @@ -158,7 +158,7 @@ func TestUserDataGeneration(t *testing.T) { cloudProviderName: stringPtr("vsphere"), httpProxy: "http://192.168.100.100:3128/", noProxy: "192.168.1.0", - insecureRegistries: []string{"192.168.100.100:5000", "10.0.0.1:5000"}, + insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { @@ -172,7 +172,7 @@ func TestUserDataGeneration(t *testing.T) { cloudProviderName: stringPtr("vsphere"), httpProxy: "http://192.168.100.100:3128/", noProxy: "192.168.1.0", - registryMirrors: map[string][]string{"docker.io": {"https://registry.docker-cn.com/"}}, + registryMirrors: "https://registry.docker-cn.com/", pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { @@ -240,6 +240,16 @@ func TestUserDataGeneration(t *testing.T) { t.Fatalf("failed to get cloud config: %v", err) } + containerRuntimeOpts := containerruntime.Opts{ + ContainerRuntime: test.containerruntime, + InsecureRegistries: test.insecureRegistries, + RegistryMirrors: test.registryMirrors, + } + containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) + if err != nil { + t.Fatalf("failed to generate container runtime config: %v", err) + } + req := plugin.UserDataRequest{ MachineSpec: test.spec, Kubeconfig: kubeconfig, @@ -252,11 +262,7 @@ func TestUserDataGeneration(t *testing.T) { NoProxy: test.noProxy, PauseImage: test.pauseImage, KubeletFeatureGates: kubeletFeatureGates, - ContainerRuntime: containerruntime.Get( - test.containerruntime, - containerruntime.WithInsecureRegistries(test.insecureRegistries), - containerruntime.WithRegistryMirrors(test.registryMirrors), - ), + ContainerRuntime: containerRuntimeConfig, } s, err := provider.UserData(req) diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index 86f30751d..5d37023c4 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -87,8 +87,8 @@ type userDataTestCase struct { externalCloudProvider bool httpProxy string noProxy string - insecureRegistries []string - registryMirrors map[string][]string + insecureRegistries string + registryMirrors string pauseImage string containerruntime string } @@ -158,7 +158,7 @@ func TestUserDataGeneration(t *testing.T) { cloudProviderName: stringPtr("vsphere"), httpProxy: "http://192.168.100.100:3128/", noProxy: "192.168.1.0", - insecureRegistries: []string{"192.168.100.100:5000", "10.0.0.1:5000"}, + insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { @@ -172,7 +172,7 @@ func TestUserDataGeneration(t *testing.T) { cloudProviderName: stringPtr("vsphere"), httpProxy: "http://192.168.100.100:3128/", noProxy: "192.168.1.0", - registryMirrors: map[string][]string{"docker.io": {"https://registry.docker-cn.com/"}}, + registryMirrors: "https://registry.docker-cn.com/", pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { @@ -250,6 +250,16 @@ func TestUserDataGeneration(t *testing.T) { t.Fatalf("failed to get cloud config: %v", err) } + containerRuntimeOpts := containerruntime.Opts{ + ContainerRuntime: test.containerruntime, + InsecureRegistries: test.insecureRegistries, + RegistryMirrors: test.registryMirrors, + } + containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) + if err != nil { + t.Fatalf("failed to generate container runtime config: %v", err) + } + req := plugin.UserDataRequest{ MachineSpec: test.spec, Kubeconfig: kubeconfig, @@ -262,11 +272,7 @@ func TestUserDataGeneration(t *testing.T) { NoProxy: test.noProxy, PauseImage: test.pauseImage, KubeletFeatureGates: kubeletFeatureGates, - ContainerRuntime: containerruntime.Get( - test.containerruntime, - containerruntime.WithInsecureRegistries(test.insecureRegistries), - containerruntime.WithRegistryMirrors(test.registryMirrors), - ), + ContainerRuntime: containerRuntimeConfig, } s, err := provider.UserData(req) diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index 530e891af..3836bc6cf 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -109,8 +109,8 @@ type userDataTestCase struct { externalCloudProvider bool httpProxy string noProxy string - insecureRegistries []string - registryMirrors map[string][]string + insecureRegistries string + registryMirrors string pauseImage string containerruntime string } @@ -420,6 +420,16 @@ func TestUserDataGeneration(t *testing.T) { t.Fatalf("failed to get cloud config: %v", err) } + containerRuntimeOpts := containerruntime.Opts{ + ContainerRuntime: test.containerruntime, + InsecureRegistries: test.insecureRegistries, + RegistryMirrors: test.registryMirrors, + } + containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) + if err != nil { + t.Fatalf("failed to generate container runtime config: %v", err) + } + req := plugin.UserDataRequest{ MachineSpec: test.spec, Kubeconfig: kubeconfig, @@ -432,11 +442,7 @@ func TestUserDataGeneration(t *testing.T) { NoProxy: test.noProxy, PauseImage: test.pauseImage, KubeletFeatureGates: kubeletFeatureGates, - ContainerRuntime: containerruntime.Get( - test.containerruntime, - containerruntime.WithInsecureRegistries(test.insecureRegistries), - containerruntime.WithRegistryMirrors(test.registryMirrors), - ), + ContainerRuntime: containerRuntimeConfig, } s, err := provider.UserData(req) diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index 08d37b427..3c0ac70f1 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -87,8 +87,8 @@ type userDataTestCase struct { externalCloudProvider bool httpProxy string noProxy string - insecureRegistries []string - registryMirrors map[string][]string + insecureRegistries string + registryMirrors string pauseImage string containerruntime string } @@ -176,7 +176,7 @@ func TestUserDataGeneration(t *testing.T) { cloudProviderName: stringPtr("vsphere"), httpProxy: "http://192.168.100.100:3128/", noProxy: "192.168.1.0", - insecureRegistries: []string{"192.168.100.100:5000", "10.0.0.1:5000"}, + insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { @@ -190,7 +190,7 @@ func TestUserDataGeneration(t *testing.T) { cloudProviderName: stringPtr("vsphere"), httpProxy: "http://192.168.100.100:3128/", noProxy: "192.168.1.0", - registryMirrors: map[string][]string{"docker.io": {"https://registry.docker-cn.com/"}}, + registryMirrors: "https://registry.docker-cn.com/", pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { @@ -250,6 +250,16 @@ func TestUserDataGeneration(t *testing.T) { t.Fatalf("failed to get cloud config: %v", err) } + containerRuntimeOpts := containerruntime.Opts{ + ContainerRuntime: test.containerruntime, + InsecureRegistries: test.insecureRegistries, + RegistryMirrors: test.registryMirrors, + } + containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) + if err != nil { + t.Fatalf("failed to generate container runtime config: %v", err) + } + req := plugin.UserDataRequest{ MachineSpec: test.spec, Kubeconfig: kubeconfig, @@ -262,11 +272,7 @@ func TestUserDataGeneration(t *testing.T) { NoProxy: test.noProxy, PauseImage: test.pauseImage, KubeletFeatureGates: kubeletFeatureGates, - ContainerRuntime: containerruntime.Get( - test.containerruntime, - containerruntime.WithInsecureRegistries(test.insecureRegistries), - containerruntime.WithRegistryMirrors(test.registryMirrors), - ), + ContainerRuntime: containerRuntimeConfig, } s, err := provider.UserData(req) if err != nil { diff --git a/pkg/userdata/sles/provider_test.go b/pkg/userdata/sles/provider_test.go index dba3dfccf..b04304099 100644 --- a/pkg/userdata/sles/provider_test.go +++ b/pkg/userdata/sles/provider_test.go @@ -118,8 +118,8 @@ type userDataTestCase struct { externalCloudProvider bool httpProxy string noProxy string - insecureRegistries []string - registryMirrors map[string][]string + insecureRegistries string + registryMirrors string pauseImage string containerruntime string } @@ -350,7 +350,7 @@ func TestUserDataGeneration(t *testing.T) { }, httpProxy: "http://192.168.100.100:3128/", noProxy: "192.168.1.0", - registryMirrors: map[string][]string{"docker.io": {"https://registry.docker-cn.com/"}}, + registryMirrors: "https://registry.docker-cn.com/", pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { @@ -380,7 +380,7 @@ func TestUserDataGeneration(t *testing.T) { }, httpProxy: "http://192.168.100.100:3128/", noProxy: "192.168.1.0", - insecureRegistries: []string{"192.168.100.100:5000", "10.0.0.1:5000"}, + insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { @@ -437,6 +437,16 @@ func TestUserDataGeneration(t *testing.T) { t.Fatalf("failed to get cloud config: %v", err) } + containerRuntimeOpts := containerruntime.Opts{ + ContainerRuntime: test.containerruntime, + InsecureRegistries: test.insecureRegistries, + RegistryMirrors: test.registryMirrors, + } + containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) + if err != nil { + t.Fatalf("failed to generate container runtime config: %v", err) + } + req := plugin.UserDataRequest{ MachineSpec: test.spec, Kubeconfig: kubeconfig, @@ -449,11 +459,7 @@ func TestUserDataGeneration(t *testing.T) { NoProxy: test.noProxy, PauseImage: test.pauseImage, KubeletFeatureGates: kubeletFeatureGates, - ContainerRuntime: containerruntime.Get( - test.containerruntime, - containerruntime.WithInsecureRegistries(test.insecureRegistries), - containerruntime.WithRegistryMirrors(test.registryMirrors), - ), + ContainerRuntime: containerRuntimeConfig, } s, err := provider.UserData(req) if err != nil { diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 0962a064d..6c1ecd00e 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -107,21 +107,22 @@ func (p *fakeCloudConfigProvider) GetCloudConfig(spec clusterv1alpha1.MachineSpe // userDataTestCase contains the data for a table-driven test. type userDataTestCase struct { - name string - spec clusterv1alpha1.MachineSpec - ccProvider cloud.ConfigProvider - osConfig *Config - providerSpec *providerconfigtypes.Config - DNSIPs []net.IP - kubernetesCACert string - externalCloudProvider bool - httpProxy string - noProxy string - insecureRegistries []string - registryMirrors map[string][]string - registryCredentials map[string]containerruntime.AuthConfig - pauseImage string - containerruntime string + name string + spec clusterv1alpha1.MachineSpec + ccProvider cloud.ConfigProvider + osConfig *Config + providerSpec *providerconfigtypes.Config + DNSIPs []net.IP + kubernetesCACert string + externalCloudProvider bool + httpProxy string + noProxy string + insecureRegistries string + registryMirrors string + containerdRegistryMirrors containerruntime.RegistryMirrorsFlags + registryCredentials map[string]containerruntime.AuthConfig + pauseImage string + containerruntime string } func simpleVersionTests() []userDataTestCase { @@ -375,7 +376,7 @@ func TestUserDataGeneration(t *testing.T) { }, httpProxy: "http://192.168.100.100:3128/", noProxy: "192.168.1.0", - insecureRegistries: []string{"192.168.100.100:5000", "10.0.0.1:5000"}, + insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { @@ -405,7 +406,7 @@ func TestUserDataGeneration(t *testing.T) { }, httpProxy: "http://192.168.100.100:3128/", noProxy: "192.168.1.0", - registryMirrors: map[string][]string{"docker.io": {"https://registry.docker-cn.com/"}}, + registryMirrors: "https://registry.docker-cn.com/", pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { @@ -417,8 +418,8 @@ func TestUserDataGeneration(t *testing.T) { Password: "passwd1", }, }, - insecureRegistries: []string{"k8s.gcr.io"}, - registryMirrors: map[string][]string{ + insecureRegistries: "k8s.gcr.io", + containerdRegistryMirrors: map[string][]string{ "k8s.gcr.io": {"https://intranet.local/"}, }, providerSpec: &providerconfigtypes.Config{ @@ -499,6 +500,18 @@ func TestUserDataGeneration(t *testing.T) { t.Fatalf("failed to get cloud config: %v", err) } + containerRuntimeOpts := containerruntime.Opts{ + ContainerRuntime: test.containerruntime, + InsecureRegistries: test.insecureRegistries, + RegistryMirrors: test.registryMirrors, + ContainerdRegistryMirrors: test.containerdRegistryMirrors, + } + containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) + if err != nil { + t.Fatalf("failed to generate container runtime config: %v", err) + } + containerRuntimeConfig.RegistryCredentials = test.registryCredentials + req := plugin.UserDataRequest{ MachineSpec: test.spec, Kubeconfig: kubeconfig, @@ -511,12 +524,7 @@ func TestUserDataGeneration(t *testing.T) { NoProxy: test.noProxy, PauseImage: test.pauseImage, KubeletFeatureGates: kubeletFeatureGates, - ContainerRuntime: containerruntime.Get( - test.containerruntime, - containerruntime.WithInsecureRegistries(test.insecureRegistries), - containerruntime.WithRegistryMirrors(test.registryMirrors), - containerruntime.WithRegistryCredentials(test.registryCredentials), - ), + ContainerRuntime: containerRuntimeConfig, } s, err := provider.UserData(req) if err != nil { From fef99436e1e017ebcb85bac675247e961c24ee7e Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Wed, 26 Jan 2022 13:32:51 +0100 Subject: [PATCH 071/489] Enable SeccompDefault setting if feature gate is passed (#1176) Signed-off-by: Marvin Beckers --- pkg/userdata/helper/kubelet.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index d243acdf4..c6504be33 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -235,6 +235,10 @@ func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGate } } + if enabled, ok := featureGates["SeccompDefault"]; ok && enabled { + cfg.SeccompDefault = pointer.Bool(true) + } + buf, err := kyaml.Marshal(cfg) return string(buf), err } From 429d65343db78adfa697c7b84a172791f682a7ec Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 27 Jan 2022 14:07:53 +0500 Subject: [PATCH 072/489] Update compatibility-matrix (#1170) * Update compatibility-matrix * Remove container linux from compatibility matrix --- docs/operating-system.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/operating-system.md b/docs/operating-system.md index fea7bb582..df44f62f8 100644 --- a/docs/operating-system.md +++ b/docs/operating-system.md @@ -4,15 +4,15 @@ ### Cloud provider -| | Ubuntu | Container Linux | CentOS | Flatcar | RHEL | SLES | Amazon Linux 2 | -|---|---|---|---|---|---|---|---| -| AWS | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Azure | ✓ | ✓ | ✓ | ✓ | ✓ | x | x | -| Digitalocean | ✓ | ✓ | ✓ | x | x | x | x | -| Google Cloud Platform | ✓ | ✓ | x | x | ✓ | x | x | -| Hetzner | ✓ | x | ✓ | x | x | x | x | -| Equinix Metal | ✓ | ✓ | ✓ | x | x | x | x | -| Openstack | ✓ | ✓ | ✓ | x | ✓ | x | x | +| | Ubuntu | CentOS | Flatcar | RHEL | SLES | Amazon Linux 2 | +|---|---|---|---|---|---|---| +| AWS | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Azure | ✓ | ✓ | ✓ | ✓ | x | x | +| Digitalocean | ✓ | ✓ | x | x | x | x | +| Google Cloud Platform | ✓ | x | x | x | x | x | +| Hetzner | ✓ | ✓ | x | x | x | x | +| Equinix Metal | ✓ | ✓ | x | x | x | x | +| Openstack | ✓ | ✓ | ✓ | ✓ | x | x | ## Configuring a operating system From 46caa87665e060334af50c09d4e554dd6948f21e Mon Sep 17 00:00:00 2001 From: Obinna Odirionye Date: Fri, 28 Jan 2022 01:43:52 +0400 Subject: [PATCH 073/489] Add support for specifying logging configuration for kubelet (#1173) * chore: add container logs maxsize with tests * Add support for configuring logging configuration for kubelet Signed-off-by: Waleed Malik * Refactored code Signed-off-by: Waleed Malik * Use const for ContainerLogMaxSize default value Signed-off-by: Waleed Malik * Use const for ContainerLogMaxSize default value Signed-off-by: Waleed Malik Co-authored-by: Waleed Malik --- pkg/apis/cluster/common/consts.go | 8 ++- pkg/containerruntime/containerruntime.go | 20 ++++--- pkg/containerruntime/docker.go | 8 ++- pkg/controller/machine/machine_controller.go | 15 ++++- pkg/userdata/amzn2/provider.go | 2 +- .../containerd-kubelet-v1.20-aws.yaml | 1 + .../amzn2/testdata/kubelet-v1.20-aws.yaml | 3 +- .../testdata/kubelet-v1.21-aws-external.yaml | 3 +- .../amzn2/testdata/kubelet-v1.21-aws.yaml | 3 +- .../kubelet-v1.21-vsphere-mirrors.yaml | 3 +- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 3 +- .../amzn2/testdata/kubelet-v1.21-vsphere.yaml | 3 +- .../amzn2/testdata/kubelet-v1.22-aws.yaml | 3 +- .../amzn2/testdata/kubelet-v1.23-aws.yaml | 3 +- pkg/userdata/centos/provider.go | 4 +- .../kubelet-containerd-v1.20-aws.yaml | 1 + .../centos/testdata/kubelet-v1.20-aws.yaml | 3 +- .../testdata/kubelet-v1.21-aws-external.yaml | 3 +- .../centos/testdata/kubelet-v1.21-aws.yaml | 3 +- .../testdata/kubelet-v1.21-nutanix.yaml | 3 +- .../kubelet-v1.21-vsphere-mirrors.yaml | 3 +- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 3 +- .../testdata/kubelet-v1.21-vsphere.yaml | 3 +- .../centos/testdata/kubelet-v1.22-aws.yaml | 3 +- .../centos/testdata/kubelet-v1.23-aws.yaml | 3 +- pkg/userdata/flatcar/provider.go | 4 +- .../flatcar/testdata/cloud-init_v1.20.14.yaml | 3 +- .../flatcar/testdata/cloud-init_v1.21.8.yaml | 3 +- .../flatcar/testdata/cloud-init_v1.22.5.yaml | 3 +- .../flatcar/testdata/cloud-init_v1.23.0.yaml | 3 +- pkg/userdata/flatcar/testdata/containerd.yaml | 1 + .../flatcar/testdata/ignition_v1.20.14.json | 2 +- .../flatcar/testdata/ignition_v1.21.8.json | 2 +- .../flatcar/testdata/ignition_v1.22.5.json | 2 +- .../flatcar/testdata/ignition_v1.23.0.json | 2 +- pkg/userdata/helper/helper.go | 26 +++++++- pkg/userdata/helper/kubelet.go | 25 +++++++- pkg/userdata/rhel/provider.go | 4 +- .../kubelet-containerd-v1.20-aws.yaml | 1 + .../rhel/testdata/kubelet-v1.20-aws.yaml | 3 +- .../rhel/testdata/kubelet-v1.21-aws.yaml | 3 +- .../rhel/testdata/kubelet-v1.22-aws.yaml | 3 +- .../rhel/testdata/kubelet-v1.22-nutanix.yaml | 3 +- .../testdata/kubelet-v1.23-aws-external.yaml | 3 +- .../rhel/testdata/kubelet-v1.23-aws.yaml | 3 +- .../kubelet-v1.23-vsphere-mirrors.yaml | 3 +- .../testdata/kubelet-v1.23-vsphere-proxy.yaml | 3 +- .../rhel/testdata/kubelet-v1.23-vsphere.yaml | 3 +- pkg/userdata/sles/provider.go | 59 +++++++++++-------- .../sles/testdata/dist-upgrade-on-boot.yaml | 5 +- .../kubelet-version-without-v-prefix.yaml | 5 +- .../sles/testdata/multiple-dns-servers.yaml | 5 +- .../sles/testdata/multiple-ssh-keys.yaml | 5 +- .../openstack-overwrite-cloud-config.yaml | 5 +- pkg/userdata/sles/testdata/openstack.yaml | 5 +- .../sles/testdata/version-1.20.14.yaml | 5 +- .../sles/testdata/version-1.21.8.yaml | 5 +- .../sles/testdata/version-1.22.5.yaml | 5 +- .../sles/testdata/version-1.23.0.yaml | 5 +- .../sles/testdata/vsphere-mirrors.yaml | 5 +- pkg/userdata/sles/testdata/vsphere-proxy.yaml | 5 +- pkg/userdata/sles/testdata/vsphere.yaml | 5 +- pkg/userdata/ubuntu/provider.go | 4 +- pkg/userdata/ubuntu/testdata/containerd.yaml | 1 + .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 3 +- .../kubelet-version-without-v-prefix.yaml | 3 +- .../ubuntu/testdata/multiple-dns-servers.yaml | 3 +- .../ubuntu/testdata/multiple-ssh-keys.yaml | 3 +- pkg/userdata/ubuntu/testdata/nutanix.yaml | 3 +- .../openstack-overwrite-cloud-config.yaml | 3 +- pkg/userdata/ubuntu/testdata/openstack.yaml | 3 +- .../ubuntu/testdata/version-1.20.14.yaml | 3 +- .../ubuntu/testdata/version-1.21.8.yaml | 3 +- .../ubuntu/testdata/version-1.22.5.yaml | 3 +- .../ubuntu/testdata/version-1.23.0.yaml | 3 +- .../ubuntu/testdata/vsphere-mirrors.yaml | 3 +- .../ubuntu/testdata/vsphere-proxy.yaml | 3 +- pkg/userdata/ubuntu/testdata/vsphere.yaml | 3 +- 78 files changed, 273 insertions(+), 116 deletions(-) diff --git a/pkg/apis/cluster/common/consts.go b/pkg/apis/cluster/common/consts.go index 056a80c21..89639b1f1 100644 --- a/pkg/apis/cluster/common/consts.go +++ b/pkg/apis/cluster/common/consts.go @@ -135,9 +135,11 @@ const ( ) const ( - SystemReservedKubeletConfig = "SystemReserved" - KubeReservedKubeletConfig = "KubeReserved" - EvictionHardKubeletConfig = "EvictionHard" + SystemReservedKubeletConfig = "SystemReserved" + KubeReservedKubeletConfig = "KubeReserved" + EvictionHardKubeletConfig = "EvictionHard" + ContainerLogMaxSizeKubeletConfig = "ContainerLogMaxSize" + ContainerLogMaxFilesKubeletConfig = "ContainerLogMaxFiles" ) const ( diff --git a/pkg/containerruntime/containerruntime.go b/pkg/containerruntime/containerruntime.go index c4e2852c7..1cc2ea650 100644 --- a/pkg/containerruntime/containerruntime.go +++ b/pkg/containerruntime/containerruntime.go @@ -78,12 +78,14 @@ func get(containerRuntimeName string, opts ...Opt) Config { } type Config struct { - Docker *Docker `json:",omitempty"` - Containerd *Containerd `json:",omitempty"` - InsecureRegistries []string `json:",omitempty"` - RegistryMirrors map[string][]string `json:",omitempty"` - RegistryCredentials map[string]AuthConfig `json:",omitempty"` - SandboxImage string `json:",omitempty"` + Docker *Docker `json:",omitempty"` + Containerd *Containerd `json:",omitempty"` + InsecureRegistries []string `json:",omitempty"` + RegistryMirrors map[string][]string `json:",omitempty"` + RegistryCredentials map[string]AuthConfig `json:",omitempty"` + SandboxImage string `json:",omitempty"` + ContainerLogMaxFiles string `json:",omitempty"` + ContainerLogMaxSize string `json:",omitempty"` } func (cfg Config) String() string { @@ -99,8 +101,10 @@ func (cfg Config) String() string { func (cfg Config) Engine(kubeletVersion *semver.Version) Engine { docker := &Docker{ - insecureRegistries: cfg.InsecureRegistries, - registryMirrors: cfg.RegistryMirrors["docker.io"], + insecureRegistries: cfg.InsecureRegistries, + registryMirrors: cfg.RegistryMirrors["docker.io"], + containerLogMaxFiles: cfg.ContainerLogMaxFiles, + containerLogMaxSize: cfg.ContainerLogMaxSize, } containerd := &Containerd{ diff --git a/pkg/containerruntime/docker.go b/pkg/containerruntime/docker.go index 0c545d94a..c51f97cf0 100644 --- a/pkg/containerruntime/docker.go +++ b/pkg/containerruntime/docker.go @@ -31,12 +31,14 @@ const ( ) type Docker struct { - insecureRegistries []string - registryMirrors []string + insecureRegistries []string + registryMirrors []string + containerLogMaxFiles string + containerLogMaxSize string } func (eng *Docker) Config() (string, error) { - return helper.DockerConfig(eng.insecureRegistries, eng.registryMirrors) + return helper.DockerConfig(eng.insecureRegistries, eng.registryMirrors, eng.containerLogMaxFiles, eng.containerLogMaxSize) } func (eng *Docker) ConfigFileName() string { diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 7fee0554f..7431d2634 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -717,7 +717,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( // grab kubelet general options from the annotations kubeletFlags := common.GetKubeletFlags(machine.GetAnnotations()) - KubeletConfigs := common.GetKubeletConfigs(machine.GetAnnotations()) + kubeletConfigs := common.GetKubeletConfigs(machine.GetAnnotations()) // look up for ExternalCloudProvider feature, with fallback to command-line input externalCloudProvider := r.nodeSettings.ExternalCloudProvider @@ -733,6 +733,14 @@ func (r *Reconciler) ensureInstanceExistsForMachine( crRuntime := r.nodeSettings.ContainerRuntime crRuntime.RegistryCredentials = registryCredentials + if val, ok := kubeletConfigs[common.ContainerLogMaxSizeKubeletConfig]; ok { + crRuntime.ContainerLogMaxSize = val + } + + if val, ok := kubeletConfigs[common.ContainerLogMaxFilesKubeletConfig]; ok { + crRuntime.ContainerLogMaxFiles = val + } + req := plugin.UserDataRequest{ MachineSpec: machine.Spec, Kubeconfig: kubeconfig, @@ -743,7 +751,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( PauseImage: r.nodeSettings.PauseImage, KubeletCloudProviderName: kubeletCloudProviderName, KubeletFeatureGates: kubeletFeatureGates, - KubeletConfigs: KubeletConfigs, + KubeletConfigs: kubeletConfigs, NoProxy: r.nodeSettings.NoProxy, HTTPProxy: r.nodeSettings.HTTPProxy, ContainerRuntime: crRuntime, @@ -760,8 +768,9 @@ func (r *Reconciler) ensureInstanceExistsForMachine( return nil, fmt.Errorf("failed to find machine's MachineDployment: %v", err) } - cloudConfigSecretName := fmt.Sprintf("%s-%s", + cloudConfigSecretName := fmt.Sprintf("%s-%s-%s", referencedMachineDeployment, + machine.Namespace, provisioningSuffix) // It is important to check if the secret holding cloud-config exists diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index a6ac2cc26..5e0685022 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -265,7 +265,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }} - path: "/etc/kubernetes/pki/ca.crt" content: | diff --git a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml index a0cd36bf1..9b2714553 100644 --- a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml @@ -299,6 +299,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml index 4aef3134b..c964cdfa6 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml @@ -296,6 +296,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -397,7 +398,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml index 1589d659e..2d0490115 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml @@ -296,6 +296,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -397,7 +398,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml index d12483f32..60269c6be 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml @@ -296,6 +296,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -397,7 +398,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 0bb089e0b..f842297a2 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -313,6 +313,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -414,7 +415,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"},"registry-mirrors":["https://registry.docker-cn.com/"]} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"registry-mirrors":["https://registry.docker-cn.com/"]} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml index 9d8770ec9..21d3d0b23 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -313,6 +313,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -414,7 +415,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"},"insecure-registries":["192.168.100.100:5000","10.0.0.1:5000"]} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"insecure-registries":["192.168.100.100:5000","10.0.0.1:5000"]} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml index e8888cd18..b89c333f8 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml @@ -304,6 +304,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -405,7 +406,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml index 90cf27ffd..2c5ffec9d 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml @@ -296,6 +296,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -397,7 +398,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml index 86bd8353e..ff061954d 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml @@ -294,6 +294,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -395,7 +396,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index 57973aa6c..051cc643d 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -224,7 +224,7 @@ write_files: iscsi-initiator-utils \ {{- end }} ipvsadm - + {{- /* iscsid service is required on Nutanix machines for CSI driver to attach volumes. */}} {{- if eq .CloudProviderName "nutanix" }} systemctl enable --now iscsid @@ -272,7 +272,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }} - path: "/etc/kubernetes/pki/ca.crt" content: | diff --git a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml index 4a2bc488f..fd82085db 100644 --- a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml @@ -299,6 +299,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml index 71417c801..66f695396 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml @@ -300,6 +300,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -401,7 +402,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml index eb9c0e540..fb5063453 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml @@ -300,6 +300,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -401,7 +402,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml index e95d9e99b..3c9b1c75e 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml @@ -300,6 +300,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -401,7 +402,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml index bee7b653e..ea64b0cb0 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml @@ -308,6 +308,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -409,7 +410,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 238c7ebf6..4a41b8b61 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -317,6 +317,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -418,7 +419,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"},"registry-mirrors":["https://registry.docker-cn.com/"]} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"registry-mirrors":["https://registry.docker-cn.com/"]} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml index 04783408f..07af4033a 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -317,6 +317,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -418,7 +419,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"},"insecure-registries":["192.168.100.100:5000","10.0.0.1:5000"]} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"insecure-registries":["192.168.100.100:5000","10.0.0.1:5000"]} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml index d7fd521f7..dcb2e08c7 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml @@ -308,6 +308,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -409,7 +410,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index ba9391fc1..6c660ca72 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -300,6 +300,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -401,7 +402,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index 642ff989e..5b8f9a98a 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -298,6 +298,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -399,7 +400,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index 3bc8ca7b3..45f93a5c8 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -277,7 +277,7 @@ storage: mode: 0644 contents: inline: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 10 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 10 }} - path: /opt/load-kernel-modules.sh filesystem: root @@ -551,7 +551,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" permissions: "0644" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }} - path: /opt/load-kernel-modules.sh permissions: "0755" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml index a4fcbc840..20b48a9b5 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml @@ -174,6 +174,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -458,7 +459,7 @@ write_files: permissions: "0644" user: root content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/crictl.yaml permissions: "0644" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml index 10b2beef3..bc9e03f0a 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml @@ -174,6 +174,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -458,7 +459,7 @@ write_files: permissions: "0644" user: root content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/crictl.yaml permissions: "0644" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml index e5158b13a..8e28e9949 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml @@ -174,6 +174,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -458,7 +459,7 @@ write_files: permissions: "0644" user: root content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/crictl.yaml permissions: "0644" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml index 70214dcec..bf4d03c73 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml @@ -172,6 +172,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -456,7 +457,7 @@ write_files: permissions: "0644" user: root content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/crictl.yaml permissions: "0644" diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index f6525236d..a6f63ee8a 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -157,6 +157,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json b/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json index b5072e336..8872fa45c 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.14%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.14%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json b/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json index 7679a2f17..ebe926e02 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.8%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.8%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json b/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json index 245525c21..0d4408f32 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json index baf23fc1e..f6dd00901 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%2210m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/helper.go b/pkg/userdata/helper/helper.go index 5d22d4c81..fbf9e1657 100644 --- a/pkg/userdata/helper/helper.go +++ b/pkg/userdata/helper/helper.go @@ -25,6 +25,11 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" ) +const ( + DefaultDockerContainerLogMaxFiles = "5" + DefaultDockerContainerLogMaxSize = "100m" +) + func GetServerAddressFromKubeconfig(kubeconfig *clientcmdapi.Config) (string, error) { if len(kubeconfig.Clusters) != 1 { return "", fmt.Errorf("kubeconfig does not contain exactly one cluster, can not extract server address") @@ -112,14 +117,29 @@ type dockerConfig struct { } // DockerConfig returns the docker daemon.json. -func DockerConfig(insecureRegistries, registryMirrors []string) (string, error) { +func DockerConfig(insecureRegistries, registryMirrors []string, logMaxFiles string, logMaxSize string) (string, error) { + if len(logMaxSize) > 0 { + // Parse log max size to ensure that it has the correct units + logMaxSize = strings.ToLower(logMaxSize) + logMaxSize = strings.ReplaceAll(logMaxSize, "ki", "k") + logMaxSize = strings.ReplaceAll(logMaxSize, "mi", "m") + logMaxSize = strings.ReplaceAll(logMaxSize, "gi", "g") + } else { + logMaxSize = DefaultDockerContainerLogMaxSize + } + + // Default if value is not provided + if len(logMaxFiles) == 0 { + logMaxFiles = DefaultDockerContainerLogMaxFiles + } + cfg := dockerConfig{ ExecOpts: []string{"native.cgroupdriver=systemd"}, StorageDriver: "overlay2", LogDriver: "json-file", LogOpts: map[string]string{ - "max-size": "10m", - "max-file": "5", + "max-size": logMaxSize, + "max-file": logMaxFiles, }, InsecureRegistries: insecureRegistries, RegistryMirrors: registryMirrors, diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index c6504be33..d5d3e603c 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -19,6 +19,7 @@ package helper import ( "fmt" "net" + "strconv" "strings" "text/template" @@ -28,11 +29,16 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/klog" kubeletv1b1 "k8s.io/kubelet/config/v1beta1" "k8s.io/utils/pointer" kyaml "sigs.k8s.io/yaml" ) +const ( + defaultKubeletContainerLogMaxSize = "100Mi" +) + const ( kubeletFlagsTpl = `--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ @@ -164,7 +170,7 @@ func KubeletSystemdUnit(containerRuntime, kubeletVersion, cloudProvider, hostnam } // kubeletConfiguration returns marshaled kubelet.config.k8s.io/v1beta1 KubeletConfiguration -func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGates map[string]bool, kubeletConfigs map[string]string) (string, error) { +func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGates map[string]bool, kubeletConfigs map[string]string, containerRuntime string) (string, error) { clusterDNSstr := make([]string, 0, len(clusterDNS)) for _, ip := range clusterDNS { clusterDNSstr = append(clusterDNSstr, ip.String()) @@ -203,6 +209,7 @@ func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGate EvictionHard: map[string]string{"memory.available": "100Mi", "nodefs.available": "10%", "nodefs.inodesFree": "5%", "imagefs.available": "15%"}, VolumePluginDir: "/var/lib/kubelet/volumeplugins", TLSCipherSuites: kubeletTLSCipherSuites, + ContainerLogMaxSize: defaultKubeletContainerLogMaxSize, } if kubeReserved, ok := kubeletConfigs[common.KubeReservedKubeletConfig]; ok { @@ -235,6 +242,22 @@ func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGate } } + // ContainerLogMaxSize and ContainerLogMaxFiles have no effect if container runtime is docker i.e. not remote + if containerRuntime != "docker" { + if containerLogMaxSize, ok := kubeletConfigs[common.ContainerLogMaxSizeKubeletConfig]; ok { + cfg.ContainerLogMaxSize = containerLogMaxSize + } + if containerLogMaxFiles, ok := kubeletConfigs[common.ContainerLogMaxFilesKubeletConfig]; ok { + maxFiles, err := strconv.Atoi(containerLogMaxFiles) + if err != nil || maxFiles < 0 { + // Instead of breaking the workflow, just print a warning and skip the configuration + klog.Warningf("Skipping invalid ContainerLogMaxSize value %v for Kubelet configuration", containerLogMaxFiles) + } else { + cfg.ContainerLogMaxFiles = pointer.Int32Ptr(int32(maxFiles)) + } + } + } + if enabled, ok := featureGates["SeccompDefault"]; ok && enabled { cfg.SeccompDefault = pointer.Bool(true) } diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 411ebd32b..306f863f1 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -229,7 +229,7 @@ write_files: iscsi-initiator-utils \ {{- end }} ipvsadm - + {{- /* iscsid service is required on Nutanix machines for CSI driver to attach volumes. */}} {{- if eq .CloudProviderName "nutanix" }} systemctl enable --now iscsid @@ -285,7 +285,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }} - path: "/etc/kubernetes/pki/ca.crt" content: | diff --git a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml index c57587acd..79685ec63 100644 --- a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml @@ -300,6 +300,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml index f956a6cb6..bf5edafb4 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml @@ -301,6 +301,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -402,7 +403,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml index 9c539c954..a08bf51df 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml @@ -301,6 +301,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -402,7 +403,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index d23515508..fc62de7de 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -301,6 +301,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -402,7 +403,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml index a95765679..24e55ecd8 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml @@ -310,6 +310,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -411,7 +412,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index b33e71100..353463c22 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -299,6 +299,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -400,7 +401,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index 1216b9206..75e193f4d 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -299,6 +299,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -400,7 +401,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 7a245c7d9..f56a3f063 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -317,6 +317,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -418,7 +419,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"},"registry-mirrors":["https://registry.docker-cn.com/"]} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"registry-mirrors":["https://registry.docker-cn.com/"]} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index 7d9464d77..04a47939c 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -317,6 +317,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -418,7 +419,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"},"insecure-registries":["192.168.100.100:5000","10.0.0.1:5000"]} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"insecure-registries":["192.168.100.100:5000","10.0.0.1:5000"]} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index 0eb0919b9..173e97e39 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -308,6 +308,7 @@ write_files: cacheUnauthorizedTTL: 0s cgroupDriver: systemd clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -409,7 +410,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/sles/provider.go b/pkg/userdata/sles/provider.go index 4ccdb75b6..366ce072e 100644 --- a/pkg/userdata/sles/provider.go +++ b/pkg/userdata/sles/provider.go @@ -82,29 +82,38 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { return "", fmt.Errorf("error extracting cacert: %v", err) } + crEngine := req.ContainerRuntime.Engine(kubeletVersion) + crConfig, err := crEngine.Config() + if err != nil { + return "", fmt.Errorf("failed to generate container runtime config: %w", err) + } + data := struct { plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - OSConfig *Config - ServerAddr string - KubeletVersion string - Kubeconfig string - KubernetesCACert string - NodeIPScript string - ExtraKubeletFlags []string - InsecureRegistries []string - RegistryMirrors []string + ProviderSpec *providerconfigtypes.Config + OSConfig *Config + ServerAddr string + KubeletVersion string + Kubeconfig string + KubernetesCACert string + NodeIPScript string + ExtraKubeletFlags []string + ContainerRuntimeConfigFileName string + ContainerRuntimeConfig string + ContainerRuntimeName string }{ - UserDataRequest: req, - ProviderSpec: pconfig, - OSConfig: slesConfig, - ServerAddr: serverAddr, - KubeletVersion: kubeletVersion.String(), - Kubeconfig: kubeconfigString, - KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), - InsecureRegistries: req.ContainerRuntime.InsecureRegistries, - RegistryMirrors: req.ContainerRuntime.RegistryMirrors["docker.io"], + UserDataRequest: req, + ProviderSpec: pconfig, + OSConfig: slesConfig, + ServerAddr: serverAddr, + KubeletVersion: kubeletVersion.String(), + Kubeconfig: kubeconfigString, + KubernetesCACert: kubernetesCACert, + NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), + ExtraKubeletFlags: crEngine.KubeletFlags(), + ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), + ContainerRuntimeConfig: crConfig, + ContainerRuntimeName: crEngine.String(), } b := &bytes.Buffer{} err = tmpl.Execute(b, data) @@ -202,7 +211,7 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntime.String .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} - path: "/etc/systemd/system/kubelet.service.d/extras.conf" content: | @@ -246,17 +255,17 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }} - path: "/etc/profile.d/opt-bin-path.sh" permissions: "0644" content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: {{ .ContainerRuntimeConfigFileName }} permissions: "0644" content: | -{{ dockerConfig .InsecureRegistries .RegistryMirrors | indent 4 }} +{{ .ContainerRuntimeConfig | indent 4 }} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" @@ -266,7 +275,7 @@ write_files: - path: /etc/systemd/system/docker-healthcheck.service permissions: "0644" content: | -{{ containerRuntimeHealthCheckSystemdUnit .ContainerRuntime.String | indent 4 }} +{{ containerRuntimeHealthCheckSystemdUnit .ContainerRuntimeName | indent 4 }} - path: /etc/systemd/system/docker.service.d/environment.conf permissions: "0644" diff --git a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml index 02831730c..b8549a686 100644 --- a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml @@ -178,6 +178,8 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} @@ -313,6 +315,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -369,7 +372,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml index f187a8d87..84b045912 100644 --- a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml @@ -176,6 +176,8 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} @@ -311,6 +313,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -367,7 +370,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml index 1dd38a7f2..d2875d8fe 100644 --- a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml @@ -176,6 +176,8 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} @@ -313,6 +315,7 @@ write_files: - 10.10.10.11 - 10.10.10.12 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -369,7 +372,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml index 69919657b..ed16832ef 100644 --- a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml @@ -178,6 +178,8 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} @@ -313,6 +315,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -369,7 +372,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml index 90f47e9fd..72a7bfc94 100644 --- a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml @@ -178,6 +178,8 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} @@ -315,6 +317,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -371,7 +374,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/sles/testdata/openstack.yaml b/pkg/userdata/sles/testdata/openstack.yaml index 408a4bbc3..1fa4cfbf8 100644 --- a/pkg/userdata/sles/testdata/openstack.yaml +++ b/pkg/userdata/sles/testdata/openstack.yaml @@ -178,6 +178,8 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} @@ -315,6 +317,7 @@ write_files: - 10.10.10.11 - 10.10.10.12 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -371,7 +374,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/sles/testdata/version-1.20.14.yaml b/pkg/userdata/sles/testdata/version-1.20.14.yaml index 4aac8a172..5355db2b3 100644 --- a/pkg/userdata/sles/testdata/version-1.20.14.yaml +++ b/pkg/userdata/sles/testdata/version-1.20.14.yaml @@ -176,6 +176,8 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} @@ -311,6 +313,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -367,7 +370,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/sles/testdata/version-1.21.8.yaml b/pkg/userdata/sles/testdata/version-1.21.8.yaml index e1af02656..7fc71dbe6 100644 --- a/pkg/userdata/sles/testdata/version-1.21.8.yaml +++ b/pkg/userdata/sles/testdata/version-1.21.8.yaml @@ -176,6 +176,8 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} @@ -311,6 +313,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -367,7 +370,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/sles/testdata/version-1.22.5.yaml b/pkg/userdata/sles/testdata/version-1.22.5.yaml index f187a8d87..84b045912 100644 --- a/pkg/userdata/sles/testdata/version-1.22.5.yaml +++ b/pkg/userdata/sles/testdata/version-1.22.5.yaml @@ -176,6 +176,8 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} @@ -311,6 +313,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -367,7 +370,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/sles/testdata/version-1.23.0.yaml b/pkg/userdata/sles/testdata/version-1.23.0.yaml index b8c35b6be..3f4c181a5 100644 --- a/pkg/userdata/sles/testdata/version-1.23.0.yaml +++ b/pkg/userdata/sles/testdata/version-1.23.0.yaml @@ -176,6 +176,8 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --node-ip ${KUBELET_NODE_IP} [Install] @@ -309,6 +311,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -365,7 +368,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml index 0c4253402..e92b70481 100644 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml @@ -189,6 +189,8 @@ write_files: --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} @@ -326,6 +328,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -382,7 +385,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"},"registry-mirrors":["https://registry.docker-cn.com/"]} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"registry-mirrors":["https://registry.docker-cn.com/"]} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml index 78ec370d5..937369f93 100644 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/sles/testdata/vsphere-proxy.yaml @@ -189,6 +189,8 @@ write_files: --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} @@ -326,6 +328,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -382,7 +385,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"},"insecure-registries":["192.168.100.100:5000","10.0.0.1:5000"]} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"insecure-registries":["192.168.100.100:5000","10.0.0.1:5000"]} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml index d8ed00fc8..337322d8f 100644 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ b/pkg/userdata/sles/testdata/vsphere.yaml @@ -179,6 +179,8 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} @@ -316,6 +318,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% @@ -372,7 +375,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index e9bcff1e1..3e435fed8 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -219,7 +219,7 @@ write_files: open-iscsi \ {{- end }} ipvsadm - + {{- /* iscsid service is required on Nutanix machines for CSI driver to attach volumes. */}} {{- if eq .CloudProviderName "nutanix" }} systemctl enable --now iscsid @@ -309,7 +309,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index 3d21ca6c1..cf7fdbb2a 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -396,6 +396,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index fa0faca4e..d08f60028 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -347,7 +347,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: "/etc/kubernetes/kubelet.conf" content: | @@ -369,6 +369,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index 27d75a820..c31cbd8d0 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -345,7 +345,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: "/etc/kubernetes/kubelet.conf" content: | @@ -367,6 +367,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 8a2a594a8..e96c48b0c 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -345,7 +345,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: "/etc/kubernetes/kubelet.conf" content: | @@ -369,6 +369,7 @@ write_files: - 10.10.10.11 - 10.10.10.12 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 4845acaf2..82f61d628 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -347,7 +347,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: "/etc/kubernetes/kubelet.conf" content: | @@ -369,6 +369,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 234501799..6880fca57 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -352,7 +352,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: "/etc/kubernetes/kubelet.conf" content: | @@ -374,6 +374,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index 4f890536b..13196ae4d 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -349,7 +349,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: "/etc/kubernetes/kubelet.conf" content: | @@ -371,6 +371,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 4e1210d21..9ff6357ed 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -347,7 +347,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: "/etc/kubernetes/kubelet.conf" content: | @@ -371,6 +371,7 @@ write_files: - 10.10.10.11 - 10.10.10.12 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml b/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml index fca5c5cc8..4374b4550 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml @@ -345,7 +345,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: "/etc/kubernetes/kubelet.conf" content: | @@ -367,6 +367,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml b/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml index 5194458b8..a3598fc76 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml @@ -345,7 +345,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: "/etc/kubernetes/kubelet.conf" content: | @@ -367,6 +367,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml index 27d75a820..c31cbd8d0 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml @@ -345,7 +345,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: "/etc/kubernetes/kubelet.conf" content: | @@ -367,6 +367,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml index f110deffb..9f080bb13 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml @@ -343,7 +343,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: "/etc/kubernetes/kubelet.conf" content: | @@ -365,6 +365,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index 682b6a924..58c4fcccd 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -360,7 +360,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"},"registry-mirrors":["https://registry.docker-cn.com/"]} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"registry-mirrors":["https://registry.docker-cn.com/"]} - path: "/etc/kubernetes/kubelet.conf" content: | @@ -382,6 +382,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 97b4d6b6c..89fbdc265 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -360,7 +360,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"},"insecure-registries":["192.168.100.100:5000","10.0.0.1:5000"]} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"insecure-registries":["192.168.100.100:5000","10.0.0.1:5000"]} - path: "/etc/kubernetes/kubelet.conf" content: | @@ -382,6 +382,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 5c7559782..1f260b7f5 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -350,7 +350,7 @@ write_files: - path: /etc/docker/daemon.json permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}} + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - path: "/etc/kubernetes/kubelet.conf" content: | @@ -372,6 +372,7 @@ write_files: clusterDNS: - 10.10.10.10 clusterDomain: cluster.local + containerLogMaxSize: 100Mi cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% From 854a4b22a430d654d12e53a60e06dcaf4ec0aa80 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Fri, 28 Jan 2022 14:51:53 +0100 Subject: [PATCH 074/489] Upgrade Operating System Manager (#1179) * fix conflicts Signed-off-by: Moath Qasim # Conflicts: # go.sum * update go sum Signed-off-by: Moath Qasim # Conflicts: # go.sum --- go.mod | 2 +- go.sum | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index c35691f53..f7f070b3e 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( google.golang.org/grpc v1.38.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b - k8c.io/operating-system-manager v0.3.9 + k8c.io/operating-system-manager v0.4.0 k8s.io/api v0.22.2 k8s.io/apiextensions-apiserver v0.22.2 k8s.io/apimachinery v0.22.2 diff --git a/go.sum b/go.sum index b9aafb928..3d1129134 100644 --- a/go.sum +++ b/go.sum @@ -859,6 +859,7 @@ github.com/kubermatic/machine-controller v1.23.1/go.mod h1:mXWbT7SjqpgFhzCFT3yME github.com/kubermatic/machine-controller v1.26.0/go.mod h1:dcJ+GdDSCxCwM0poxwOK8hVO7epiOORDmNMmb2veyw4= github.com/kubermatic/machine-controller v1.36.1/go.mod h1:6BFZEvEMZi8OT8aHOsS7DXYsF6ZSpmsNxsci7OLTTn8= github.com/kubermatic/machine-controller v1.40.1/go.mod h1:5LVcN4tCybGg+55hIHcVzCjNsBJy2PlnXG0xIzKmXGY= +github.com/kubermatic/machine-controller v1.42.2/go.mod h1:vr6i5XWfd5FIq2yodXcgdlKvOhMnM5uzn2XEZ2wcoFM= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -1971,8 +1972,9 @@ k8c.io/kubermatic/v2 v2.16.2 h1:tjPfI+VV51pggXCvcDL/qG1r7KHDBQPSPYngPxpRtp8= k8c.io/kubermatic/v2 v2.16.2/go.mod h1:NdW+2mq4ynRtfZs9yPnvcnFWQpzmM7ngntW6GeuQicU= k8c.io/operating-system-manager v0.1.0/go.mod h1:ULyZQO1irKjsQTNjIdrHld7SZ+joHjmPnOEs5Db8G8M= k8c.io/operating-system-manager v0.3.0/go.mod h1:ME5GOCNUrHG+57igEKP1JCJKVHynaLfodT8bRiYH3MY= -k8c.io/operating-system-manager v0.3.9 h1:GcZgXqh90XYKdDXRMaMenA9AE30T8PZ47fM3yrwnygc= k8c.io/operating-system-manager v0.3.9/go.mod h1:aFyB/RH9DBAk0Kj5JVtCixhm9ugTeC8akgRGMW28lPg= +k8c.io/operating-system-manager v0.4.0 h1:6F9kxELwHmhqLDLAAlodihBOnSfWM+8FPtbWcOshPGU= +k8c.io/operating-system-manager v0.4.0/go.mod h1:pJImhsLb5GJdZunZ47r5Db0ydBwhWxhgL6mUKbU4Vps= k8s.io/api v0.0.0-20181018013834-843ad2d9b9ae/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20181115043458-b799cb063522/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20190725062911-6607c48751ae/go.mod h1:1O0xzX/RAtnm7l+5VEUxZ1ysO2ghatfq/OZED4zM9kA= From 99ad41562e365d42743aa6f639e657b7a324c36c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Thu, 3 Feb 2022 10:03:07 +0100 Subject: [PATCH 075/489] Change baseurl to vault.centos.org for CentOS 8 (#1182) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Change CentOS baseurl to vault.centos.org Signed-off-by: Marko Mudrinić * Update fixtures Signed-off-by: Marko Mudrinić --- pkg/userdata/centos/provider.go | 7 +++++++ .../centos/testdata/kubelet-containerd-v1.20-aws.yaml | 5 +++++ pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml | 5 +++++ .../centos/testdata/kubelet-v1.21-aws-external.yaml | 5 +++++ pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml | 5 +++++ pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml | 5 +++++ .../centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml | 5 +++++ .../centos/testdata/kubelet-v1.21-vsphere-proxy.yaml | 5 +++++ pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml | 5 +++++ pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml | 5 +++++ pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml | 5 +++++ 11 files changed, 57 insertions(+) diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index 051cc643d..eac197eb5 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -206,6 +206,13 @@ write_files: hostnamectl set-hostname {{ .MachineSpec.Name }} {{ end }} +{{- /* CentOS 8 has reached EOL and all packages were moved to vault.centos.org -- https://www.centos.org/centos-linux-eol/ */}} + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi + yum install -y \ device-mapper-persistent-data \ lvm2 \ diff --git a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml index fd82085db..a2e8acb7f 100644 --- a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml @@ -66,6 +66,11 @@ write_files: sed -i.orig '/.*swap.*/d' /etc/fstab swapoff -a + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi yum install -y \ device-mapper-persistent-data \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml index 66f695396..57a36fc33 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml @@ -66,6 +66,11 @@ write_files: sed -i.orig '/.*swap.*/d' /etc/fstab swapoff -a + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi yum install -y \ device-mapper-persistent-data \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml index fb5063453..2ff821ad9 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml @@ -66,6 +66,11 @@ write_files: sed -i.orig '/.*swap.*/d' /etc/fstab swapoff -a + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi yum install -y \ device-mapper-persistent-data \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml index 3c9b1c75e..98cccafbe 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml @@ -66,6 +66,11 @@ write_files: sed -i.orig '/.*swap.*/d' /etc/fstab swapoff -a + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi yum install -y \ device-mapper-persistent-data \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml index ea64b0cb0..84737013e 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml @@ -70,6 +70,11 @@ write_files: hostnamectl set-hostname node1 + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi yum install -y \ device-mapper-persistent-data \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 4a41b8b61..be65ea5c1 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -78,6 +78,11 @@ write_files: hostnamectl set-hostname node1 + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi yum install -y \ device-mapper-persistent-data \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml index 07af4033a..ccd0349f8 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -78,6 +78,11 @@ write_files: hostnamectl set-hostname node1 + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi yum install -y \ device-mapper-persistent-data \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml index dcb2e08c7..4b0bb7760 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml @@ -70,6 +70,11 @@ write_files: hostnamectl set-hostname node1 + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi yum install -y \ device-mapper-persistent-data \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index 6c660ca72..5a0ed8714 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -66,6 +66,11 @@ write_files: sed -i.orig '/.*swap.*/d' /etc/fstab swapoff -a + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi yum install -y \ device-mapper-persistent-data \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index 5b8f9a98a..839bd2ee6 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -66,6 +66,11 @@ write_files: sed -i.orig '/.*swap.*/d' /etc/fstab swapoff -a + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi yum install -y \ device-mapper-persistent-data \ From 85ed54c3745c417259cfe50a8508cf9bbfe22979 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Thu, 3 Feb 2022 13:32:04 +0200 Subject: [PATCH 076/489] Apply band-aid to reanimate CentOS8 bootstraping (#1183) Signed-off-by: Artiom Diomin --- pkg/controller/machine/bootstrap.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/controller/machine/bootstrap.go b/pkg/controller/machine/bootstrap.go index 02666d281..d68705583 100644 --- a/pkg/controller/machine/bootstrap.go +++ b/pkg/controller/machine/bootstrap.go @@ -230,6 +230,11 @@ systemctl restart kubelet-healthcheck.service bootstrapYumBinContentTemplate = `#!/bin/bash set -xeuo pipefail +source /etc/os-release +if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* +fi {{- if .EnterpriseLinux }} yum install epel-release -y {{- end }} @@ -241,7 +246,7 @@ systemctl daemon-reload systemctl restart setup.service systemctl restart kubelet.service systemctl restart kubelet-healthcheck.service - ` + ` bootstrapZypperBinContentTemplate = `#!/bin/bash set -xeuo pipefail From 889e10294560ab19ad42cfdbd79bd80182a2655b Mon Sep 17 00:00:00 2001 From: Oliver <10700296+ol-iver@users.noreply.github.com> Date: Tue, 8 Feb 2022 10:30:52 +0100 Subject: [PATCH 077/489] Introduce Hetzner placement groups into machinedeployments (#1184) * Introduce Hetzner placement groups into machinedeployments Signed-off-by: ol-iver * Clean up orphan placement groups Signed-off-by: ol-iver * Avoid missleading machine-uid label in placement group Signed-off-by: ol-iver * Fix linting issue / adjust naming of variables Signed-off-by: ol-iver * Reduce complexity of Create() function Signed-off-by: ol-iver --- examples/hetzner-machinedeployment.yaml | 2 + go.mod | 96 +++++++++++- go.sum | 16 +- .../provider/hetzner/provider.go | 146 ++++++++++++------ .../provider/hetzner/types/types.go | 17 +- 5 files changed, 209 insertions(+), 68 deletions(-) diff --git a/examples/hetzner-machinedeployment.yaml b/examples/hetzner-machinedeployment.yaml index 53c85b533..6ca5784dc 100644 --- a/examples/hetzner-machinedeployment.yaml +++ b/examples/hetzner-machinedeployment.yaml @@ -48,6 +48,8 @@ spec: datacenter: "" location: "fsn1" image: "ubuntu-20.04" + # Optional: placement group prefix + placementGroupPrefix: "<< YOUR_PLACEMENT_GROUP_PREFIX >>" # Optional: network IDs or names networks: - "<< YOUR_NETWORK >>" diff --git a/go.mod b/go.mod index f7f070b3e..fd23411b7 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/kubermatic/machine-controller -go 1.13 +go 1.17 require ( cloud.google.com/go v0.73.0 @@ -23,7 +23,7 @@ require ( github.com/google/uuid v1.1.2 github.com/gophercloud/gophercloud v0.24.0 github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 - github.com/hetznercloud/hcloud-go v1.25.0 + github.com/hetznercloud/hcloud-go v1.33.1 github.com/linode/linodego v0.24.0 github.com/packethost/packngo v0.5.1 github.com/patrickmn/go-cache v2.1.0+incompatible @@ -56,6 +56,98 @@ require ( sigs.k8s.io/yaml v1.2.0 ) +require ( + github.com/Azure/go-autorest v14.2.0+incompatible // indirect + github.com/Azure/go-autorest/autorest v0.11.18 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect + github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect + github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect + github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect + github.com/Azure/go-autorest/logger v0.2.1 // indirect + github.com/Azure/go-autorest/tracing v0.6.0 // indirect + github.com/Masterminds/goutils v1.1.1 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect + github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd // indirect + github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect + github.com/coreos/ignition v0.35.0 // indirect + github.com/dimchansky/utfbom v1.1.0 // indirect + github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/emicklei/go-restful v2.11.2+incompatible // indirect + github.com/evanphx/json-patch v4.11.0+incompatible // indirect + github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/go-logr/logr v0.4.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.5 // indirect + github.com/go-openapi/spec v0.19.15 // indirect + github.com/go-openapi/swag v0.19.15 // indirect + github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.6 // indirect + github.com/google/go-querystring v1.0.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.0.5 // indirect + github.com/googleapis/gnostic v0.5.5 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/huandu/xstrings v1.3.2 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.11 // indirect + github.com/jstemmer/go-junit-report v0.9.1 // indirect + github.com/kr/pretty v0.2.1 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/mitchellh/copystructure v1.0.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/reflectwalk v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/opencontainers/go-digest v1.0.0-rc1 // indirect + github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect + github.com/shopspring/decimal v1.2.0 // indirect + github.com/smartystreets/assertions v1.2.0 // indirect + github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50 // indirect + go.opencensus.io v0.22.5 // indirect + go4.org v0.0.0-20201209231011-d4a079459e60 // indirect + golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect + golang.org/x/mod v0.4.2 // indirect + golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 // indirect + golang.org/x/term v0.0.0-20210503060354-a79de5458b56 // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect + golang.org/x/tools v0.1.2 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect + google.golang.org/protobuf v1.26.0 // indirect + gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.57.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/component-base v0.22.2 // indirect + k8s.io/klog/v2 v2.9.0 // indirect + k8s.io/kube-openapi v0.0.0-20210527164424-3c818078ee3d // indirect + kubevirt.io/controller-lifecycle-operator-sdk v0.2.1 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect +) + replace ( github.com/packethost/packngo => github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888 diff --git a/go.sum b/go.sum index 3d1129134..1cf1d49de 100644 --- a/go.sum +++ b/go.sum @@ -41,7 +41,6 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= code.cloudfoundry.org/go-pubsub v0.0.0-20180503211407-becd51dc37cb/go.mod h1:2mohpzdn59JWHT85lXjjglNpGLF51tk6hHqfxpc0utk= contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= @@ -207,7 +206,6 @@ github.com/aws/aws-sdk-go v1.36.2/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zK github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/bazelbuild/buildtools v0.0.0-20190917191645-69366ca98f89/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -321,7 +319,6 @@ github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -643,10 +640,8 @@ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/licenseclassifier v0.0.0-20190926221455-842c0d70d702/go.mod h1:qsqn2hxC+vURpyBRygGUuinTO42MFRLcsmQ/P8v94+M= -github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -758,8 +753,9 @@ github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKe github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 h1:GT4RsKmHh1uZyhmTkWJTDALRjSHYQp6FRKrotf0zhAs= github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= github.com/hetznercloud/hcloud-go v1.23.1/go.mod h1:xng8lbDUg+xM1dgc0yGHX5EeqbwIq7UYlMWMTx3SQVg= -github.com/hetznercloud/hcloud-go v1.25.0 h1:QAaFKtGKWRxjwjKJWBGMxGYUxVEQmIkb35j/WXrsazY= github.com/hetznercloud/hcloud-go v1.25.0/go.mod h1:2C5uMtBiMoFr3m7lBFPf7wXTdh33CevmZpQIIDPGYJI= +github.com/hetznercloud/hcloud-go v1.33.1 h1:W1HdO2bRLTKU4WsyqAasDSpt54fYO4WNckWYfH5AuCQ= +github.com/hetznercloud/hcloud-go v1.33.1/go.mod h1:XX/TQub3ge0yWR2yHWmnDVIrB+MQbda1pHxkUmDlUME= github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= @@ -1142,7 +1138,6 @@ github.com/rcrowley/go-metrics v0.0.0-20190706150252-9beb055b7962/go.mod h1:bCqn github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= -github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -1187,7 +1182,6 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= @@ -1968,7 +1962,6 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.4/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -k8c.io/kubermatic/v2 v2.16.2 h1:tjPfI+VV51pggXCvcDL/qG1r7KHDBQPSPYngPxpRtp8= k8c.io/kubermatic/v2 v2.16.2/go.mod h1:NdW+2mq4ynRtfZs9yPnvcnFWQpzmM7ngntW6GeuQicU= k8c.io/operating-system-manager v0.1.0/go.mod h1:ULyZQO1irKjsQTNjIdrHld7SZ+joHjmPnOEs5Db8G8M= k8c.io/operating-system-manager v0.3.0/go.mod h1:ME5GOCNUrHG+57igEKP1JCJKVHynaLfodT8bRiYH3MY= @@ -1982,7 +1975,6 @@ k8s.io/api v0.0.0-20190918155943-95b840bb6a1f/go.mod h1:uWuOHnjmNrtQomJrvEBg0c0H k8s.io/api v0.0.0-20190918195907-bd6ac527cfd2/go.mod h1:AOxZTnaXR/xiarlQL0JUfwQPxjmKDvVYoRp58cA7lUo= k8s.io/api v0.16.4/go.mod h1:AtzMnsR45tccQss5q8RnF+W8L81DH6XwXwo/joEx9u0= k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= -k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= k8s.io/api v0.17.2/go.mod h1:BS9fjjLc4CMuqfSO8vgbHPKMt5+SF0ET6u/RVDihTo4= k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= @@ -2021,7 +2013,6 @@ k8s.io/apimachinery v0.0.0-20190817020851-f2f3a405f61d/go.mod h1:3jediapYqJ2w1BF k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4= k8s.io/apimachinery v0.16.4/go.mod h1:llRdnznGEAqC3DcNm6yEj472xaFVfLM7hnYofMb12tQ= k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apimachinery v0.17.1/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apimachinery v0.17.2/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= @@ -2041,7 +2032,6 @@ k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8Nld k8s.io/apiserver v0.0.0-20190918200908-1e17798da8c1/go.mod h1:4FuDU+iKPjdsdQSN3GsEKZLB/feQsj1y9dhhBDVV2Ns= k8s.io/apiserver v0.16.4/go.mod h1:kbLJOak655g6W7C+muqu1F76u9wnEycfKMqbVaXIdAc= k8s.io/apiserver v0.17.0/go.mod h1:ABM+9x/prjINN6iiffRVNCBR2Wk7uY4z+EtEGZD48cg= -k8s.io/apiserver v0.17.0/go.mod h1:ABM+9x/prjINN6iiffRVNCBR2Wk7uY4z+EtEGZD48cg= k8s.io/apiserver v0.17.2/go.mod h1:lBmw/TtQdtxvrTk0e2cgtOxHizXI+d0mmGQURIHQZlo= k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw= k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw= @@ -2081,7 +2071,6 @@ k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3 k8s.io/component-base v0.0.0-20190918200425-ed2f0867c778/go.mod h1:DFWQCXgXVLiWtzFaS17KxHdlUeUymP7FLxZSkmL9/jU= k8s.io/component-base v0.16.4/go.mod h1:GYQ+4hlkEwdlpAp59Ztc4gYuFhdoZqiAJD1unYDJ3FM= k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= -k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= k8s.io/component-base v0.17.2/go.mod h1:zMPW3g5aH7cHJpKYQ/ZsGMcgbsA/VyhEugF3QT1awLs= k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c= k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM= @@ -2140,7 +2129,6 @@ k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3 k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 0f80496e4..5dd548b03 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -23,6 +23,7 @@ import ( "fmt" "net/http" "strconv" + "strings" "github.com/hetznercloud/hcloud-go/hcloud" @@ -38,6 +39,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" "k8s.io/klog" ) @@ -55,14 +57,15 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes } type Config struct { - Token string - ServerType string - Datacenter string - Image string - Location string - Networks []string - Firewalls []string - Labels map[string]string + Token string + ServerType string + Datacenter string + Image string + Location string + PlacementGroupPrefix string + Networks []string + Firewalls []string + Labels map[string]string } func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { @@ -124,6 +127,11 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return nil, nil, err } + c.PlacementGroupPrefix, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.PlacementGroupPrefix) + if err != nil { + return nil, nil, err + } + for _, network := range rawConfig.Networks { networkValue, err := p.configVarResolver.GetConfigVarStringValue(network) if err != nil { @@ -144,6 +152,36 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return &c, &pconfig, err } +func (p *provider) getServerPlacementGroup(ctx context.Context, client *hcloud.Client, c *Config) (*hcloud.PlacementGroup, error) { + placementGroups, _, err := client.PlacementGroup.List(ctx, hcloud.PlacementGroupListOpts{Type: hcloud.PlacementGroupTypeSpread}) + if err != nil { + return nil, hzErrorToTerminalError(err, "failed to get placement groups of type spread") + } + for _, pg := range placementGroups { + if !strings.HasPrefix(pg.Name, c.PlacementGroupPrefix) { + continue + } + if len(pg.Servers) < 10 { + return pg, nil + } + } + pgLabels := map[string]string{} + for k, v := range c.Labels { + if k != machineUIDLabelKey { + pgLabels[k] = v + } + } + createdPg, _, err := client.PlacementGroup.Create(ctx, hcloud.PlacementGroupCreateOpts{ + Name: fmt.Sprintf("%s-%s", c.PlacementGroupPrefix, rand.SafeEncodeString(rand.String(5))), + Labels: pgLabels, + Type: hcloud.PlacementGroupTypeSpread, + }) + if err != nil { + return nil, hzErrorToTerminalError(err, "failed to create placement group") + } + return createdPg.PlacementGroup, nil +} + func (p *provider) Validate(spec v1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { @@ -184,23 +222,19 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { } } - if len(c.Networks) != 0 { - for _, network := range c.Networks { - if _, _, err = client.Network.Get(ctx, network); err != nil { - return fmt.Errorf("failed to get network %q: %v", network, err) - } + for _, network := range c.Networks { + if _, _, err = client.Network.Get(ctx, network); err != nil { + return fmt.Errorf("failed to get network %q: %v", network, err) } } - if len(c.Firewalls) != 0 { - for _, firewall := range c.Firewalls { - f, _, err := client.Firewall.Get(ctx, firewall) - if err != nil { - return fmt.Errorf("failed to get firewall %q: %v", firewall, err) - } - if f == nil { - return fmt.Errorf("firewall %q does not exist", firewall) - } + for _, firewall := range c.Firewalls { + f, _, err := client.Firewall.Get(ctx, firewall) + if err != nil { + return fmt.Errorf("failed to get firewall %q: %v", firewall, err) + } + if f == nil { + return fmt.Errorf("firewall %q does not exist", firewall) } } @@ -267,31 +301,34 @@ func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provi serverCreateOpts.Location = location } - if len(c.Networks) != 0 { - serverCreateOpts.Networks = []*hcloud.Network{} - for _, network := range c.Networks { - n, _, err := client.Network.Get(ctx, network) - if err != nil { - return nil, hzErrorToTerminalError(err, "failed to get network") - } - if n == nil { - return nil, fmt.Errorf("network %q does not exist", network) - } - serverCreateOpts.Networks = append(serverCreateOpts.Networks, n) + if c.PlacementGroupPrefix != "" { + selectedPg, err := p.getServerPlacementGroup(ctx, client, c) + if err != nil { + return nil, err } + serverCreateOpts.PlacementGroup = selectedPg } - if len(c.Firewalls) != 0 { - for _, firewall := range c.Firewalls { - n, _, err := client.Firewall.Get(ctx, firewall) - if err != nil { - return nil, hzErrorToTerminalError(err, "failed to get firewall") - } - if n == nil { - return nil, fmt.Errorf("firewall %q does not exist", firewall) - } - serverCreateOpts.Firewalls = append(serverCreateOpts.Firewalls, &hcloud.ServerCreateFirewall{Firewall: *n}) + for _, network := range c.Networks { + n, _, err := client.Network.Get(ctx, network) + if err != nil { + return nil, hzErrorToTerminalError(err, "failed to get network") + } + if n == nil { + return nil, fmt.Errorf("network %q does not exist", network) } + serverCreateOpts.Networks = append(serverCreateOpts.Networks, n) + } + + for _, firewall := range c.Firewalls { + n, _, err := client.Firewall.Get(ctx, firewall) + if err != nil { + return nil, hzErrorToTerminalError(err, "failed to get firewall") + } + if n == nil { + return nil, fmt.Errorf("firewall %q does not exist", firewall) + } + serverCreateOpts.Firewalls = append(serverCreateOpts.Firewalls, &hcloud.ServerCreateFirewall{Firewall: *n}) } image, _, err := client.Image.Get(ctx, c.Image) @@ -368,14 +405,35 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.P ctx := context.TODO() client := getClient(c.Token) + hzServer := instance.(*hetznerServer).server - res, err := client.Server.Delete(ctx, instance.(*hetznerServer).server) + res, err := client.Server.Delete(ctx, hzServer) if err != nil { return false, hzErrorToTerminalError(err, "failed to delete the server") } if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusNotFound { return false, fmt.Errorf("invalid status code returned. expected=%d got=%d", http.StatusOK, res.StatusCode) } + + if hzServer.PlacementGroup != nil { + pgHzServer, _, err := client.PlacementGroup.Get(ctx, hzServer.PlacementGroup.Name) + if err != nil { + return false, hzErrorToTerminalError(err, "failed to get placement group") + } + count := 0 + for _, s := range pgHzServer.Servers { + if s != hzServer.ID { + count++ + } + } + if count == 0 { + _, err := client.PlacementGroup.Delete(ctx, pgHzServer) + if err != nil { + return false, hzErrorToTerminalError(err, "failed to delete empty placement group") + } + } + } + return false, nil } diff --git a/pkg/cloudprovider/provider/hetzner/types/types.go b/pkg/cloudprovider/provider/hetzner/types/types.go index 5756e1548..b972fc768 100644 --- a/pkg/cloudprovider/provider/hetzner/types/types.go +++ b/pkg/cloudprovider/provider/hetzner/types/types.go @@ -21,12 +21,13 @@ import ( ) type RawConfig struct { - Token providerconfigtypes.ConfigVarString `json:"token,omitempty"` - ServerType providerconfigtypes.ConfigVarString `json:"serverType"` - Datacenter providerconfigtypes.ConfigVarString `json:"datacenter"` - Image providerconfigtypes.ConfigVarString `json:"image"` - Location providerconfigtypes.ConfigVarString `json:"location"` - Networks []providerconfigtypes.ConfigVarString `json:"networks"` - Firewalls []providerconfigtypes.ConfigVarString `json:"firewalls"` - Labels map[string]string `json:"labels,omitempty"` + Token providerconfigtypes.ConfigVarString `json:"token,omitempty"` + ServerType providerconfigtypes.ConfigVarString `json:"serverType"` + Datacenter providerconfigtypes.ConfigVarString `json:"datacenter"` + Image providerconfigtypes.ConfigVarString `json:"image"` + Location providerconfigtypes.ConfigVarString `json:"location"` + PlacementGroupPrefix providerconfigtypes.ConfigVarString `json:"placementGroupPrefix"` + Networks []providerconfigtypes.ConfigVarString `json:"networks"` + Firewalls []providerconfigtypes.ConfigVarString `json:"firewalls"` + Labels map[string]string `json:"labels,omitempty"` } From f26e531d41e8bc13e5b3620d297349eadd0dc377 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Tue, 8 Feb 2022 11:25:58 +0100 Subject: [PATCH 078/489] Add E2E test case for Nutanix cloud provider (#1145) * Add TestNutanixProvisioningE2E test case Signed-off-by: Marvin Beckers * Add initial job definition to Prow Signed-off-by: Marvin Beckers * Use squid preset and pull proxy from NUTANIX_E2E_PROXY_URL Signed-off-by: Marvin Beckers * Start e2e testing with Ubuntu and CentOS only Signed-off-by: Marvin Beckers * Add NUTANIX_ALLOW_INSECURE Signed-off-by: Marvin Beckers * Use environment variables that are passed by preset Signed-off-by: Marvin Beckers * Set up port forwarding on VM if proxy host is defined for Nutanix Signed-off-by: Marvin Beckers * Add network zone as variable Signed-off-by: Marvin Beckers * Fix private address usage Signed-off-by: Marvin Beckers * Fix username ref and SSH port-forward kill Signed-off-by: Marvin Beckers * add prefix to port Signed-off-by: Marvin Beckers * Use smaller port Signed-off-by: Marvin Beckers * force ipv4 on ssh port-forwarding Signed-off-by: Marvin Beckers * Use correct flag for SSH tunnel/port-forward Signed-off-by: Marvin Beckers * correctly pass IP to proxy URL Signed-off-by: Marvin Beckers * hardcode allowInsecure in manifest Signed-off-by: Marvin Beckers * Use IP instead of hostname Signed-off-by: Marvin Beckers * Add GatewayPorts config to sshd Signed-off-by: Marvin Beckers * keep SSH tunnel for proxy alive Signed-off-by: Marvin Beckers * Exclude migrateUID test case Signed-off-by: Marvin Beckers * fix yamllint issue Signed-off-by: Marvin Beckers * Address review comments Signed-off-by: Marvin Beckers --- .prow.yaml | 24 ++++++++++ hack/ci-e2e-test.sh | 8 ++++ test/e2e/provisioning/all_e2e_test.go | 43 ++++++++++++++++++ test/e2e/provisioning/helper.go | 31 +++++++++++++ .../testdata/machinedeployment-nutanix.yaml | 44 +++++++++++++++++++ test/tools/integration/hetzner.tf | 17 +++++++ .../integration/master_install_script.sh | 4 +- test/tools/integration/output.tf | 4 ++ test/tools/integration/provision_master.sh | 9 ++++ test/tools/integration/variables.tf | 6 +++ 10 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml diff --git a/.prow.yaml b/.prow.yaml index 3c2bf5822..d4c745208 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -565,6 +565,30 @@ presubmits: memory: 1Gi cpu: 500m + - name: pull-machine-controller-e2e-nutanix + optional: true + always_run: false + run_if_changed: "(pkg/cloudprovider/provider/nutanix/|pkg/userdata/|test/e2e/provisioning/)" + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-nutanix: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.17.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestNutanixProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + - name: pull-machine-controller-e2e-anexia always_run: false decorate: true diff --git a/hack/ci-e2e-test.sh b/hack/ci-e2e-test.sh index fff145bdd..3a5f00a44 100755 --- a/hack/ci-e2e-test.sh +++ b/hack/ci-e2e-test.sh @@ -36,6 +36,9 @@ function cleanup { echo "Sleeping for $try seconds" sleep ${try}s done + + # Kill background port forward if it's there + pkill ssh || true } trap cleanup EXIT @@ -95,6 +98,11 @@ export E2E_SSH_PUBKEY="$(cat ~/.ssh/id_rsa.pub)" ./test/tools/integration/provision_master.sh echo "Running e2e tests..." +if [[ ! -z "${NUTANIX_E2E_PROXY_HOST:-}" ]]; then + vm_priv_addr=$(cat ./priv_addr) + export NUTANIX_E2E_PROXY_URL="http://${NUTANIX_E2E_PROXY_USERNAME}:${NUTANIX_E2E_PROXY_PASSWORD}@${vm_priv_addr}:${NUTANIX_E2E_PROXY_PORT}/" +fi + export KUBECONFIG=$GOPATH/src/github.com/kubermatic/machine-controller/.kubeconfig EXTRA_ARGS="" if [[ $# -gt 0 ]]; then diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index cb38e133a..26923e446 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -75,6 +75,7 @@ const ( kubevirtManifest = "./testdata/machinedeployment-kubevirt.yaml" alibabaManifest = "./testdata/machinedeployment-alibaba.yaml" anexiaManifest = "./testdata/machinedeployment-anexia.yaml" + nutanixManifest = "./testdata/machinedeployment-nutanix.yaml" ) var testRunIdentifier = flag.String("identifier", "local", "The unique identifier for this test run") @@ -880,6 +881,48 @@ func TestScalewayProvisioningE2E(t *testing.T) { runScenarios(t, selector, params, ScalewayManifest, fmt.Sprintf("scw-%s", *testRunIdentifier)) } +func getNutanixTestParams(t *testing.T) []string { + // test data + password := os.Getenv("NUTANIX_E2E_PASSWORD") + username := os.Getenv("NUTANIX_E2E_USERNAME") + cluster := os.Getenv("NUTANIX_E2E_CLUSTER_NAME") + project := os.Getenv("NUTANIX_E2E_PROJECT_NAME") + subnet := os.Getenv("NUTANIX_E2E_SUBNET_NAME") + endpoint := os.Getenv("NUTANIX_E2E_ENDPOINT") + + if password == "" || username == "" || endpoint == "" || cluster == "" || project == "" || subnet == "" { + t.Fatal("unable to run the test suite, NUTANIX_E2E_PASSWORD, NUTANIX_E2E_USERNAME, NUTANIX_E2E_CLUSTER_NAME, " + + "NUTANIX_E2E_ENDPOINT, NUTANIX_E2E_PROJECT_NAME or NUTANIX_E2E_SUBNET_NAME environment variables cannot be empty") + } + + // a proxy URL will be passed in our e2e test environment so + // a HTTP proxy can be used to access the Nutanix API in a different + // network segment. + proxyURL := os.Getenv("NUTANIX_E2E_PROXY_URL") + + // set up parameters + params := []string{fmt.Sprintf("<< NUTANIX_PASSWORD >>=%s", password), + fmt.Sprintf("<< NUTANIX_USERNAME >>=%s", username), + fmt.Sprintf("<< NUTANIX_ENDPOINT >>=%s", endpoint), + fmt.Sprintf("<< NUTANIX_CLUSTER >>=%s", cluster), + fmt.Sprintf("<< NUTANIX_PROJECT >>=%s", project), + fmt.Sprintf("<< NUTANIX_SUBNET >>=%s", subnet), + fmt.Sprintf("<< NUTANIX_PROXY_URL >>=%s", proxyURL), + } + return params +} + +// TestNutanixProvisioningE2E tests provisioning on Nutanix as cloud provider +func TestNutanixProvisioningE2E(t *testing.T) { + t.Parallel() + + // exclude migrateUID test case because it's a no-op for Nutanix and runs from a different + // location, thus possibly blocking access a HTTP proxy if it is configured + selector := And(OsSelector("ubuntu", "centos"), Not(NameSelector("migrateUID"))) + params := getNutanixTestParams(t) + runScenarios(t, selector, params, nutanixManifest, fmt.Sprintf("nx-%s", *testRunIdentifier)) +} + // TestUbuntuProvisioningWithUpgradeE2E will create an instance from an old Ubuntu 1604 // image and upgrade it prior to joining the cluster func TestUbuntuProvisioningWithUpgradeE2E(t *testing.T) { diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 44ee994d8..8583dd074 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -109,6 +109,37 @@ func (os *osSelector) Match(testCase scenario) bool { return false } +// And is used to match against two selectors. +func And(s1 Selector, s2 Selector) Selector { + return &and{s1, s2} +} + +type and struct { + s1 Selector + s2 Selector +} + +var _ Selector = &and{} + +func (a *and) Match(tc scenario) bool { + return a.s1.Match(tc) && a.s2.Match(tc) +} + +// NameSelector is used to match against a test case name +func NameSelector(tcName string) Selector { + return &name{tcName} +} + +type name struct { + name string +} + +var _ Selector = &name{} + +func (n *name) Match(tc scenario) bool { + return tc.name == n.name +} + func runScenarios(st *testing.T, selector Selector, testParams []string, manifestPath string, cloudProvider string) { for _, testCase := range scenarios { if selector != nil && !selector.Match(testCase) { diff --git a/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml b/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml new file mode 100644 index 000000000..ddc753588 --- /dev/null +++ b/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml @@ -0,0 +1,44 @@ +apiVersion: "cluster.k8s.io/v1alpha1" +kind: MachineDeployment +metadata: + name: << MACHINE_NAME >> + namespace: kube-system +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + name: << MACHINE_NAME >> + template: + metadata: + labels: + name: << MACHINE_NAME >> + spec: + providerSpec: + value: + sshPublicKeys: + - "<< YOUR_PUBLIC_KEY >>" + cloudProvider: "nutanix" + cloudProviderSpec: + username: '<< NUTANIX_USERNAME >>' + password: '<< NUTANIX_PASSWORD >>' + endpoint: '<< NUTANIX_ENDPOINT >>' + proxyURL: '<< NUTANIX_PROXY_URL >>' + allowInsecure: true + clusterName: '<< NUTANIX_CLUSTER >>' + projectName: '<< NUTANIX_PROJECT >>' + subnetName: '<< NUTANIX_SUBNET >>' + imageName: 'machine-controller-e2e-<< OS_NAME >>' + cpus: 2 + memoryMB: 2048 + diskSize: 20 + operatingSystem: "<< OS_NAME >>" + operatingSystemSpec: + distUpgradeOnBoot: false + disableAutoUpdate: true + versions: + kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/tools/integration/hetzner.tf b/test/tools/integration/hetzner.tf index badaf2b39..c87525e8d 100644 --- a/test/tools/integration/hetzner.tf +++ b/test/tools/integration/hetzner.tf @@ -7,6 +7,11 @@ resource "hcloud_ssh_key" "default" { public_key = var.hcloud_sshkey_content } +resource "hcloud_network" "net" { + name = var.hcloud_test_server_name + ip_range = "192.168.0.0/16" +} + resource "hcloud_server" "machine-controller-test" { name = var.hcloud_test_server_name image = "ubuntu-20.04" @@ -14,3 +19,15 @@ resource "hcloud_server" "machine-controller-test" { ssh_keys = [hcloud_ssh_key.default.id] location = "nbg1" } + +resource "hcloud_network_subnet" "machine_controller" { + network_id = hcloud_network.net.id + type = "server" + network_zone = var.hcloud_network_zone + ip_range = "192.168.0.0/16" +} + +resource "hcloud_server_network" "machine_controller" { + server_id = hcloud_server.machine-controller-test.id + subnet_id = hcloud_network_subnet.machine_controller.id +} diff --git a/test/tools/integration/master_install_script.sh b/test/tools/integration/master_install_script.sh index a507bb52b..19ddfe152 100644 --- a/test/tools/integration/master_install_script.sh +++ b/test/tools/integration/master_install_script.sh @@ -19,6 +19,8 @@ set -x K8S_VERSION=1.23.0 echo "$LC_E2E_SSH_PUBKEY" >> .ssh/authorized_keys +echo "GatewayPorts clientspecified" >> /etc/ssh/sshd_config +systemctl restart sshd.service # Hetzner's Ubuntu Bionic comes with swap pre-configured, so we force it off. systemctl mask swap.target @@ -159,4 +161,4 @@ done echo "Error: machine-controller didn't come up within 100 seconds!" echo "Logs:" kubectl logs -n kube-system $(kubectl get pods -n kube-system|egrep '^machine-controller'|awk '{ print $1}') -exit 1 \ No newline at end of file +exit 1 diff --git a/test/tools/integration/output.tf b/test/tools/integration/output.tf index 4d1863ff9..71a9b3175 100644 --- a/test/tools/integration/output.tf +++ b/test/tools/integration/output.tf @@ -1,3 +1,7 @@ output "ip" { value = hcloud_server.machine-controller-test.ipv4_address } + +output "private_ip" { + value = hcloud_server_network.machine_controller.ip +} diff --git a/test/tools/integration/provision_master.sh b/test/tools/integration/provision_master.sh index d32e038dd..6df0e6279 100755 --- a/test/tools/integration/provision_master.sh +++ b/test/tools/integration/provision_master.sh @@ -25,6 +25,7 @@ MC_ROOT="$(cd ./../../.. && pwd -P)" # 'AcceptEnv LANG LC_*'. export LC_DEPLOY_MACHINE="${1:-}" export LC_ADDR=$(terraform output -json|jq '.ip.value' -r) +export LC_PRIV_ADDR=$(terraform output -json|jq '.private_ip.value' -r) export LC_E2E_SSH_PUBKEY="${E2E_SSH_PUBKEY:-$(cat ~/.ssh/id_rsa.pub)}" export LC_JOB_NAME="${JOB_NAME:-}" @@ -59,3 +60,11 @@ scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ if [[ $? == 0 ]]; then break; fi sleep ${try}s done + +# set up SSH port-forwarding if necessary +if [[ ! -z "${NUTANIX_E2E_PROXY_HOST:-}" ]]; then + echo -n "${LC_PRIV_ADDR}" > ${MC_ROOT}/./priv_addr + + ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ServerAliveInterval=5 -fNT -R ${LC_PRIV_ADDR}:${NUTANIX_E2E_PROXY_PORT}:${NUTANIX_E2E_PROXY_HOST}:${NUTANIX_E2E_PROXY_PORT} root@${LC_ADDR} +fi + diff --git a/test/tools/integration/variables.tf b/test/tools/integration/variables.tf index 111646de9..a78472096 100644 --- a/test/tools/integration/variables.tf +++ b/test/tools/integration/variables.tf @@ -6,3 +6,9 @@ variable "hcloud_sshkey_name" { } variable "hcloud_test_server_name" {} + +variable "hcloud_network_zone" { + default = "eu-central" + description = "network zone to use for private network" + type = string +} From 0f4cae862f3f8446b41f3bc623fab2aa1cdcc01b Mon Sep 17 00:00:00 2001 From: Matthias Osthues Date: Thu, 10 Feb 2022 10:27:20 +0100 Subject: [PATCH 079/489] set kubelet ContainerLogMaxSize conf even if docker is used to be consistent and not apply default values. (#1185) --- pkg/userdata/helper/kubelet.go | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index d5d3e603c..b74c5ab84 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -242,19 +242,16 @@ func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGate } } - // ContainerLogMaxSize and ContainerLogMaxFiles have no effect if container runtime is docker i.e. not remote - if containerRuntime != "docker" { - if containerLogMaxSize, ok := kubeletConfigs[common.ContainerLogMaxSizeKubeletConfig]; ok { - cfg.ContainerLogMaxSize = containerLogMaxSize - } - if containerLogMaxFiles, ok := kubeletConfigs[common.ContainerLogMaxFilesKubeletConfig]; ok { - maxFiles, err := strconv.Atoi(containerLogMaxFiles) - if err != nil || maxFiles < 0 { - // Instead of breaking the workflow, just print a warning and skip the configuration - klog.Warningf("Skipping invalid ContainerLogMaxSize value %v for Kubelet configuration", containerLogMaxFiles) - } else { - cfg.ContainerLogMaxFiles = pointer.Int32Ptr(int32(maxFiles)) - } + if containerLogMaxSize, ok := kubeletConfigs[common.ContainerLogMaxSizeKubeletConfig]; ok { + cfg.ContainerLogMaxSize = containerLogMaxSize + } + if containerLogMaxFiles, ok := kubeletConfigs[common.ContainerLogMaxFilesKubeletConfig]; ok { + maxFiles, err := strconv.Atoi(containerLogMaxFiles) + if err != nil || maxFiles < 0 { + // Instead of breaking the workflow, just print a warning and skip the configuration + klog.Warningf("Skipping invalid ContainerLogMaxSize value %v for Kubelet configuration", containerLogMaxFiles) + } else { + cfg.ContainerLogMaxFiles = pointer.Int32Ptr(int32(maxFiles)) } } From 1aa138694d3b781bb40bdd1d522e8ba2a4c80cb6 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Thu, 10 Feb 2022 15:18:01 +0100 Subject: [PATCH 080/489] enable vSphere tests (#1180) * enable vSphere tests Signed-off-by: Moath Qasim # Conflicts: # go.sum * refactor vSphere datastore cluster Signed-off-by: Moath Qasim * refactor vSphere tests Signed-off-by: Moath Qasim * enable vsphere test Signed-off-by: Moath Qasim * debug vsphere datastore test Signed-off-by: Moath Qasim * debug vsphere datastore test Signed-off-by: Moath Qasim --- .prow.yaml | 2 +- test/e2e/provisioning/all_e2e_test.go | 2 +- .../provisioning/testdata/machinedeployment-vsphere.yaml | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.prow.yaml b/.prow.yaml index d4c745208..5c5c58db2 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -543,7 +543,7 @@ presubmits: cpu: 500m - name: pull-machine-controller-e2e-vsphere - always_run: false + always_run: true decorate: true error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 26923e446..caa4da840 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -821,7 +821,7 @@ func TestVsphereProvisioningE2E(t *testing.T) { func TestVsphereDatastoreClusterProvisioningE2E(t *testing.T) { t.Parallel() - selector := OsSelector("ubuntu", "centos") + selector := OsSelector("ubuntu", "centos", "rhel", "flatcar") params := getVSphereTestParams(t) runScenarios(t, selector, params, VSPhereDSCManifest, fmt.Sprintf("vs-dsc-%s", *testRunIdentifier)) diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml index f45f4ff0d..084f59c4b 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml @@ -31,10 +31,10 @@ spec: folder: '/dc-1/vm/e2e-tests' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically - cluster: '<< VSPHERE_CLUSTER >>' - datastore: exsi-nas + cluster: 'cl-1' + datastore: HS-FreeNAS cpus: 2 - MemoryMB: 2048 + MemoryMB: 4096 diskSizeGB: << DISK_SIZE >> operatingSystem: "<< OS_NAME >>" operatingSystemSpec: From a8c2066f552085e95dacf0630f295a2841e1b214 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Sat, 12 Feb 2022 23:58:56 +0100 Subject: [PATCH 081/489] Remove cluster field from vSphere provider (#1187) * remove cluster field from vsphere provider Signed-off-by: Moath Qasim * revert back the cluster id in the cloud-config Signed-off-by: Moath Qasim --- .../vsphere-datastore-cluster-machinedeployment.yaml | 1 - examples/vsphere-machinedeployment.yaml | 1 - pkg/cloudprovider/provider/vsphere/provider.go | 12 ------------ pkg/cloudprovider/provider/vsphere/provider_test.go | 1 - pkg/cloudprovider/provider/vsphere/types/types.go | 1 - test/e2e/provisioning/all_e2e_test.go | 6 ++---- .../machinedeployment-vsphere-datastore-cluster.yaml | 1 - .../machinedeployment-vsphere-resource-pool.yaml | 1 - .../machinedeployment-vsphere-static-ip.yaml | 1 - .../testdata/machinedeployment-vsphere.yaml | 1 - 10 files changed, 2 insertions(+), 24 deletions(-) diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml index 18bc56084..6d2302fe8 100644 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ b/examples/vsphere-datastore-cluster-machinedeployment.yaml @@ -54,7 +54,6 @@ spec: vmNetName: network1 # Optional folder: folder1 - cluster: cluster1 datastoreCluster: datastorecluster1 # Can also be set via the env var 'VSPHERE_ALLOW_INSECURE' on the machine-controller allowInsecure: true diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index 740fe2515..f0442d72c 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -54,7 +54,6 @@ spec: vmNetName: network1 # Optional folder: folder1 - cluster: cluster1 datastore: datastore1 # Can also be set via the env var 'VSPHERE_ALLOW_INSECURE' on the machine-controller allowInsecure: true diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 0c33a80c9..56b5b1e15 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -64,7 +64,6 @@ type Config struct { Password string VSphereURL string Datacenter string - Cluster string Folder string ResourcePool string Datastore string @@ -160,11 +159,6 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return nil, nil, nil, err } - c.Cluster, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Cluster) - if err != nil { - return nil, nil, nil, err - } - c.Folder, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Folder) if err != nil { return nil, nil, nil, err @@ -225,10 +219,6 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { return fmt.Errorf("one between datastore and datastore cluster should be specified: %v", err) } - if _, err := session.Finder.ClusterComputeResource(ctx, config.Cluster); err != nil { - return fmt.Errorf("failed to get cluster: %s: %v", config.Cluster, err) - } - if _, err := session.Finder.Folder(ctx, config.Folder); err != nil { return fmt.Errorf("failed to get folder %q: %v", config.Folder, err) } @@ -546,7 +536,6 @@ func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, nam Password: c.Password, InsecureFlag: c.AllowInsecure, VCenterPort: u.Port(), - ClusterID: c.Cluster, }, Disk: vspheretypes.DiskOpts{ SCSIControllerType: "pvscsi", @@ -582,7 +571,6 @@ func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]s if err == nil { labels["size"] = fmt.Sprintf("%d-cpus-%d-mb", c.CPUs, c.MemoryMB) labels["dc"] = c.Datacenter - labels["cluster"] = c.Cluster } return labels, err diff --git a/pkg/cloudprovider/provider/vsphere/provider_test.go b/pkg/cloudprovider/provider/vsphere/provider_test.go index 34c89a902..4503b2fa6 100644 --- a/pkg/cloudprovider/provider/vsphere/provider_test.go +++ b/pkg/cloudprovider/provider/vsphere/provider_test.go @@ -47,7 +47,6 @@ func (v vsphereProviderSpecConf) rawProviderSpec(t *testing.T) []byte { "cloudProvider": "vsphere", "cloudProviderSpec": { "allowInsecure": false, - "cluster": "DC0_C0", "cpus": 1, "datacenter": "DC0", {{- if .Datastore }} diff --git a/pkg/cloudprovider/provider/vsphere/types/types.go b/pkg/cloudprovider/provider/vsphere/types/types.go index 49d6e6117..62a0701a7 100644 --- a/pkg/cloudprovider/provider/vsphere/types/types.go +++ b/pkg/cloudprovider/provider/vsphere/types/types.go @@ -28,7 +28,6 @@ type RawConfig struct { Password providerconfigtypes.ConfigVarString `json:"password"` VSphereURL providerconfigtypes.ConfigVarString `json:"vsphereURL"` Datacenter providerconfigtypes.ConfigVarString `json:"datacenter"` - Cluster providerconfigtypes.ConfigVarString `json:"cluster"` Folder providerconfigtypes.ConfigVarString `json:"folder"` ResourcePool providerconfigtypes.ConfigVarString `json:"resourcePool"` diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index caa4da840..8b517f3b9 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -788,11 +788,10 @@ func getVSphereTestParams(t *testing.T) []string { // test data vsPassword := os.Getenv("VSPHERE_E2E_PASSWORD") vsUsername := os.Getenv("VSPHERE_E2E_USERNAME") - vsCluster := os.Getenv("VSPHERE_E2E_CLUSTER") vsAddress := os.Getenv("VSPHERE_E2E_ADDRESS") - if vsPassword == "" || vsUsername == "" || vsAddress == "" || vsCluster == "" { - t.Fatal("unable to run the test suite, VSPHERE_E2E_PASSWORD, VSPHERE_E2E_USERNAME, VSPHERE_E2E_CLUSTER " + + if vsPassword == "" || vsUsername == "" || vsAddress == "" { + t.Fatal("unable to run the test suite, VSPHERE_E2E_PASSWORD, VSPHERE_E2E_USERNAME" + "or VSPHERE_E2E_ADDRESS environment variables cannot be empty") } @@ -800,7 +799,6 @@ func getVSphereTestParams(t *testing.T) []string { params := []string{fmt.Sprintf("<< VSPHERE_PASSWORD >>=%s", vsPassword), fmt.Sprintf("<< VSPHERE_USERNAME >>=%s", vsUsername), fmt.Sprintf("<< VSPHERE_ADDRESS >>=%s", vsAddress), - fmt.Sprintf("<< VSPHERE_CLUSTER >>=%s", vsCluster), } return params } diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml index a92caf029..5ea15ba14 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml @@ -31,7 +31,6 @@ spec: folder: '/dc-1/vm/e2e-tests' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically - cluster: '<< VSPHERE_CLUSTER >>' datastoreCluster: 'dsc-1' cpus: 2 MemoryMB: 2048 diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml index 08f239a67..3db382d2f 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml @@ -31,7 +31,6 @@ spec: folder: '/dc-1/vm/e2e-tests' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically - cluster: '<< VSPHERE_CLUSTER >>' datastoreCluster: 'dsc-1' resourcePool: 'e2e-resource-pool' cpus: 2 diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml index d312bf7f5..70b867f20 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml @@ -31,7 +31,6 @@ spec: folder: '/Customer-A/vm/e2e-tests' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically - cluster: '<< VSPHERE_CLUSTER >>' datastore: datastore1 allowInsecure: true cpus: 2 diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml index 084f59c4b..49101efc5 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml @@ -31,7 +31,6 @@ spec: folder: '/dc-1/vm/e2e-tests' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically - cluster: 'cl-1' datastore: HS-FreeNAS cpus: 2 MemoryMB: 4096 From 67e65cfb06df9da86f9e2fe9dfa9cad142c41060 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Mon, 14 Feb 2022 12:58:52 +0200 Subject: [PATCH 082/489] Unmarshal provider configs in strict mode (#1186) * GetConfig in strict mode Signed-off-by: Artiom Diomin * Alias alpha1 to clusterv1alpha1 Signed-off-by: Artiom Diomin * Rename s to provSpec Signed-off-by: Artiom Diomin * Unmarshal providerSpec and cloudprovider.Config strictly Signed-off-by: Artiom Diomin * Fix testdata Signed-off-by: Artiom Diomin --- cmd/machine-controller/main.go | 1 - examples/openstack-machinedeployment.yaml | 1 - .../provider/alibaba/provider.go | 40 ++++++------- .../provider/alibaba/types/types.go | 7 +++ pkg/cloudprovider/provider/anexia/provider.go | 40 ++++++------- .../provider/anexia/types/types.go | 7 +++ pkg/cloudprovider/provider/aws/provider.go | 51 ++++++++-------- pkg/cloudprovider/provider/aws/types/types.go | 7 +++ pkg/cloudprovider/provider/azure/provider.go | 50 ++++++++-------- .../provider/azure/types/types.go | 7 +++ .../provider/baremetal/provider.go | 36 +++++------ .../provider/baremetal/types/types.go | 7 +++ .../provider/digitalocean/provider.go | 37 ++++++------ .../provider/digitalocean/types/types.go | 7 +++ .../provider/equinixmetal/provider.go | 45 +++++++------- .../provider/equinixmetal/types/types.go | 7 +++ pkg/cloudprovider/provider/fake/provider.go | 27 ++++----- pkg/cloudprovider/provider/gce/config.go | 17 +++--- pkg/cloudprovider/provider/gce/provider.go | 22 +++---- pkg/cloudprovider/provider/gce/types/types.go | 9 +++ .../provider/hetzner/provider.go | 36 +++++------ .../provider/hetzner/types/types.go | 7 +++ .../provider/kubevirt/provider.go | 36 +++++------ .../provider/kubevirt/types/types.go | 7 +++ pkg/cloudprovider/provider/linode/provider.go | 33 +++++----- .../provider/linode/types/types.go | 7 +++ .../provider/nutanix/provider.go | 38 ++++++------ .../provider/nutanix/types/types.go | 7 +++ .../provider/openstack/provider.go | 60 +++++++++---------- .../provider/openstack/types/types.go | 7 +++ .../provider/scaleway/provider.go | 49 ++++++++------- .../provider/scaleway/types/types.go | 7 +++ .../provider/vsphere/provider.go | 38 ++++++------ .../provider/vsphere/types/types.go | 7 +++ pkg/jsonutil/strict.go | 29 +++++++++ pkg/providerconfig/types/types.go | 22 ++++--- .../machinedeployment-openstack-upgrade.yml | 2 +- ...edeployment-vsphere-datastore-cluster.yaml | 1 - ...chinedeployment-vsphere-resource-pool.yaml | 2 +- .../machinedeployment-vsphere-static-ip.yaml | 2 +- 40 files changed, 481 insertions(+), 341 deletions(-) create mode 100644 pkg/jsonutil/strict.go diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index ebb821e5b..0fa532d20 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -42,7 +42,6 @@ import ( machinesv1alpha1 "github.com/kubermatic/machine-controller/pkg/machines/v1alpha1" "github.com/kubermatic/machine-controller/pkg/node" "github.com/kubermatic/machine-controller/pkg/signals" - osmv1alpha1 "k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index 90dc25c4c..cc7d4e23c 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -131,7 +131,6 @@ spec: # UUID of the server group # used to configure affinity or anti-affinity of the VM instaces relative to hypervisor serverGroup: "" - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" securityGroups: - configMapKeyRef: namespace: kube-system diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index 298a0e841..a98be284a 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -18,7 +18,6 @@ package alibaba import ( "encoding/base64" - "encoding/json" "errors" "fmt" "net/http" @@ -27,7 +26,7 @@ import ( "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" alibabatypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/alibaba/types" @@ -104,11 +103,11 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes return &provider{configVarResolver: configVarResolver} } -func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } -func (p *provider) Validate(machineSpec v1alpha1.MachineSpec) error { +func (p *provider) Validate(machineSpec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(machineSpec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %v", err) @@ -149,7 +148,7 @@ func (p *provider) Validate(machineSpec v1alpha1.MachineSpec) error { return nil } -func (p *provider) Get(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -195,11 +194,11 @@ func (p *provider) Get(machine *v1alpha1.Machine, data *cloudprovidertypes.Provi return nil, fmt.Errorf("instance %v is not ready", foundInstance.InstanceId) } -func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } -func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -243,7 +242,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr return nil, fmt.Errorf("failed to create instance at Alibaba cloud: %v", err) } - if err = data.Update(machine, func(updatedMachine *v1alpha1.Machine) { + if err = data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { if !kuberneteshelper.HasFinalizer(updatedMachine, finalizerInstance) { updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerInstance) } @@ -259,7 +258,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr return &alibabaInstance{instance: foundInstance}, nil } -func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { foundInstance, err := p.Get(machine, data) if err != nil { if err == cloudprovidererrors.ErrInstanceNotFound { @@ -292,7 +291,7 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.P return false, nil } -func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) c, _, err := p.getConfig(machine.Spec.ProviderSpec) @@ -304,7 +303,7 @@ func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]s return labels, err } -func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to decode providerconfig: %v", err) @@ -337,16 +336,16 @@ func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { return nil } -func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { return nil } -func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if s.Value == nil { +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { + if provSpec.Value == nil { return nil, nil, errors.New("machine.spec.providerconfig.value is nil") } - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pconfig) + + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, fmt.Errorf("failed to decode providers config: %v", err) } @@ -355,8 +354,8 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } - rawConfig := alibabatypes.RawConfig{} - if err = json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawConfig); err != nil { + rawConfig, err := alibabatypes.GetConfig(*pconfig) + if err != nil { return nil, nil, fmt.Errorf("failed to decode alibaba providers config: %v", err) } @@ -398,7 +397,8 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"diskSize\" field, error = %v", err) } - return &c, &pconfig, err + + return &c, pconfig, err } func getClient(regionID, accessKeyID, accessKeySecret string) (*ecs.Client, error) { @@ -434,7 +434,7 @@ func getInstance(client *ecs.Client, instanceName string, uid string) (*ecs.Inst return &response.Instances.Instance[0], nil } -func (p *provider) getImageIDForOS(machineSpec v1alpha1.MachineSpec, os providerconfigtypes.OperatingSystem) (string, error) { +func (p *provider) getImageIDForOS(machineSpec clusterv1alpha1.MachineSpec, os providerconfigtypes.OperatingSystem) (string, error) { c, _, err := p.getConfig(machineSpec.ProviderSpec) if err != nil { return "", fmt.Errorf("failed to get alibaba client: %v", err) diff --git a/pkg/cloudprovider/provider/alibaba/types/types.go b/pkg/cloudprovider/provider/alibaba/types/types.go index a8ef006e0..9e58cd401 100644 --- a/pkg/cloudprovider/provider/alibaba/types/types.go +++ b/pkg/cloudprovider/provider/alibaba/types/types.go @@ -17,6 +17,7 @@ limitations under the License. package types import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) @@ -33,3 +34,9 @@ type RawConfig struct { DiskType providerconfigtypes.ConfigVarString `json:"diskType,omitempty"` DiskSize providerconfigtypes.ConfigVarString `json:"diskSize,omitempty"` } + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 304928cde..8d53160ed 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -30,7 +30,7 @@ import ( anxvm "github.com/anexia-it/go-anxcloud/pkg/vsphere/provisioning/vm" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" @@ -57,22 +57,22 @@ type provider struct { configVarResolver *providerconfig.ConfigVarResolver } -func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if s.Value == nil { +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { + if provSpec.Value == nil { return nil, nil, fmt.Errorf("machine.spec.providerSpec.value is nil") } - pConfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pConfig) + + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err } - if pConfig.OperatingSystemSpec.Raw == nil { + if pconfig.OperatingSystemSpec.Raw == nil { return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } - rawConfig := anxtypes.RawConfig{} - if err = json.Unmarshal(pConfig.CloudProviderSpec.Raw, &rawConfig); err != nil { + rawConfig, err := anxtypes.GetConfig(*pconfig) + if err != nil { return nil, nil, err } @@ -101,7 +101,7 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return nil, nil, fmt.Errorf("failed to get 'vlanID': %v", err) } - return &c, &pConfig, nil + return &c, pconfig, nil } // New returns an Anexia provider @@ -110,12 +110,12 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes } // AddDefaults adds omitted optional values to the given MachineSpec -func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } // Validate returns success or failure based according to its ProviderSpec -func (p *provider) Validate(machinespec v1alpha1.MachineSpec) error { +func (p *provider) Validate(machinespec clusterv1alpha1.MachineSpec) error { config, _, err := p.getConfig(machinespec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -152,7 +152,7 @@ func (p *provider) Validate(machinespec v1alpha1.MachineSpec) error { return nil } -func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, "failed to parse MachineSpec: %v", err) @@ -185,12 +185,12 @@ func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provider }, nil } -func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (string, string, error) { +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (string, string, error) { return "", "", nil } // Create creates a cloud instance according to the given machine -func (p *provider) Create(machine *v1alpha1.Machine, providerData *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, providerData *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, "failed to parse MachineSpec: %v", err) @@ -273,7 +273,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, providerData *cloudprovider return p.Get(machine, providerData) } -func (p *provider) Cleanup(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, newError(common.InvalidConfigurationMachineError, "failed to parse MachineSpec: %v", err) @@ -305,15 +305,15 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, _ *cloudprovidertypes.Prov return true, nil } -func (p *provider) MigrateUID(_ *v1alpha1.Machine, _ k8stypes.UID) error { +func (p *provider) MigrateUID(_ *clusterv1alpha1.Machine, _ k8stypes.UID) error { return nil } -func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { return map[string]string{}, nil } -func (p *provider) SetMetricsForMachines(machine v1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(machine clusterv1alpha1.MachineList) error { return nil } @@ -340,12 +340,12 @@ func newError(reason common.MachineStatusError, msg string, args ...interface{}) } } -func updateStatus(machine *v1alpha1.Machine, status *anxtypes.ProviderStatus, updater cloudprovidertypes.MachineUpdater) error { +func updateStatus(machine *clusterv1alpha1.Machine, status *anxtypes.ProviderStatus, updater cloudprovidertypes.MachineUpdater) error { rawStatus, err := json.Marshal(status) if err != nil { return err } - err = updater(machine, func(machine *v1alpha1.Machine) { + err = updater(machine, func(machine *clusterv1alpha1.Machine) { machine.Status.ProviderStatus = &runtime.RawExtension{ Raw: rawStatus, } diff --git a/pkg/cloudprovider/provider/anexia/types/types.go b/pkg/cloudprovider/provider/anexia/types/types.go index 699e904ee..9bdc929fd 100644 --- a/pkg/cloudprovider/provider/anexia/types/types.go +++ b/pkg/cloudprovider/provider/anexia/types/types.go @@ -19,6 +19,7 @@ package types import ( "time" + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) @@ -48,3 +49,9 @@ type ProviderStatus struct { ProvisioningID string `json:"provisioningID"` // TODO: add conditions to track progress on the provider side } + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 1381be5fe..00bad52ca 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -36,7 +36,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" awstypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/aws/types" @@ -347,12 +347,12 @@ func getDefaultRootDevicePath(os providerconfigtypes.OperatingSystem) (string, e return "", fmt.Errorf("no default root path found for %s operating system", os) } -func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *awstypes.RawConfig, error) { - if s.Value == nil { +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *awstypes.RawConfig, error) { + if provSpec.Value == nil { return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pconfig) + + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, nil, err } @@ -361,10 +361,11 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return nil, nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } - rawConfig := awstypes.RawConfig{} - if err := json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawConfig); err != nil { + rawConfig, err := awstypes.GetConfig(*pconfig) + if err != nil { return nil, nil, nil, fmt.Errorf("failed to unmarshal: %v", err) } + c := Config{} c.AccessKeyID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AccessKeyID, "AWS_ACCESS_KEY_ID") if err != nil { @@ -473,7 +474,7 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt } c.AssumeRoleExternalID = assumeRoleExternalID - return &c, &pconfig, &rawConfig, err + return &c, pconfig, rawConfig, err } func getSession(id, secret, token, region, assumeRoleARN, assumeRoleExternalID string) (*session.Session, error) { @@ -538,7 +539,7 @@ func getEC2client(id, secret, region, assumeRoleArn, assumeRoleExternalID string return ec2.New(sess), nil } -func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { _, _, rawConfig, err := p.getConfig(spec.ProviderSpec) if err != nil { return spec, err @@ -559,7 +560,7 @@ func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, return spec, err } -func (p *provider) Validate(spec v1alpha1.MachineSpec) error { +func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { config, pc, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %v", err) @@ -653,7 +654,7 @@ func getVpc(client *ec2.EC2, id string) (*ec2.Vpc, error) { return vpcOut.Vpcs[0], nil } -func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -801,7 +802,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr return &awsInstance{instance: runOut.Instances[0]}, nil } -func (p *provider) Cleanup(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { ec2instance, err := p.get(machine) if err != nil { if err == cloudprovidererrors.ErrInstanceNotFound { @@ -855,11 +856,11 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, _ *cloudprovidertypes.Prov return false, nil } -func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { return p.get(machine) } -func (p *provider) get(machine *v1alpha1.Machine) (*awsInstance, error) { +func (p *provider) get(machine *clusterv1alpha1.Machine) (*awsInstance, error) { config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -906,7 +907,7 @@ func (p *provider) get(machine *v1alpha1.Machine) (*awsInstance, error) { return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { c, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return "", "", fmt.Errorf("failed to parse config: %v", err) @@ -929,7 +930,7 @@ func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, nam } -func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) @@ -943,7 +944,7 @@ func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]s return labels, err } -func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { machineInstance, err := p.get(machine) if err != nil { if err == cloudprovidererrors.ErrInstanceNotFound { @@ -1065,19 +1066,21 @@ func awsErrorToTerminalError(err error, msg string) error { return nil } -func setProviderSpec(rawConfig awstypes.RawConfig, s v1alpha1.ProviderSpec) (*runtime.RawExtension, error) { - if s.Value == nil { +func setProviderSpec(rawConfig awstypes.RawConfig, provSpec clusterv1alpha1.ProviderSpec) (*runtime.RawExtension, error) { + if provSpec.Value == nil { return nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pconfig) + + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, err } + rawCloudProviderSpec, err := json.Marshal(rawConfig) if err != nil { return nil, err } + pconfig.CloudProviderSpec = runtime.RawExtension{Raw: rawCloudProviderSpec} rawPconfig, err := json.Marshal(pconfig) if err != nil { @@ -1087,7 +1090,7 @@ func setProviderSpec(rawConfig awstypes.RawConfig, s v1alpha1.ProviderSpec) (*ru return &runtime.RawExtension{Raw: rawPconfig}, nil } -func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { metricInstancesForMachines.Reset() if len(machines.Items) < 1 { @@ -1148,7 +1151,7 @@ func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { return nil } -func getIntanceCountForMachine(machine v1alpha1.Machine, reservations []*ec2.Reservation) float64 { +func getIntanceCountForMachine(machine clusterv1alpha1.Machine, reservations []*ec2.Reservation) float64 { var count float64 for _, reservation := range reservations { for _, i := range reservation.Instances { @@ -1195,7 +1198,7 @@ func filterSupportedRHELImages(images []*ec2.Image) ([]*ec2.Image, error) { // That could result in two or more instances created for one Machine object. // This happens more often in some AWS regions because some regions have // slower instance creation (e.g. us-east-1 and us-west-2). -func (p *provider) waitForInstance(machine *v1alpha1.Machine) error { +func (p *provider) waitForInstance(machine *clusterv1alpha1.Machine) error { return wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) { _, err := p.get(machine) if err == cloudprovidererrors.ErrInstanceNotFound { diff --git a/pkg/cloudprovider/provider/aws/types/types.go b/pkg/cloudprovider/provider/aws/types/types.go index 8f2072325..11974b9e3 100644 --- a/pkg/cloudprovider/provider/aws/types/types.go +++ b/pkg/cloudprovider/provider/aws/types/types.go @@ -17,6 +17,7 @@ limitations under the License. package types import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) @@ -60,3 +61,9 @@ const ( CPUArchitectureX86_64 CPUArchitecture = "x86_64" CPUArchitectureI386 CPUArchitecture = "i386" ) + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index a4c707b14..5ff671e11 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -19,7 +19,6 @@ package azure import ( "context" "encoding/base64" - "encoding/json" "errors" "fmt" "strings" @@ -29,7 +28,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" @@ -186,12 +185,12 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes return &provider{configVarResolver: configVarResolver} } -func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*config, *providerconfigtypes.Config, error) { - if s.Value == nil { +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *providerconfigtypes.Config, error) { + if provSpec.Value == nil { return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pconfig) + + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err } @@ -200,8 +199,7 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*config, *providerconfigt return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } - rawCfg := azuretypes.RawConfig{} - err = json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawCfg) + rawCfg, err := azuretypes.GetConfig(*pconfig) if err != nil { return nil, nil, err } @@ -315,7 +313,7 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*config, *providerconfigt return nil, nil, fmt.Errorf("failed to get image id: %v", err) } - return &c, &pconfig, nil + return &c, pconfig, nil } func getVMIPAddresses(ctx context.Context, c *config, vm *compute.VirtualMachine) (map[string]v1.NodeAddressType, error) { @@ -449,7 +447,7 @@ func getInternalIPAddresses(ctx context.Context, c *config, inetface, ipconfigNa return ipAddresses, nil } -func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } @@ -482,7 +480,7 @@ func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) return sp, nil } -func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { config, providerCfg, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -506,7 +504,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr publicIPName := ifaceName + "-pubip" var publicIP *network.PublicIPAddress if config.AssignPublicIP { - if err = data.Update(machine, func(updatedMachine *v1alpha1.Machine) { + if err = data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { if !kuberneteshelper.HasFinalizer(updatedMachine, finalizerPublicIP) { updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerPublicIP) } @@ -519,7 +517,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr } } - if err := data.Update(machine, func(updatedMachine *v1alpha1.Machine) { + if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { if !kuberneteshelper.HasFinalizer(updatedMachine, finalizerNIC) { updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerNIC) } @@ -590,7 +588,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr } klog.Infof("Creating machine %q", machine.Name) - if err := data.Update(machine, func(updatedMachine *v1alpha1.Machine) { + if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { if !kuberneteshelper.HasFinalizer(updatedMachine, finalizerDisks) { updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerDisks) } @@ -635,7 +633,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr return &azureVM{vm: &vm, ipAddresses: ipAddresses, status: status}, nil } -func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, fmt.Errorf("failed to parse MachineSpec: %v", err) @@ -656,7 +654,7 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.P return false, fmt.Errorf("failed to delete instance for machine %q: %v", machine.Name, err) } - if err := data.Update(machine, func(updatedMachine *v1alpha1.Machine) { + if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerVM) }); err != nil { return false, err @@ -666,7 +664,7 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.P if err := deleteDisksByMachineUID(context.TODO(), config, machine.UID); err != nil { return false, fmt.Errorf("failed to remove disks of machine %q: %v", machine.Name, err) } - if err := data.Update(machine, func(updatedMachine *v1alpha1.Machine) { + if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerDisks) }); err != nil { return false, err @@ -676,7 +674,7 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.P if err := deleteInterfacesByMachineUID(context.TODO(), config, machine.UID); err != nil { return false, fmt.Errorf("failed to remove network interfaces of machine %q: %v", machine.Name, err) } - if err := data.Update(machine, func(updatedMachine *v1alpha1.Machine) { + if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerNIC) }); err != nil { return false, err @@ -686,7 +684,7 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.P if err := deleteIPAddressesByMachineUID(context.TODO(), config, machine.UID); err != nil { return false, fmt.Errorf("failed to remove public IP addresses of machine %q: %v", machine.Name, err) } - if err := data.Update(machine, func(updatedMachine *v1alpha1.Machine) { + if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerPublicIP) }); err != nil { return false, err @@ -779,11 +777,11 @@ func getVMStatus(ctx context.Context, c *config, vmName string) (instance.Status } } -func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { return p.get(machine) } -func (p *provider) get(machine *v1alpha1.Machine) (*azureVM, error) { +func (p *provider) get(machine *clusterv1alpha1.Machine) (*azureVM, error) { config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("failed to parse MachineSpec: %v", err) @@ -811,7 +809,7 @@ func (p *provider) get(machine *v1alpha1.Machine) (*azureVM, error) { return &azureVM{vm: vm, ipAddresses: ipAddresses, status: status}, nil } -func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { c, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return "", "", fmt.Errorf("failed to parse config: %v", err) @@ -849,7 +847,7 @@ func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, nam return s, "azure", nil } -func (p *provider) Validate(spec v1alpha1.MachineSpec) error { +func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, providerCfg, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %v", err) @@ -909,7 +907,7 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { return err } -func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -986,7 +984,7 @@ func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { return nil } -func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) c, _, err := p.getConfig(machine.Spec.ProviderSpec) @@ -998,7 +996,7 @@ func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]s return labels, err } -func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/azure/types/types.go b/pkg/cloudprovider/provider/azure/types/types.go index f5c725cf4..7c64c9cd7 100644 --- a/pkg/cloudprovider/provider/azure/types/types.go +++ b/pkg/cloudprovider/provider/azure/types/types.go @@ -17,6 +17,7 @@ limitations under the License. package types import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) @@ -63,3 +64,9 @@ type ImageReference struct { Sku string `json:"sku,omitempty"` Version string `json:"version,omitempty"` } + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/baremetal/provider.go b/pkg/cloudprovider/provider/baremetal/provider.go index d65a7f3b6..3956a12b1 100644 --- a/pkg/cloudprovider/provider/baremetal/provider.go +++ b/pkg/cloudprovider/provider/baremetal/provider.go @@ -23,7 +23,7 @@ import ( "fmt" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins" @@ -80,12 +80,12 @@ type Config struct { driverSpec runtime.RawExtension } -func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if s.Value == nil { +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { + if provSpec.Value == nil { return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pconfig) + + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err } @@ -94,10 +94,11 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } - rawConfig := baremetaltypes.RawConfig{} - if err := json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawConfig); err != nil { + rawConfig, err := baremetaltypes.GetConfig(*pconfig) + if err != nil { return nil, nil, fmt.Errorf("failed to unmarshal: %v", err) } + c := Config{} endpoint, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.MetadataClient.Endpoint, "METADATA_SERVER_ENDPOINT") if err != nil { @@ -156,15 +157,16 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt default: return nil, nil, fmt.Errorf("unsupported baremetal driver: %s", pconfig.CloudProvider) } - return &c, &pconfig, err + + return &c, pconfig, err } -func (p provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (p provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { _, _, err := p.getConfig(spec.ProviderSpec) return spec, err } -func (p provider) Validate(spec v1alpha1.MachineSpec) error { +func (p provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %v", err) @@ -181,7 +183,7 @@ func (p provider) Validate(spec v1alpha1.MachineSpec) error { return nil } -func (p provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -204,11 +206,11 @@ func (p provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderD }, nil } -func (p provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) { +func (p provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } -func (p provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -244,7 +246,7 @@ func (p provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pro }, nil } -func (p provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, cloudprovidererrors.TerminalError{ @@ -274,14 +276,14 @@ func (p provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr return true, nil } -func (p provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (p provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { return nil, nil } -func (p provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { +func (p provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { return nil } -func (p provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { +func (p provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/baremetal/types/types.go b/pkg/cloudprovider/provider/baremetal/types/types.go index 447df9067..374cc2fb2 100644 --- a/pkg/cloudprovider/provider/baremetal/types/types.go +++ b/pkg/cloudprovider/provider/baremetal/types/types.go @@ -17,6 +17,7 @@ limitations under the License. package types import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" "k8s.io/apimachinery/pkg/runtime" @@ -35,3 +36,9 @@ type MetadataClientConfig struct { Password providerconfigtypes.ConfigVarString `json:"password,omitempty"` Token providerconfigtypes.ConfigVarString `json:"token,omitempty"` } + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index 81e99d99b..15a889ff4 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -18,7 +18,6 @@ package digitalocean import ( "context" - "encoding/json" "errors" "fmt" "net/http" @@ -29,7 +28,7 @@ import ( "golang.org/x/oauth2" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" @@ -101,12 +100,12 @@ func getClient(token string) *godo.Client { return godo.NewClient(oauthClient) } -func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if s.Value == nil { +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { + if provSpec.Value == nil { return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pconfig) + + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err } @@ -114,8 +113,8 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt if pconfig.OperatingSystemSpec.Raw == nil { return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } - rawConfig := digitaloceantypes.RawConfig{} - err = json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawConfig) + + rawConfig, err := digitaloceantypes.GetConfig(*pconfig) if err != nil { return nil, nil, err } @@ -157,14 +156,14 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt c.Tags = append(c.Tags, tagVal) } - return &c, &pconfig, err + return &c, pconfig, err } -func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } -func (p *provider) Validate(spec v1alpha1.MachineSpec) error { +func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %v", err) @@ -264,7 +263,7 @@ func uploadRandomSSHPublicKey(ctx context.Context, service godo.KeysService) (st return newDoKey.Fingerprint, nil } -func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -336,7 +335,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provi return &doInstance{droplet: droplet}, err } -func (p *provider) Cleanup(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { instance, err := p.get(machine) if err != nil { if err == cloudprovidererrors.ErrInstanceNotFound { @@ -368,11 +367,11 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, _ *cloudprovidertypes.Prov return false, nil } -func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { return p.get(machine) } -func (p *provider) get(machine *v1alpha1.Machine) (*doInstance, error) { +func (p *provider) get(machine *clusterv1alpha1.Machine) (*doInstance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -427,7 +426,7 @@ func (p *provider) listDroplets(token string) ([]godo.Droplet, error) { return result, nil } -func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -469,11 +468,11 @@ func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { return nil } -func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } -func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) c, _, err := p.getConfig(machine.Spec.ProviderSpec) @@ -546,6 +545,6 @@ func doStatusAndErrToTerminalError(status int, err error) error { } } -func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/digitalocean/types/types.go b/pkg/cloudprovider/provider/digitalocean/types/types.go index 485fc19f0..a0fdb6830 100644 --- a/pkg/cloudprovider/provider/digitalocean/types/types.go +++ b/pkg/cloudprovider/provider/digitalocean/types/types.go @@ -17,6 +17,7 @@ limitations under the License. package types import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) @@ -30,3 +31,9 @@ type RawConfig struct { Monitoring providerconfigtypes.ConfigVarBool `json:"monitoring"` Tags []providerconfigtypes.ConfigVarString `json:"tags,omitempty"` } + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index d045435be..81b492ea1 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -26,7 +26,7 @@ import ( "github.com/packethost/packngo" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" equinixmetaltypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/equinixmetal/types" @@ -77,18 +77,18 @@ type provider struct { configVarResolver *providerconfig.ConfigVarResolver } -func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *equinixmetaltypes.RawConfig, *providerconfigtypes.Config, error) { - if s.Value == nil { +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *equinixmetaltypes.RawConfig, *providerconfigtypes.Config, error) { + if provSpec.Value == nil { return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pconfig) + + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, nil, err } - rawConfig := equinixmetaltypes.RawConfig{} - if err = json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawConfig); err != nil { + rawConfig, err := equinixmetaltypes.GetConfig(*pconfig) + if err != nil { return nil, nil, nil, err } @@ -143,10 +143,10 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *equinixmetaltyp // ensure we have defaults c.populateDefaults() - return &c, &rawConfig, &pconfig, err + return &c, rawConfig, pconfig, err } -func (p *provider) getMetalDevice(machine *v1alpha1.Machine) (*packngo.Device, *packngo.Client, error) { +func (p *provider) getMetalDevice(machine *clusterv1alpha1.Machine) (*packngo.Device, *packngo.Client, error) { c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, nil, cloudprovidererrors.TerminalError{ @@ -163,7 +163,7 @@ func (p *provider) getMetalDevice(machine *v1alpha1.Machine) (*packngo.Device, * return device, client, nil } -func (p *provider) Validate(spec v1alpha1.MachineSpec) error { +func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, _, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %v", err) @@ -214,7 +214,7 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, _, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -254,7 +254,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provi return &metalDevice{device: device}, nil } -func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { instance, err := p.Get(machine, data) if err != nil { if err == cloudprovidererrors.ErrInstanceNotFound { @@ -280,7 +280,7 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.P return false, nil } -func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { _, rawConfig, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return spec, err @@ -293,7 +293,7 @@ func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, return spec, nil } -func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { device, _, err := p.getMetalDevice(machine) if err != nil { return nil, err @@ -305,7 +305,7 @@ func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provider return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(machine *v1alpha1.Machine, newID types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newID types.UID) error { device, client, err := p.getMetalDevice(machine) if err != nil { return err @@ -340,11 +340,11 @@ func (p *provider) MigrateUID(machine *v1alpha1.Machine, newID types.UID) error return nil } -func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } -func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) @@ -356,7 +356,7 @@ func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]s return labels, err } -func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { return nil } @@ -400,24 +400,27 @@ func (s *metalDevice) Status() instance.Status { /****** CONVENIENCE INTERNAL FUNCTIONS ******/ -func setProviderSpec(rawConfig equinixmetaltypes.RawConfig, s v1alpha1.ProviderSpec) (*runtime.RawExtension, error) { +func setProviderSpec(rawConfig equinixmetaltypes.RawConfig, s clusterv1alpha1.ProviderSpec) (*runtime.RawExtension, error) { if s.Value == nil { return nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pconfig) + + pconfig, err := providerconfigtypes.GetConfig(s) if err != nil { return nil, err } + rawCloudProviderSpec, err := json.Marshal(rawConfig) if err != nil { return nil, err } + pconfig.CloudProviderSpec = runtime.RawExtension{Raw: rawCloudProviderSpec} rawPconfig, err := json.Marshal(pconfig) if err != nil { return nil, err } + return &runtime.RawExtension{Raw: rawPconfig}, nil } diff --git a/pkg/cloudprovider/provider/equinixmetal/types/types.go b/pkg/cloudprovider/provider/equinixmetal/types/types.go index 4ac292e0c..676c0f14a 100644 --- a/pkg/cloudprovider/provider/equinixmetal/types/types.go +++ b/pkg/cloudprovider/provider/equinixmetal/types/types.go @@ -17,6 +17,7 @@ limitations under the License. package types import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) @@ -28,3 +29,9 @@ type RawConfig struct { Facilities []providerconfigtypes.ConfigVarString `json:"facilities"` Tags []providerconfigtypes.ConfigVarString `json:"tags,omitempty"` } + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/fake/provider.go b/pkg/cloudprovider/provider/fake/provider.go index ea6126580..48f6dd240 100644 --- a/pkg/cloudprovider/provider/fake/provider.go +++ b/pkg/cloudprovider/provider/fake/provider.go @@ -20,13 +20,13 @@ import ( "encoding/json" "fmt" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/klog" ) @@ -47,7 +47,7 @@ func (f CloudProviderInstance) ID() string { return "" } -func (f CloudProviderInstance) Addresses() map[string]v1.NodeAddressType { +func (f CloudProviderInstance) Addresses() map[string]corev1.NodeAddressType { return nil } @@ -60,14 +60,13 @@ func New(_ *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{} } -func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } // Validate returns success or failure based according to its FakeCloudProviderSpec -func (p *provider) Validate(machinespec v1alpha1.MachineSpec) error { - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(machinespec.ProviderSpec.Value.Raw, &pconfig) +func (p *provider) Validate(machinespec clusterv1alpha1.MachineSpec) error { + pconfig, err := providerconfigtypes.GetConfig(machinespec.ProviderSpec) if err != nil { return err } @@ -86,31 +85,31 @@ func (p *provider) Validate(machinespec v1alpha1.MachineSpec) error { return fmt.Errorf("failing validation as requested") } -func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { return CloudProviderInstance{}, nil } -func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (string, string, error) { +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (string, string, error) { return "", "", nil } // Create creates a cloud instance according to the given machine -func (p *provider) Create(_ *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData, _ string) (instance.Instance, error) { +func (p *provider) Create(_ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, _ string) (instance.Instance, error) { return CloudProviderInstance{}, nil } -func (p *provider) Cleanup(_ *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(_ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { return true, nil } -func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { return nil } -func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { return map[string]string{}, nil } -func (p *provider) SetMetricsForMachines(_ v1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index 092e2ff1d..6befe983d 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -66,27 +66,28 @@ const ( // newCloudProviderSpec creates a cloud provider specification out of the // given ProviderSpec. -func newCloudProviderSpec(spec v1alpha1.ProviderSpec) (*gcetypes.CloudProviderSpec, *providerconfigtypes.Config, error) { +func newCloudProviderSpec(provSpec v1alpha1.ProviderSpec) (*gcetypes.CloudProviderSpec, *providerconfigtypes.Config, error) { // Retrieve provider configuration from machine specification. - if spec.Value == nil { + if provSpec.Value == nil { return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - providerConfig := providerconfigtypes.Config{} - err := json.Unmarshal(spec.Value.Raw, &providerConfig) + + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, fmt.Errorf("cannot unmarshal machine.spec.providerconfig.value: %v", err) } - if providerConfig.OperatingSystemSpec.Raw == nil { + if pconfig.OperatingSystemSpec.Raw == nil { return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } + // Retrieve cloud provider specification from cloud provider specification. - cpSpec := &gcetypes.CloudProviderSpec{} - err = json.Unmarshal(providerConfig.CloudProviderSpec.Raw, cpSpec) + cpSpec, err := gcetypes.GetConfig(*pconfig) if err != nil { return nil, nil, fmt.Errorf("cannot unmarshal cloud provider specification: %v", err) } - return cpSpec, &providerConfig, nil + + return cpSpec, pconfig, nil } // config contains the configuration of the Provider. diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index c90ce1ab2..0297843c4 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -31,7 +31,7 @@ import ( "google.golang.org/api/googleapi" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" gcetypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/gce/types" @@ -81,7 +81,7 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) *Provider { } // AddDefaults reads the MachineSpec and applies defaults for provider specific fields -func (p *Provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (p *Provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { // Read cloud provider spec. cpSpec, _, err := newCloudProviderSpec(spec.ProviderSpec) if err != nil { @@ -99,7 +99,7 @@ func (p *Provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, } // Validate checks the given machine's specification. -func (p *Provider) Validate(spec v1alpha1.MachineSpec) error { +func (p *Provider) Validate(spec clusterv1alpha1.MachineSpec) error { // Read configuration. cfg, err := newConfig(p.resolver, spec.ProviderSpec) if err != nil { @@ -129,11 +129,11 @@ func (p *Provider) Validate(spec v1alpha1.MachineSpec) error { } // Get retrieves a node instance that is associated with the given machine. -func (p *Provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *Provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { return p.get(machine) } -func (p *Provider) get(machine *v1alpha1.Machine) (*googleInstance, error) { +func (p *Provider) get(machine *clusterv1alpha1.Machine) (*googleInstance, error) { // Read configuration. cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) if err != nil { @@ -169,7 +169,7 @@ func (p *Provider) get(machine *v1alpha1.Machine) (*googleInstance, error) { } // GetCloudConfig returns the cloud provider specific cloud-config for the kubelet. -func (p *Provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) { +func (p *Provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { // Read configuration. cfg, err := newConfig(p.resolver, spec.ProviderSpec) if err != nil { @@ -196,7 +196,7 @@ func (p *Provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, nam // Create inserts a cloud instance according to the given machine. func (p *Provider) Create( - machine *v1alpha1.Machine, + machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, ) (instance.Instance, error) { @@ -270,7 +270,7 @@ func (p *Provider) Create( } // Cleanup deletes the instance associated with the machine and all associated resources. -func (p *Provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *Provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { // Read configuration. cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) if err != nil { @@ -300,7 +300,7 @@ func (p *Provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.P } // MachineMetricsLabels returns labels used for the Prometheus metrics about created machines. -func (p *Provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (p *Provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { // Read configuration. cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) if err != nil { @@ -320,7 +320,7 @@ func (p *Provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]s // MigrateUID updates the UID of an instance after the controller migrates types // and the UID of the machine object changed. -func (p *Provider) MigrateUID(machine *v1alpha1.Machine, newUID types.UID) error { +func (p *Provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { // Read configuration. cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) if err != nil { @@ -362,7 +362,7 @@ func (p *Provider) MigrateUID(machine *v1alpha1.Machine, newUID types.UID) error } // SetMetricsForMachines allows providers to provide provider-specific metrics. -func (p *Provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { +func (p *Provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/gce/types/types.go b/pkg/cloudprovider/provider/gce/types/types.go index 254659d86..8eeed7207 100644 --- a/pkg/cloudprovider/provider/gce/types/types.go +++ b/pkg/cloudprovider/provider/gce/types/types.go @@ -21,6 +21,7 @@ import ( "fmt" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" "k8s.io/apimachinery/pkg/runtime" @@ -67,3 +68,11 @@ func (cpSpec *CloudProviderSpec) UpdateProviderSpec(spec v1alpha1.ProviderSpec) } return &runtime.RawExtension{Raw: rawProviderConfig}, nil } + +type RawConfig = CloudProviderSpec + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 5dd548b03..f066d7290 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -18,7 +18,6 @@ package hetzner import ( "context" - "encoding/json" "errors" "fmt" "net/http" @@ -28,7 +27,7 @@ import ( "github.com/hetznercloud/hcloud-go/hcloud" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" @@ -82,12 +81,12 @@ func getClient(token string) *hcloud.Client { return hcloud.NewClient(hcloud.WithToken(token)) } -func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if s.Value == nil { +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { + if provSpec.Value == nil { return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pconfig) + + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err } @@ -96,8 +95,8 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } - rawConfig := hetznertypes.RawConfig{} - if err = json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawConfig); err != nil { + rawConfig, err := hetznertypes.GetConfig(*pconfig) + if err != nil { return nil, nil, err } @@ -149,7 +148,8 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt } c.Labels = rawConfig.Labels - return &c, &pconfig, err + + return &c, pconfig, err } func (p *provider) getServerPlacementGroup(ctx context.Context, client *hcloud.Client, c *Config) (*hcloud.PlacementGroup, error) { @@ -182,7 +182,7 @@ func (p *provider) getServerPlacementGroup(ctx context.Context, client *hcloud.C return createdPg.PlacementGroup, nil } -func (p *provider) Validate(spec v1alpha1.MachineSpec) error { +func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %v", err) @@ -245,7 +245,7 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -386,7 +386,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provi return &hetznerServer{server: serverCreateRes.Server}, nil } -func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { instance, err := p.Get(machine, data) if err != nil { if err == cloudprovidererrors.ErrInstanceNotFound { @@ -437,11 +437,11 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.P return false, nil } -func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } -func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -469,7 +469,7 @@ func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provider return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -509,11 +509,11 @@ func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { return nil } -func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } -func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) c, _, err := p.getConfig(machine.Spec.ProviderSpec) @@ -587,6 +587,6 @@ func hzErrorToTerminalError(err error, msg string) error { return err } -func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/hetzner/types/types.go b/pkg/cloudprovider/provider/hetzner/types/types.go index b972fc768..dd1b86471 100644 --- a/pkg/cloudprovider/provider/hetzner/types/types.go +++ b/pkg/cloudprovider/provider/hetzner/types/types.go @@ -17,6 +17,7 @@ limitations under the License. package types import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) @@ -31,3 +32,9 @@ type RawConfig struct { Firewalls []providerconfigtypes.ConfigVarString `json:"firewalls"` Labels map[string]string `json:"labels,omitempty"` } + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index f8818e2c7..2222a71b4 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -18,7 +18,6 @@ package kubevirt import ( "context" - "encoding/json" "errors" "fmt" "strconv" @@ -29,7 +28,7 @@ import ( cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" kubevirttypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/kubevirt/types" @@ -116,12 +115,12 @@ func (k *kubeVirtServer) Status() instance.Status { var _ instance.Instance = &kubeVirtServer{} -func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if s.Value == nil { +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { + if provSpec.Value == nil { return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pconfig) + + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err } @@ -130,10 +129,11 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } - rawConfig := kubevirttypes.RawConfig{} - if err = json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawConfig); err != nil { + rawConfig, err := kubevirttypes.GetConfig(*pconfig) + if err != nil { return nil, nil, err } + config := Config{} config.Kubeconfig, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Kubeconfig, "KUBEVIRT_KUBECONFIG") if err != nil { @@ -185,10 +185,10 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt config.DNSConfig = rawConfig.DNSConfig } - return &config, &pconfig, nil + return &config, pconfig, nil } -func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -245,11 +245,11 @@ func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provider // We don't use the UID for kubevirt because the name of a VMI must stay stable // in order for the node name to stay stable. The operator is responsible for ensuring // there are no conflicts, e.G. by using one Namespace per Kubevirt user cluster -func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { return nil } -func (p *provider) Validate(spec v1alpha1.MachineSpec) error { +func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %v", err) @@ -278,11 +278,11 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { return nil } -func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } -func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { c, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return "", "", fmt.Errorf("failed to parse config: %v", err) @@ -295,7 +295,7 @@ func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, nam return ccs, string(providerconfigtypes.CloudProviderExternal), err } -func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) c, _, err := p.getConfig(machine.Spec.ProviderSpec) @@ -308,7 +308,7 @@ func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]s return labels, err } -func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -461,7 +461,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provi } -func (p *provider) Cleanup(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, cloudprovidererrors.TerminalError{ @@ -502,7 +502,7 @@ func parseResources(cpus, memory string) (*corev1.ResourceList, error) { }, nil } -func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index 2055bebe8..4924f8e49 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -17,6 +17,7 @@ limitations under the License. package types import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" @@ -33,3 +34,9 @@ type RawConfig struct { DNSPolicy providerconfigtypes.ConfigVarString `json:"dnsPolicy,omitempty"` DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` } + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index dbcd01c0d..32ce14863 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -32,7 +32,7 @@ import ( "golang.org/x/oauth2" common "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" @@ -107,12 +107,12 @@ func getClient(token string) linodego.Client { return client } -func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if s.Value == nil { +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { + if provSpec.Value == nil { return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pconfig) + + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err } @@ -121,8 +121,7 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } - rawConfig := linodetypes.RawConfig{} - err = json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawConfig) + rawConfig, err := linodetypes.GetConfig(*pconfig) if err != nil { return nil, nil, err } @@ -157,14 +156,14 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt c.Tags = append(c.Tags, tagVal) } - return &c, &pconfig, err + return &c, pconfig, err } -func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } -func (p *provider) Validate(spec v1alpha1.MachineSpec) error { +func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %v", err) @@ -213,7 +212,7 @@ func createRandomPassword() (string, error) { return rootPass, nil } -func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -275,7 +274,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provi return &linodeInstance{linode: linode}, err } -func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { instance, err := p.Get(machine, data) if err != nil { if err == cloudprovidererrors.ErrInstanceNotFound { @@ -316,7 +315,7 @@ func getListOptions(name string) *linodego.ListOptions { return listOptions } -func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -344,7 +343,7 @@ func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provider return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -383,11 +382,11 @@ func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { return nil } -func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } -func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) c, _, err := p.getConfig(machine.Spec.ProviderSpec) @@ -459,6 +458,6 @@ func linodeStatusAndErrToTerminalError(err error) error { } } -func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/linode/types/types.go b/pkg/cloudprovider/provider/linode/types/types.go index 1c95bef3e..f2f2b7ef3 100644 --- a/pkg/cloudprovider/provider/linode/types/types.go +++ b/pkg/cloudprovider/provider/linode/types/types.go @@ -17,6 +17,7 @@ limitations under the License. package types import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) @@ -28,3 +29,9 @@ type RawConfig struct { PrivateNetworking providerconfigtypes.ConfigVarBool `json:"private_networking"` Tags []providerconfigtypes.ConfigVarString `json:"tags,omitempty"` } + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 5884d776a..6e2a9f2b1 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -17,14 +17,13 @@ limitations under the License. package nutanix import ( - "encoding/json" "errors" "fmt" "strconv" "time" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" nutanixtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/nutanix/types" @@ -99,12 +98,12 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes return provider } -func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *nutanixtypes.RawConfig, error) { - if s.Value == nil { +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *nutanixtypes.RawConfig, error) { + if provSpec.Value == nil { return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pconfig) + + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, nil, err } @@ -113,8 +112,7 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return nil, nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } - rawConfig := nutanixtypes.RawConfig{} - err = json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawConfig) + rawConfig, err := nutanixtypes.GetConfig(*pconfig) if err != nil { return nil, nil, nil, err } @@ -190,14 +188,14 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt c.MemoryMB = rawConfig.MemoryMB c.DiskSizeGB = rawConfig.DiskSize - return &c, &pconfig, &rawConfig, nil + return &c, pconfig, rawConfig, nil } -func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } -func (p *provider) Validate(spec v1alpha1.MachineSpec) error { +func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { config, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse machineSpec: %v", err) @@ -243,7 +241,7 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { vm, err := p.create(machine, userdata) if err != nil { _, cleanupErr := p.Cleanup(machine, data) @@ -255,7 +253,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr return vm, nil } -func (p *provider) create(machine *v1alpha1.Machine, userdata string) (instance.Instance, error) { +func (p *provider) create(machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -275,11 +273,11 @@ func (p *provider) create(machine *v1alpha1.Machine, userdata string) (instance. return createVM(client, machine.Name, *config, pc.OperatingSystem, userdata) } -func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { return p.cleanup(machine, data) } -func (p *provider) cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, cloudprovidererrors.TerminalError{ @@ -336,7 +334,7 @@ func (p *provider) cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.P return true, nil } -func (p *provider) Get(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -401,16 +399,16 @@ func (p *provider) Get(machine *v1alpha1.Machine, data *cloudprovidertypes.Provi }, nil } -func (p *provider) MigrateUID(machine *v1alpha1.Machine, new ktypes.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new ktypes.UID) error { return nil } // GetCloudConfig returns an empty cloud configuration for Nutanix as no CCM exists -func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } -func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) @@ -424,6 +422,6 @@ func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]s return labels, nil } -func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/nutanix/types/types.go b/pkg/cloudprovider/provider/nutanix/types/types.go index 59afc898d..007e270b4 100644 --- a/pkg/cloudprovider/provider/nutanix/types/types.go +++ b/pkg/cloudprovider/provider/nutanix/types/types.go @@ -17,6 +17,7 @@ limitations under the License. package types import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) @@ -65,3 +66,9 @@ type ErrorResponseMsg struct { Message string `json:"message"` Reason string `json:"reason"` } + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 0940b6d95..64007d86b 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -35,7 +35,7 @@ import ( "github.com/gophercloud/gophercloud/pagination" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" openstacktypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/openstack/types" @@ -175,19 +175,12 @@ func (p *provider) getConfigAuth(c *Config, rawConfig *openstacktypes.RawConfig) return nil } -func (p *provider) getConfig(spec v1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *openstacktypes.RawConfig, error) { - if spec.Value == nil { +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *openstacktypes.RawConfig, error) { + if provSpec.Value == nil { return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(spec.Value.Raw, &pconfig) - if err != nil { - return nil, nil, nil, err - } - - var rawConfig openstacktypes.RawConfig - err = json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawConfig) + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, nil, err } @@ -196,6 +189,11 @@ func (p *provider) getConfig(spec v1alpha1.ProviderSpec) (*Config, *providerconf return nil, nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } + rawConfig, err := openstacktypes.GetConfig(*pconfig) + if err != nil { + return nil, nil, nil, err + } + cfg := Config{} cfg.IdentityEndpoint, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.IdentityEndpoint, "OS_AUTH_URL") if err != nil { @@ -203,7 +201,7 @@ func (p *provider) getConfig(spec v1alpha1.ProviderSpec) (*Config, *providerconf } // Retrieve authentication config, username/password or application credentials - err = p.getConfigAuth(&cfg, &rawConfig) + err = p.getConfigAuth(&cfg, rawConfig) if err != nil { return nil, nil, nil, fmt.Errorf("failed to retrieve authentication credentials, error = %v", err) } @@ -296,22 +294,24 @@ func (p *provider) getConfig(spec v1alpha1.ProviderSpec) (*Config, *providerconf return nil, nil, nil, err } - return &cfg, &pconfig, &rawConfig, err + return &cfg, pconfig, rawConfig, err } -func setProviderSpec(rawConfig openstacktypes.RawConfig, s v1alpha1.ProviderSpec) (*runtime.RawExtension, error) { - if s.Value == nil { +func setProviderSpec(rawConfig openstacktypes.RawConfig, provSpec clusterv1alpha1.ProviderSpec) (*runtime.RawExtension, error) { + if provSpec.Value == nil { return nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pconfig) + + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, err } + rawCloudProviderSpec, err := json.Marshal(rawConfig) if err != nil { return nil, err } + pconfig.CloudProviderSpec = runtime.RawExtension{Raw: rawCloudProviderSpec} rawPconfig, err := json.Marshal(pconfig) if err != nil { @@ -349,7 +349,7 @@ func getClient(c *Config) (*gophercloud.ProviderClient, error) { return pc, err } -func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { c, _, rawConfig, err := p.getConfig(spec.ProviderSpec) if err != nil { return spec, cloudprovidererrors.TerminalError{ @@ -439,7 +439,7 @@ func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, return spec, nil } -func (p *provider) Validate(spec v1alpha1.MachineSpec) error { +func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, pc, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %v", err) @@ -554,7 +554,7 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { cfg, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -723,7 +723,7 @@ func deleteInstanceDueToFatalLogged(computeClient *gophercloud.ServiceClient, se klog.V(0).Infof("Instance %s got deleted", serverID) } -func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { var hasFloatingIPReleaseFinalizer bool if finalizers := sets.NewString(machine.Finalizers...); finalizers.Has(floatingIPReleaseFinalizer) { hasFloatingIPReleaseFinalizer = true @@ -771,7 +771,7 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.P return false, nil } -func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -814,7 +814,7 @@ func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provider return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return cloudprovidererrors.TerminalError{ @@ -862,7 +862,7 @@ func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { return nil } -func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { c, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return "", "", fmt.Errorf("failed to parse config: %v", err) @@ -901,7 +901,7 @@ func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, nam return s, "openstack", nil } -func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) @@ -1002,7 +1002,7 @@ type forbiddenResponse struct { } `json:"forbidden"` } -func (p *provider) cleanupFloatingIP(machine *v1alpha1.Machine, updater cloudprovidertypes.MachineUpdater) error { +func (p *provider) cleanupFloatingIP(machine *clusterv1alpha1.Machine, updater cloudprovidertypes.MachineUpdater) error { floatingIPID, exists := machine.Annotations[floatingIPIDAnnotationKey] if !exists { return osErrorToTerminalError(fmt.Errorf("failed to release floating ip"), @@ -1028,7 +1028,7 @@ func (p *provider) cleanupFloatingIP(machine *v1alpha1.Machine, updater cloudpro if err := osfloatingips.Delete(netClient, floatingIPID).ExtractErr(); err != nil && err.Error() != "Resource not found" { return fmt.Errorf("failed to delete floating ip %s: %v", floatingIPID, err) } - if err := updater(machine, func(m *v1alpha1.Machine) { + if err := updater(machine, func(m *clusterv1alpha1.Machine) { finalizers := sets.NewString(m.Finalizers...) finalizers.Delete(floatingIPReleaseFinalizer) m.Finalizers = finalizers.List() @@ -1039,7 +1039,7 @@ func (p *provider) cleanupFloatingIP(machine *v1alpha1.Machine, updater cloudpro return nil } -func assignFloatingIPToInstance(machineUpdater cloudprovidertypes.MachineUpdater, machine *v1alpha1.Machine, netClient *gophercloud.ServiceClient, instanceID, floatingIPPoolName, region string, network *osnetworks.Network) error { +func assignFloatingIPToInstance(machineUpdater cloudprovidertypes.MachineUpdater, machine *clusterv1alpha1.Machine, netClient *gophercloud.ServiceClient, instanceID, floatingIPPoolName, region string, network *osnetworks.Network) error { port, err := getInstancePort(netClient, instanceID, network.ID) if err != nil { return fmt.Errorf("failed to get instance port for network %s in region %s: %v", network.ID, region, err) @@ -1067,7 +1067,7 @@ func assignFloatingIPToInstance(machineUpdater cloudprovidertypes.MachineUpdater if ip, err = createFloatingIP(netClient, port.ID, floatingIPPool); err != nil { return osErrorToTerminalError(err, "failed to allocate a floating ip") } - if err := machineUpdater(machine, func(m *v1alpha1.Machine) { + if err := machineUpdater(machine, func(m *clusterv1alpha1.Machine) { m.Finalizers = append(m.Finalizers, floatingIPReleaseFinalizer) if m.Annotations == nil { m.Annotations = map[string]string{} @@ -1103,6 +1103,6 @@ func assignFloatingIPToInstance(machineUpdater cloudprovidertypes.MachineUpdater return nil } -func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/openstack/types/types.go b/pkg/cloudprovider/provider/openstack/types/types.go index a4ebb27f9..b6d33369a 100644 --- a/pkg/cloudprovider/provider/openstack/types/types.go +++ b/pkg/cloudprovider/provider/openstack/types/types.go @@ -17,6 +17,7 @@ limitations under the License. package types import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) @@ -54,3 +55,9 @@ type RawConfig struct { // This tag is related to server metadata, not compute server's tag Tags map[string]string `json:"tags,omitempty"` } + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go index 1f1738293..59c4a6942 100644 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ b/pkg/cloudprovider/provider/scaleway/provider.go @@ -18,7 +18,6 @@ package scaleway import ( "context" - "encoding/json" "errors" "fmt" "strings" @@ -28,7 +27,7 @@ import ( "github.com/scaleway/scaleway-sdk-go/validation" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" cloudInstance "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" scalewaytypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/scaleway/types" @@ -36,7 +35,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog" @@ -87,12 +86,12 @@ func getImageNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { return "", providerconfigtypes.ErrOSNotSupported } -func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if s.Value == nil { +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { + if provSpec.Value == nil { return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pconfig) + + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err } @@ -101,11 +100,11 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } - rawConfig := scalewaytypes.RawConfig{} - err = json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawConfig) + rawConfig, err := scalewaytypes.GetConfig(*pconfig) if err != nil { return nil, nil, err } + c := Config{} c.AccessKey, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AccessKey, scw.ScwAccessKeyEnv) if err != nil { @@ -133,14 +132,14 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt } c.Tags = rawConfig.Tags - return &c, &pconfig, err + return &c, pconfig, err } -func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } -func (p *provider) Validate(spec v1alpha1.MachineSpec) error { +func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %v", err) @@ -173,7 +172,7 @@ func (p *provider) Validate(spec v1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (cloudInstance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (cloudInstance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -222,7 +221,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provi return &scwServer{server: serverResp.Server}, err } -func (p *provider) Cleanup(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { i, err := p.get(machine) if err != nil { if err == cloudprovidererrors.ErrInstanceNotFound { @@ -255,7 +254,7 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, _ *cloudprovidertypes.Prov return false, nil } -func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (cloudInstance.Instance, error) { +func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (cloudInstance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -289,7 +288,7 @@ func (p *provider) Get(machine *v1alpha1.Machine, _ *cloudprovidertypes.Provider return i, nil } -func (p *provider) get(machine *v1alpha1.Machine) (*scwServer, error) { +func (p *provider) get(machine *clusterv1alpha1.Machine) (*scwServer, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -320,7 +319,7 @@ func (p *provider) get(machine *v1alpha1.Machine) (*scwServer, error) { return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to decode providerconfig: %v", err) @@ -354,11 +353,11 @@ func (p *provider) MigrateUID(machine *v1alpha1.Machine, new types.UID) error { return nil } -func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } -func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) c, _, err := p.getConfig(machine.Spec.ProviderSpec) @@ -382,18 +381,18 @@ func (s *scwServer) ID() string { return s.server.ID } -func (s *scwServer) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{} +func (s *scwServer) Addresses() map[string]corev1.NodeAddressType { + addresses := map[string]corev1.NodeAddressType{} if s.server.PrivateIP != nil { - addresses[*s.server.PrivateIP] = v1.NodeInternalIP + addresses[*s.server.PrivateIP] = corev1.NodeInternalIP } if s.server.PublicIP != nil { - addresses[s.server.PublicIP.Address.String()] = v1.NodeExternalIP + addresses[s.server.PublicIP.Address.String()] = corev1.NodeExternalIP } if s.server.IPv6 != nil { - addresses[s.server.IPv6.Address.String()] = v1.NodeExternalIP + addresses[s.server.IPv6.Address.String()] = corev1.NodeExternalIP } return addresses @@ -444,6 +443,6 @@ func scalewayErrToTerminalError(err error) error { } } -func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/scaleway/types/types.go b/pkg/cloudprovider/provider/scaleway/types/types.go index 80c71d7b3..8ecbd5cb1 100644 --- a/pkg/cloudprovider/provider/scaleway/types/types.go +++ b/pkg/cloudprovider/provider/scaleway/types/types.go @@ -17,6 +17,7 @@ limitations under the License. package types import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) @@ -29,3 +30,9 @@ type RawConfig struct { IPv6 providerconfigtypes.ConfigVarBool `json:"ipv6"` Tags []string `json:"tags,omitempty"` } + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 56b5b1e15..9cfd9c986 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -18,7 +18,6 @@ package vsphere import ( "context" - "encoding/json" "errors" "fmt" "net/url" @@ -32,7 +31,7 @@ import ( "github.com/vmware/govmomi/vim25/types" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" vspheretypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vsphere/types" @@ -104,16 +103,16 @@ func (vsphereServer Server) Status() instance.Status { // Ensures that provider implements Provider interface. var _ cloudprovidertypes.Provider = &provider{} -func (p *provider) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } -func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *vspheretypes.RawConfig, error) { - if s.Value == nil { +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *vspheretypes.RawConfig, error) { + if provSpec.Value == nil { return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig := providerconfigtypes.Config{} - err := json.Unmarshal(s.Value.Raw, &pconfig) + + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, nil, err } @@ -122,8 +121,7 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt return nil, nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } - rawConfig := vspheretypes.RawConfig{} - err = json.Unmarshal(pconfig.CloudProviderSpec.Raw, &rawConfig) + rawConfig, err := vspheretypes.GetConfig(*pconfig) if err != nil { return nil, nil, nil, err } @@ -188,10 +186,10 @@ func (p *provider) getConfig(s v1alpha1.ProviderSpec) (*Config, *providerconfigt c.MemoryMB = rawConfig.MemoryMB c.DiskSizeGB = rawConfig.DiskSizeGB - return &c, &pconfig, &rawConfig, nil + return &c, pconfig, rawConfig, nil } -func (p *provider) Validate(spec v1alpha1.MachineSpec) error { +func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() config, pc, _, err := p.getConfig(spec.ProviderSpec) @@ -264,7 +262,7 @@ func machineInvalidConfigurationTerminalError(err error) error { } } -func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { vm, err := p.create(machine, userdata) if err != nil { _, cleanupErr := p.Cleanup(machine, data) @@ -276,7 +274,7 @@ func (p *provider) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.Pr return vm, nil } -func (p *provider) create(machine *v1alpha1.Machine, userdata string) (instance.Instance, error) { +func (p *provider) create(machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { ctx := context.Background() config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) @@ -344,7 +342,7 @@ func (p *provider) create(machine *v1alpha1.Machine, userdata string) (instance. return Server{name: virtualMachine.Name(), status: instance.StatusRunning, id: virtualMachine.Reference().Value}, nil } -func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -437,7 +435,7 @@ func (p *provider) Cleanup(machine *v1alpha1.Machine, data *cloudprovidertypes.P return true, nil } -func (p *provider) Get(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { ctx := context.Background() config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) @@ -503,11 +501,11 @@ func (p *provider) Get(machine *v1alpha1.Machine, data *cloudprovidertypes.Provi return Server{name: virtualMachine.Name(), status: instance.StatusRunning, addresses: addresses, id: virtualMachine.Reference().Value}, nil } -func (p *provider) MigrateUID(machine *v1alpha1.Machine, new ktypes.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new ktypes.UID) error { return nil } -func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { c, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return "", "", fmt.Errorf("failed to parse config: %v", err) @@ -564,7 +562,7 @@ func (p *provider) GetCloudConfig(spec v1alpha1.MachineSpec) (config string, nam return s, "vsphere", nil } -func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) @@ -576,11 +574,11 @@ func (p *provider) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]s return labels, err } -func (p *provider) SetMetricsForMachines(machines v1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { return nil } -func (p *provider) get(ctx context.Context, folder string, spec v1alpha1.MachineSpec, datacenterFinder *find.Finder) (*object.VirtualMachine, error) { +func (p *provider) get(ctx context.Context, folder string, spec clusterv1alpha1.MachineSpec, datacenterFinder *find.Finder) (*object.VirtualMachine, error) { path := fmt.Sprintf("%s/%s", folder, spec.Name) virtualMachineList, err := datacenterFinder.VirtualMachineList(ctx, path) if err != nil { diff --git a/pkg/cloudprovider/provider/vsphere/types/types.go b/pkg/cloudprovider/provider/vsphere/types/types.go index 62a0701a7..3645cfd74 100644 --- a/pkg/cloudprovider/provider/vsphere/types/types.go +++ b/pkg/cloudprovider/provider/vsphere/types/types.go @@ -17,6 +17,7 @@ limitations under the License. package types import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) @@ -40,3 +41,9 @@ type RawConfig struct { DiskSizeGB *int64 `json:"diskSizeGB,omitempty"` AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` } + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/jsonutil/strict.go b/pkg/jsonutil/strict.go new file mode 100644 index 000000000..48edabdcd --- /dev/null +++ b/pkg/jsonutil/strict.go @@ -0,0 +1,29 @@ +/* +Copyright 2019 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jsonutil + +import ( + "bytes" + "encoding/json" +) + +func StrictUnmarshal(buf []byte, obj interface{}) error { + dec := json.NewDecoder(bytes.NewReader(buf)) + dec.DisallowUnknownFields() + + return dec.Decode(obj) +} diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 6bf7063c5..925d1abdb 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -23,6 +23,7 @@ import ( "fmt" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "github.com/kubermatic/machine-controller/pkg/jsonutil" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -300,26 +301,33 @@ func (configVarBool *ConfigVarBool) UnmarshalJSON(b []byte) error { } var cvbDummy configVarBoolWithoutUnmarshaller + err := json.Unmarshal(b, &cvbDummy) if err != nil { return err } + configVarBool.Value = cvbDummy.Value configVarBool.SecretKeyRef = cvbDummy.SecretKeyRef configVarBool.ConfigMapKeyRef = cvbDummy.ConfigMapKeyRef + return nil } -func GetConfig(r clusterv1alpha1.ProviderSpec) (*Config, error) { - if r.Value == nil { +func GetConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, error) { + if provSpec.Value == nil { return nil, fmt.Errorf("machine.spec.providerSpec.value is nil") } - p := new(Config) - if len(r.Value.Raw) == 0 { - return p, nil + + var cfg Config + + if len(provSpec.Value.Raw) == 0 { + return &cfg, nil } - if err := json.Unmarshal(r.Value.Raw, p); err != nil { + + if err := jsonutil.StrictUnmarshal(provSpec.Value.Raw, &cfg); err != nil { return nil, err } - return p, nil + + return &cfg, nil } diff --git a/test/e2e/provisioning/testdata/machinedeployment-openstack-upgrade.yml b/test/e2e/provisioning/testdata/machinedeployment-openstack-upgrade.yml index d4d31334c..af6fcb47e 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-openstack-upgrade.yml +++ b/test/e2e/provisioning/testdata/machinedeployment-openstack-upgrade.yml @@ -35,13 +35,13 @@ spec: region: "<< REGION >>" network: "<< NETWORK_NAME >>" rootDiskSizeGB: 10 - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" instanceReadyCheckPeriod: 5s instanceReadyCheckTimeout: 2m operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: true disableAutoUpdate: true + rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml index 5ea15ba14..e22c05001 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml @@ -35,7 +35,6 @@ spec: cpus: 2 MemoryMB: 2048 diskSizeGB: << DISK_SIZE >> - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml index 3db382d2f..c2b42657f 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml @@ -36,9 +36,9 @@ spec: cpus: 2 MemoryMB: 2048 diskSizeGB: << DISK_SIZE >> - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false + rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml index 70b867f20..e46cafe48 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml @@ -35,11 +35,11 @@ spec: allowInsecure: true cpus: 2 MemoryMB: 2048 - rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false disableAutoUpdate: true + rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" network: cidr: "192.168.44.<< IP_OCTET >>/20" gateway: "192.168.32.1" From c7139027a1b48eed70609e11fe14f67ac5a3cc6c Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Thu, 17 Feb 2022 09:22:40 +0200 Subject: [PATCH 083/489] fix: returned vSphere RawConfig.Cluster (#1188) Removal of this field is backward compatibility issue. Signed-off-by: Artiom Diomin --- pkg/cloudprovider/provider/vsphere/types/types.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/provider/vsphere/types/types.go b/pkg/cloudprovider/provider/vsphere/types/types.go index 3645cfd74..29d54b6c8 100644 --- a/pkg/cloudprovider/provider/vsphere/types/types.go +++ b/pkg/cloudprovider/provider/vsphere/types/types.go @@ -29,8 +29,12 @@ type RawConfig struct { Password providerconfigtypes.ConfigVarString `json:"password"` VSphereURL providerconfigtypes.ConfigVarString `json:"vsphereURL"` Datacenter providerconfigtypes.ConfigVarString `json:"datacenter"` - Folder providerconfigtypes.ConfigVarString `json:"folder"` - ResourcePool providerconfigtypes.ConfigVarString `json:"resourcePool"` + + // Cluster is a noop field, it's not used anywhere but left here intentionally for backward compatibility purposes + Cluster providerconfigtypes.ConfigVarString `json:"cluster"` + + Folder providerconfigtypes.ConfigVarString `json:"folder"` + ResourcePool providerconfigtypes.ConfigVarString `json:"resourcePool"` // Either Datastore or DatastoreCluster have to be provided. DatastoreCluster providerconfigtypes.ConfigVarString `json:"datastoreCluster"` From 2b93e00a524b65f025a3c5e05d52fc812c791d3a Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Fri, 18 Feb 2022 09:58:39 +0100 Subject: [PATCH 084/489] Improve nil pointer checks and consider VPC overlay networks (#1193) Signed-off-by: Marvin Beckers --- pkg/cloudprovider/provider/nutanix/client.go | 24 ++++++++----------- .../provider/nutanix/provider.go | 4 ++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pkg/cloudprovider/provider/nutanix/client.go b/pkg/cloudprovider/provider/nutanix/client.go index 760dab46b..c57a07187 100644 --- a/pkg/cloudprovider/provider/nutanix/client.go +++ b/pkg/cloudprovider/provider/nutanix/client.go @@ -232,8 +232,11 @@ func getSubnetByName(client *ClientSet, name, clusterID string) (*nutanixv3.Subn } for _, subnet := range subnets.Entities { - if *subnet.Status.Name == name && *subnet.Status.ClusterReference.UUID == clusterID { - return subnet, nil + if subnet != nil && subnet.Status != nil && subnet.Status.Name != nil && *subnet.Status.Name == name { + // some subnet types (e.g. VPC overlays) do not come with a cluster reference; we don't need to check them + if subnet.Status.ClusterReference == nil || (subnet.Status.ClusterReference.UUID != nil && *subnet.Status.ClusterReference.UUID == clusterID) { + return subnet, nil + } } } @@ -260,15 +263,7 @@ func getProjectByName(client *ClientSet, name string) (*nutanixv3.Project, error } for _, project := range projects.Entities { - if project == nil { - return nil, errors.New("project is nil") - } - - if project.Status == nil { - return nil, errors.New("project status is nil") - } - - if project.Status.Name == name { + if project != nil && project.Status != nil && project.Status.Name == name { return project, nil } } @@ -295,7 +290,7 @@ func getClusterByName(client *ClientSet, name string) (*nutanixv3.ClusterIntentR } for _, cluster := range clusters.Entities { - if *cluster.Status.Name == name { + if cluster.Status != nil && cluster.Status.Name != nil && *cluster.Status.Name == name { return cluster, nil } } @@ -322,7 +317,7 @@ func getImageByName(client *ClientSet, name string) (*nutanixv3.ImageIntentRespo } for _, image := range images.Entities { - if *image.Status.Name == name { + if image.Status != nil && image.Status.Name != nil && *image.Status.Name == name { return image, nil } } @@ -343,7 +338,8 @@ func getVMByName(client *ClientSet, name string, projectID *string) (*nutanixv3. for _, vm := range vms.Entities { if *vm.Status.Name == name { - if projectID != nil && *vm.Metadata.ProjectReference.UUID != *projectID { + if projectID != nil && vm.Metadata != nil && vm.Metadata.ProjectReference != nil && + vm.Metadata.ProjectReference.UUID != nil && *vm.Metadata.ProjectReference.UUID != *projectID { continue } return vm, nil diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 6e2a9f2b1..5ad6b0212 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -315,6 +315,10 @@ func (p *provider) cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider return false, err } + if vm.Metadata == nil || vm.Metadata.UUID == nil { + return false, fmt.Errorf("failed to get valid VM metadata for machine '%s'", machine.Name) + } + // TODO: figure out if VM is already in deleting state resp, err := client.Prism.V3.DeleteVM(*vm.Metadata.UUID) From be72ced4a7fb366add8403b8c28f2223a0e3862d Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Mon, 28 Feb 2022 12:34:21 +0100 Subject: [PATCH 085/489] validate max price in spot instance request (#1198) Signed-off-by: Moath Qasim --- pkg/cloudprovider/provider/aws/provider.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 00bad52ca..c9e243043 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -633,6 +633,12 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return fmt.Errorf("failed to validate instance profile: %v", err) } + if config.IsSpotInstance != nil && *config.IsSpotInstance { + if config.SpotMaxPrice == nil { + return errors.New("failed to validate max price for the spot instance: max price cannot be empty when spot instance ") + } + } + return nil } From e479c57203c1e2f20c1cdef2576f87496ab9eff2 Mon Sep 17 00:00:00 2001 From: Helene Durand <90329641+hdurand0710@users.noreply.github.com> Date: Wed, 2 Mar 2022 09:42:04 +0100 Subject: [PATCH 086/489] Fix wrong CPU config (#1200) Signed-off-by: Helene Durand --- pkg/cloudprovider/provider/kubevirt/provider.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 2222a71b4..5dd26ff1a 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -365,9 +365,6 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, _ *cloudprovidertype }, Spec: kubevirtv1.VirtualMachineInstanceSpec{ Domain: kubevirtv1.DomainSpec{ - CPU: &kubevirtv1.CPU{ - Cores: 2, - }, Devices: kubevirtv1.Devices{ Disks: []kubevirtv1.Disk{ { From e35da15605475924fd8b92a1f85f07d0c3e4a627 Mon Sep 17 00:00:00 2001 From: Mattia Lavacca Date: Thu, 10 Mar 2022 15:23:05 +0100 Subject: [PATCH 087/489] Waiting for volumeAttachments deletion (#1190) * Waiting for volumeAttachments deletion Signed-off-by: Mattia Lavacca * volumeAttachments check only for vSphere Signed-off-by: Mattia Lavacca * ClusterRole updated Signed-off-by: Mattia Lavacca * yaml linter fixed Signed-off-by: Mattia Lavacca * VolumeAttachments correctly handled Signed-off-by: Mattia Lavacca * Code factorized Signed-off-by: Mattia Lavacca * renaming Signed-off-by: Mattia Lavacca * fix yamllint Signed-off-by: Mattia Lavacca * Logic applied only to vSphere Signed-off-by: Mattia Lavacca --- examples/machine-controller.yaml | 9 + pkg/controller/machine/machine_controller.go | 141 ++++++++---- pkg/controller/machine/machine_test.go | 53 +++-- pkg/node/eviction/eviction.go | 70 ++---- pkg/node/nodemanager/node_manager.go | 94 ++++++++ pkg/node/poddeletion/pod_deletion.go | 216 +++++++++++++++++++ 6 files changed, 458 insertions(+), 125 deletions(-) create mode 100644 pkg/node/nodemanager/node_manager.go create mode 100644 pkg/node/poddeletion/pod_deletion.go diff --git a/examples/machine-controller.yaml b/examples/machine-controller.yaml index 084d92d20..d3ee84c21 100644 --- a/examples/machine-controller.yaml +++ b/examples/machine-controller.yaml @@ -536,6 +536,15 @@ rules: - "list" - "get" - "watch" +# volumeAttachments permissions are needed by vsphere clusters +- apiGroups: + - "storage.k8s.io" + resources: + - "volumeattachments" + verbs: + - "list" + - "get" + - "watch" - apiGroups: - "" resources: diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 7431d2634..d7977bf4d 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -39,6 +39,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/containerruntime" kuberneteshelper "github.com/kubermatic/machine-controller/pkg/kubernetes" "github.com/kubermatic/machine-controller/pkg/node/eviction" + "github.com/kubermatic/machine-controller/pkg/node/poddeletion" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" "github.com/kubermatic/machine-controller/pkg/rhsm" @@ -408,7 +409,7 @@ func (r *Reconciler) reconcile(ctx context.Context, machine *clusterv1alpha1.Mac // step 2: check if a user requested to delete the machine if machine.DeletionTimestamp != nil { - return r.deleteMachine(ctx, prov, machine) + return r.deleteMachine(ctx, prov, providerConfig.CloudProvider, machine) } // Step 3: Essentially creates an instance for the given machine. @@ -462,6 +463,30 @@ func (r *Reconciler) ensureMachineHasNodeReadyCondition(machine *clusterv1alpha1 }) } +func (r *Reconciler) shouldCleanupVolumes(ctx context.Context, machine *clusterv1alpha1.Machine, providerName providerconfigtypes.CloudProvider) (bool, error) { + // we need to wait for volumeAttachments clean up only for vSphere + if providerName != providerconfigtypes.CloudProviderVsphere { + return false, nil + } + + // No node - No volumeAttachments to be collected + if machine.Status.NodeRef == nil { + klog.V(4).Infof("Skipping eviction for machine %q since it does not have a node", machine.Name) + return false, nil + } + + node := &corev1.Node{} + if err := r.client.Get(ctx, types.NamespacedName{Name: machine.Status.NodeRef.Name}, node); err != nil { + // Node does not exist - No volumeAttachments to be collected + if kerrors.IsNotFound(err) { + klog.V(4).Infof("Skipping eviction for machine %q since it does not have a node", machine.Name) + return false, nil + } + return false, fmt.Errorf("failed to get node %q", machine.Status.NodeRef.Name) + } + return true, nil +} + // evictIfNecessary checks if the machine has a node and evicts it if necessary func (r *Reconciler) shouldEvict(ctx context.Context, machine *clusterv1alpha1.Machine) (bool, error) { // If the deletion got triggered a few hours ago, skip eviction. @@ -521,22 +546,35 @@ func (r *Reconciler) shouldEvict(ctx context.Context, machine *clusterv1alpha1.M } // deleteMachine makes sure that an instance has gone in a series of steps. -func (r *Reconciler) deleteMachine(ctx context.Context, prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine) (*reconcile.Result, error) { +func (r *Reconciler) deleteMachine(ctx context.Context, prov cloudprovidertypes.Provider, providerName providerconfigtypes.CloudProvider, machine *clusterv1alpha1.Machine) (*reconcile.Result, error) { shouldEvict, err := r.shouldEvict(ctx, machine) if err != nil { return nil, err } + shouldCleanUpVolumes, err := r.shouldCleanupVolumes(ctx, machine, providerName) + if err != nil { + return nil, err + } + var evictedSomething, deletedSomething bool + var volumesFree = true if shouldEvict { - evictedSomething, err := eviction.New(ctx, machine.Status.NodeRef.Name, r.client, r.kubeClient).Run() + evictedSomething, err = eviction.New(ctx, machine.Status.NodeRef.Name, r.client, r.kubeClient).Run() if err != nil { return nil, fmt.Errorf("failed to evict node %s: %v", machine.Status.NodeRef.Name, err) } - if evictedSomething { - return &reconcile.Result{RequeueAfter: 10 * time.Second}, nil + } + if shouldCleanUpVolumes { + deletedSomething, volumesFree, err = poddeletion.New(ctx, machine.Status.NodeRef.Name, r.client, r.kubeClient).Run() + if err != nil { + return nil, fmt.Errorf("failed to delete pods bound to volumes running on node %s: %v", machine.Status.NodeRef.Name, err) } } + if evictedSomething || deletedSomething || !volumesFree { + return &reconcile.Result{RequeueAfter: 10 * time.Second}, nil + } + if result, err := r.deleteCloudProviderInstance(prov, machine); result != nil || err != nil { return result, err } @@ -550,7 +588,52 @@ func (r *Reconciler) deleteMachine(ctx context.Context, prov cloudprovidertypes. return nil, nil } - return nil, r.deleteNodeForMachine(ctx, machine) + nodes, err := r.retrieveNodesRelatedToMachine(ctx, machine) + if err != nil { + return nil, err + } + + return nil, r.deleteNodeForMachine(ctx, nodes, machine) +} + +func (r *Reconciler) retrieveNodesRelatedToMachine(ctx context.Context, machine *clusterv1alpha1.Machine) ([]*corev1.Node, error) { + nodes := make([]*corev1.Node, 0) + + // If there's NodeRef on the Machine object, retrieve the node by using the + // value of the NodeRef. If there's no NodeRef, try to find the Node by + // listing nodes using the NodeOwner label selector. + if machine.Status.NodeRef != nil { + objKey := ctrlruntimeclient.ObjectKey{Name: machine.Status.NodeRef.Name} + node := &corev1.Node{} + if err := r.client.Get(ctx, objKey, node); err != nil { + if !kerrors.IsNotFound(err) { + return nil, fmt.Errorf("failed to get node %s: %v", machine.Status.NodeRef.Name, err) + } + klog.V(2).Infof("node %q does not longer exist for machine %q", machine.Status.NodeRef.Name, machine.Spec.Name) + } else { + nodes = append(nodes, node) + } + } else { + selector, err := labels.Parse(NodeOwnerLabelName + "=" + string(machine.UID)) + if err != nil { + return nil, fmt.Errorf("failed to parse label selector: %v", err) + } + listOpts := &ctrlruntimeclient.ListOptions{LabelSelector: selector} + nodeList := &corev1.NodeList{} + if err := r.client.List(ctx, nodeList, listOpts); err != nil { + return nil, fmt.Errorf("failed to list nodes: %v", err) + } + if len(nodeList.Items) == 0 { + // We just want log that we didn't found the node. + klog.V(3).Infof("No node found for the machine %s", machine.Spec.Name) + } + + for _, node := range nodeList.Items { + nodes = append(nodes, &node) + } + } + + return nodes, nil } func (r *Reconciler) deleteCloudProviderInstance(prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine) (*reconcile.Result, error) { @@ -623,50 +706,14 @@ func (r *Reconciler) deleteCloudProviderInstance(prov cloudprovidertypes.Provide }) } -func (r *Reconciler) deleteNodeForMachine(ctx context.Context, machine *clusterv1alpha1.Machine) error { - // If there's NodeRef on the Machine object, remove the Node by using the - // value of the NodeRef. If there's no NodeRef, try to find the Node by - // listing nodes using the NodeOwner label selector. - if machine.Status.NodeRef != nil { - objKey := ctrlruntimeclient.ObjectKey{Name: machine.Status.NodeRef.Name} - node := &corev1.Node{} - nodeFound := true - if err := r.client.Get(ctx, objKey, node); err != nil { +func (r *Reconciler) deleteNodeForMachine(ctx context.Context, nodes []*corev1.Node, machine *clusterv1alpha1.Machine) error { + // iterates on all nodes and delete them. Finally, remove the finalizer on the machine + for _, node := range nodes { + if err := r.client.Delete(ctx, node); err != nil { if !kerrors.IsNotFound(err) { - return fmt.Errorf("failed to get node %s: %v", machine.Status.NodeRef.Name, err) - } - nodeFound = false - klog.V(2).Infof("node %q does not longer exist for machine %q", machine.Status.NodeRef.Name, machine.Spec.Name) - } - - if nodeFound { - if err := r.client.Delete(ctx, node); err != nil { - if !kerrors.IsNotFound(err) { - return err - } - klog.V(2).Infof("node %q does not longer exist for machine %q", machine.Status.NodeRef.Name, machine.Spec.Name) - } - } - } else { - selector, err := labels.Parse(NodeOwnerLabelName + "=" + string(machine.UID)) - if err != nil { - return fmt.Errorf("failed to parse label selector: %v", err) - } - listOpts := &ctrlruntimeclient.ListOptions{LabelSelector: selector} - nodes := &corev1.NodeList{} - if err := r.client.List(ctx, nodes, listOpts); err != nil { - return fmt.Errorf("failed to list nodes: %v", err) - } - if len(nodes.Items) == 0 { - // We just want log that we didn't found the node. We don't want to - // return here, as we want to remove finalizers at the end. - klog.V(3).Infof("No node found for the machine %s", machine.Spec.Name) - } - - for _, node := range nodes.Items { - if err := r.client.Delete(ctx, &node); err != nil { return err } + klog.V(2).Infof("node %q does not longer exist for machine %q", machine.Status.NodeRef.Name, machine.Spec.Name) } } diff --git a/pkg/controller/machine/machine_test.go b/pkg/controller/machine/machine_test.go index 7f4a65d91..a9322ca32 100644 --- a/pkg/controller/machine/machine_test.go +++ b/pkg/controller/machine/machine_test.go @@ -474,7 +474,7 @@ func TestControllerDeleteNodeForMachine(t *testing.T) { tests := []struct { name string machine *clusterv1alpha1.Machine - nodes []runtime.Object + nodes []*corev1.Node err error shouldDeleteNode string }{ @@ -489,13 +489,17 @@ func TestControllerDeleteNodeForMachine(t *testing.T) { NodeRef: &corev1.ObjectReference{Name: "node-1"}, }, }, - nodes: []runtime.Object{&corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-0", - }}, &corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-1", - }}, + nodes: []*corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-0", + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + }, + }, }, err: nil, shouldDeleteNode: "node-1", @@ -510,8 +514,8 @@ func TestControllerDeleteNodeForMachine(t *testing.T) { }, Status: clusterv1alpha1.MachineStatus{}, }, - nodes: []runtime.Object{ - &corev1.Node{ + nodes: []*corev1.Node{ + { ObjectMeta: metav1.ObjectMeta{ Name: "node-0", Labels: map[string]string{ @@ -519,7 +523,7 @@ func TestControllerDeleteNodeForMachine(t *testing.T) { }, }, }, - &corev1.Node{ + { ObjectMeta: metav1.ObjectMeta{ Name: "node-1", }, @@ -538,13 +542,13 @@ func TestControllerDeleteNodeForMachine(t *testing.T) { }, Status: clusterv1alpha1.MachineStatus{}, }, - nodes: []runtime.Object{ - &corev1.Node{ + nodes: []*corev1.Node{ + { ObjectMeta: metav1.ObjectMeta{ Name: "node-0", }, }, - &corev1.Node{ + { ObjectMeta: metav1.ObjectMeta{ Name: "node-1", }, @@ -564,10 +568,12 @@ func TestControllerDeleteNodeForMachine(t *testing.T) { NodeRef: &corev1.ObjectReference{Name: "node-1"}, }, }, - nodes: []runtime.Object{&corev1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node-0", - }}, + nodes: []*corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-0", + }, + }, }, err: nil, shouldDeleteNode: "", @@ -579,7 +585,9 @@ func TestControllerDeleteNodeForMachine(t *testing.T) { ctx := context.Background() objects := []runtime.Object{test.machine} - objects = append(objects, test.nodes...) + for _, n := range test.nodes { + objects = append(objects, n) + } client := ctrlruntimefake.NewFakeClient(objects...) @@ -595,7 +603,12 @@ func TestControllerDeleteNodeForMachine(t *testing.T) { providerData: providerData, } - err := reconciler.deleteNodeForMachine(ctx, test.machine) + nodes, err := reconciler.retrieveNodesRelatedToMachine(ctx, test.machine) + if err != nil { + return + } + + err = reconciler.deleteNodeForMachine(ctx, nodes, test.machine) if diff := deep.Equal(err, test.err); diff != nil { t.Errorf("expected to get %v instead got: %v", test.err, err) } diff --git a/pkg/node/eviction/eviction.go b/pkg/node/eviction/eviction.go index 4962770ec..e6d1a2024 100644 --- a/pkg/node/eviction/eviction.go +++ b/pkg/node/eviction/eviction.go @@ -20,44 +20,41 @@ import ( "context" "fmt" "sync" - "time" evictiontypes "github.com/kubermatic/machine-controller/pkg/node/eviction/types" + "github.com/kubermatic/machine-controller/pkg/node/nodemanager" corev1 "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1beta1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/util/retry" "k8s.io/klog" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) type NodeEviction struct { - ctx context.Context - nodeName string - client ctrlruntimeclient.Client - kubeClient kubernetes.Interface + nodeManager *nodemanager.NodeManager + ctx context.Context + nodeName string + kubeClient kubernetes.Interface } // New returns a new NodeEviction func New(ctx context.Context, nodeName string, client ctrlruntimeclient.Client, kubeClient kubernetes.Interface) *NodeEviction { return &NodeEviction{ - ctx: ctx, - nodeName: nodeName, - client: client, - kubeClient: kubeClient, + nodeManager: nodemanager.New(ctx, client, nodeName), + ctx: ctx, + nodeName: nodeName, + kubeClient: kubeClient, } } // Run executes the eviction func (ne *NodeEviction) Run() (bool, error) { - node := &corev1.Node{} - if err := ne.client.Get(ne.ctx, types.NamespacedName{Name: ne.nodeName}, node); err != nil { + node, err := ne.nodeManager.GetNode() + if err != nil { return false, fmt.Errorf("failed to get node from lister: %v", err) } if _, exists := node.Annotations[evictiontypes.SkipEvictionAnnotationKey]; exists { @@ -66,7 +63,7 @@ func (ne *NodeEviction) Run() (bool, error) { } klog.V(3).Infof("Starting to evict node %s", ne.nodeName) - if err := ne.cordonNode(node); err != nil { + if err := ne.nodeManager.CordonNode(node); err != nil { return false, fmt.Errorf("failed to cordon node %s: %v", ne.nodeName, err) } klog.V(6).Infof("Successfully cordoned node %s", ne.nodeName) @@ -90,34 +87,6 @@ func (ne *NodeEviction) Run() (bool, error) { return true, nil } -func (ne *NodeEviction) cordonNode(node *corev1.Node) error { - if !node.Spec.Unschedulable { - _, err := ne.updateNode(func(n *corev1.Node) { - n.Spec.Unschedulable = true - }) - if err != nil { - return err - } - } - - // Be paranoid and wait until the change got propagated to the lister - // This assumes that the delay between our lister and the APIserver - // is smaller or equal to the delay the schedulers lister has - If - // that is not the case, there is a small chance the scheduler schedules - // pods in between, those will then get deleted upon node deletion and - // not evicted - return wait.Poll(1*time.Second, 10*time.Second, func() (bool, error) { - node := &corev1.Node{} - if err := ne.client.Get(ne.ctx, types.NamespacedName{Name: ne.nodeName}, node); err != nil { - return false, err - } - if node.Spec.Unschedulable { - return true, nil - } - return false, nil - }) -} - func (ne *NodeEviction) getFilteredPods() ([]corev1.Pod, error) { // The lister-backed client from the mgr automatically creates a lister for all objects requested through it. // We explicitly do not want that for pods, hence we have to use the kubernetes core client @@ -202,18 +171,3 @@ func (ne *NodeEviction) evictPod(pod *corev1.Pod) error { } return ne.kubeClient.PolicyV1beta1().Evictions(eviction.Namespace).Evict(ne.ctx, eviction) } - -func (ne *NodeEviction) updateNode(modify func(*corev1.Node)) (*corev1.Node, error) { - node := &corev1.Node{} - err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - if err := ne.client.Get(ne.ctx, types.NamespacedName{Name: ne.nodeName}, node); err != nil { - return err - } - // Apply modifications - modify(node) - // Update the node - return ne.client.Update(ne.ctx, node) - }) - - return node, err -} diff --git a/pkg/node/nodemanager/node_manager.go b/pkg/node/nodemanager/node_manager.go new file mode 100644 index 000000000..342a2bf57 --- /dev/null +++ b/pkg/node/nodemanager/node_manager.go @@ -0,0 +1,94 @@ +/* +Copyright 2019 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodemanager + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/util/retry" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +type NodeManager struct { + ctx context.Context + client ctrlruntimeclient.Client + nodeName string +} + +func New(ctx context.Context, client ctrlruntimeclient.Client, nodeName string) *NodeManager { + return &NodeManager{ + ctx: ctx, + client: client, + nodeName: nodeName, + } +} + +func (nm *NodeManager) GetNode() (*corev1.Node, error) { + node := &corev1.Node{} + if err := nm.client.Get(nm.ctx, types.NamespacedName{Name: nm.nodeName}, node); err != nil { + return nil, fmt.Errorf("failed to get node from lister: %v", err) + } + return node, nil +} + +func (nm *NodeManager) CordonNode(node *corev1.Node) error { + if !node.Spec.Unschedulable { + _, err := nm.updateNode(func(n *corev1.Node) { + n.Spec.Unschedulable = true + }) + if err != nil { + return err + } + } + + // Be paranoid and wait until the change got propagated to the lister + // This assumes that the delay between our lister and the APIserver + // is smaller or equal to the delay the schedulers lister has - If + // that is not the case, there is a small chance the scheduler schedules + // pods in between, those will then get deleted upon node deletion and + // not evicted + return wait.Poll(1*time.Second, 10*time.Second, func() (bool, error) { + node := &corev1.Node{} + if err := nm.client.Get(nm.ctx, types.NamespacedName{Name: nm.nodeName}, node); err != nil { + return false, err + } + if node.Spec.Unschedulable { + return true, nil + } + return false, nil + }) +} + +func (nm *NodeManager) updateNode(modify func(*corev1.Node)) (*corev1.Node, error) { + node := &corev1.Node{} + err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + if err := nm.client.Get(nm.ctx, types.NamespacedName{Name: nm.nodeName}, node); err != nil { + return err + } + // Apply modifications + modify(node) + // Update the node + return nm.client.Update(nm.ctx, node) + }) + + return node, err +} diff --git a/pkg/node/poddeletion/pod_deletion.go b/pkg/node/poddeletion/pod_deletion.go new file mode 100644 index 000000000..1b9874aa8 --- /dev/null +++ b/pkg/node/poddeletion/pod_deletion.go @@ -0,0 +1,216 @@ +/* +Copyright 2019 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package poddeletion + +import ( + "context" + "fmt" + "sync" + + "github.com/kubermatic/machine-controller/pkg/node/nodemanager" + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + errorQueueLen = 100 +) + +type NodeVolumeAttachmentsCleanup struct { + nodeManager *nodemanager.NodeManager + ctx context.Context + nodeName string + kubeClient kubernetes.Interface +} + +// New returns a new NodeVolumeAttachmentsCleanup +func New(ctx context.Context, nodeName string, client ctrlruntimeclient.Client, kubeClient kubernetes.Interface) *NodeVolumeAttachmentsCleanup { + return &NodeVolumeAttachmentsCleanup{ + nodeManager: nodemanager.New(ctx, client, nodeName), + ctx: ctx, + nodeName: nodeName, + kubeClient: kubeClient, + } +} + +// Run executes the pod deletion +func (vc *NodeVolumeAttachmentsCleanup) Run() (bool, bool, error) { + node, err := vc.nodeManager.GetNode() + if err != nil { + return false, false, fmt.Errorf("failed to get node from lister: %v", err) + } + klog.V(3).Infof("Starting to cleanup node %s", vc.nodeName) + + // if there are no more volumeAttachments related to the node, then it can be deleted + volumeAttachmentsDeleted, err := vc.nodeCanBeDeleted() + if err != nil { + return false, false, fmt.Errorf("failed to check volumeAttachments deletion: %v", err) + } + if volumeAttachmentsDeleted { + return false, true, nil + } + + // cordon the node to be sure that the deleted pods are re-scheduled in the same node + if err := vc.nodeManager.CordonNode(node); err != nil { + return false, false, fmt.Errorf("failed to cordon node %s: %v", vc.nodeName, err) + } + klog.V(6).Infof("Successfully cordoned node %s", vc.nodeName) + + // get all the pods that needs to be deleted (i.e. those mounting volumes attached to the node that is going to be deleted) + podsToDelete, errors := vc.getFilteredPods() + if len(errors) > 0 { + return false, false, fmt.Errorf("failed to get Pods to delete for node %s, errors encountered: %v", vc.nodeName, err) + } + klog.V(6).Infof("Found %v pods to delete for node %s", len(podsToDelete), vc.nodeName) + + if len(podsToDelete) == 0 { + return false, false, nil + } + + // delete the previously filtered pods, then tells the controller to retry later + if errs := vc.deletePods(podsToDelete); len(errs) > 0 { + return false, false, fmt.Errorf("failed to delete pods, errors encountered: %v", errs) + } + klog.V(6).Infof("Successfully deleted all pods mounting persistent volumes attached on node %s", vc.nodeName) + return true, false, err +} + +func (vc *NodeVolumeAttachmentsCleanup) getFilteredPods() ([]corev1.Pod, []error) { + filteredPods := []corev1.Pod{} + lock := sync.Mutex{} + retErrs := []error{} + + volumeAttachments, err := vc.kubeClient.StorageV1().VolumeAttachments().List(vc.ctx, metav1.ListOptions{}) + if err != nil { + retErrs = append(retErrs, fmt.Errorf("failed to list pods: %v", err)) + return nil, retErrs + } + + persistentVolumeClaims, err := vc.kubeClient.CoreV1().PersistentVolumeClaims(metav1.NamespaceAll).List(vc.ctx, metav1.ListOptions{}) + if err != nil { + retErrs = append(retErrs, fmt.Errorf("failed to list persistent volumes: %v", err)) + return nil, retErrs + } + + errCh := make(chan error, errorQueueLen) + wg := sync.WaitGroup{} + for _, va := range volumeAttachments.Items { + if va.Spec.NodeName == vc.nodeName { + for _, pvc := range persistentVolumeClaims.Items { + if va.Spec.Source.PersistentVolumeName != nil && *va.Spec.Source.PersistentVolumeName == pvc.Spec.VolumeName { + wg.Add(1) + go func(pvc corev1.PersistentVolumeClaim) { + defer wg.Done() + pods, err := vc.kubeClient.CoreV1().Pods(pvc.Namespace).List(vc.ctx, metav1.ListOptions{}) + switch { + case kerrors.IsTooManyRequests(err): + return + case err != nil: + errCh <- fmt.Errorf("failed to list pod: %v", err) + default: + for _, pod := range pods.Items { + if doesPodClaimVolume(pod, pvc.Name) && pod.Spec.NodeName == vc.nodeName { + lock.Lock() + filteredPods = append(filteredPods, pod) + lock.Unlock() + } + } + } + }(pvc) + } + } + } + } + wg.Wait() + close(errCh) + + for err := range errCh { + retErrs = append(retErrs, err) + } + + return filteredPods, nil +} + +// nodeCanBeDeleted checks if all the volumeAttachments related to the node have already been collected by the external CSI driver +func (vc *NodeVolumeAttachmentsCleanup) nodeCanBeDeleted() (bool, error) { + volumeAttachments, err := vc.kubeClient.StorageV1().VolumeAttachments().List(vc.ctx, metav1.ListOptions{}) + if err != nil { + return false, fmt.Errorf("error while listing volumeAttachments: %v", err) + } + for _, va := range volumeAttachments.Items { + if va.Spec.NodeName == vc.nodeName { + klog.V(3).Infof("waiting for the volumeAttachment %s to be deleted before deleting node %s", va.Name, vc.nodeName) + return false, nil + } + } + return true, nil +} + +func (vc *NodeVolumeAttachmentsCleanup) deletePods(pods []corev1.Pod) []error { + + errCh := make(chan error, len(pods)) + retErrs := []error{} + + var wg sync.WaitGroup + var isDone bool + defer func() { isDone = true }() + + wg.Add(len(pods)) + for _, pod := range pods { + go func(p corev1.Pod) { + defer wg.Done() + for { + if isDone { + return + } + err := vc.kubeClient.CoreV1().Pods(p.Namespace).Delete(vc.ctx, p.Name, metav1.DeleteOptions{}) + if err == nil || kerrors.IsNotFound(err) { + klog.V(6).Infof("Successfully deleted pod %s/%s on node %s", p.Namespace, p.Name, vc.nodeName) + return + } else if kerrors.IsTooManyRequests(err) { + // PDB prevents pod deletion, return and make the controller retry later + return + } else { + errCh <- fmt.Errorf("error deleting pod %s/%s on node %s: %v", p.Namespace, p.Name, vc.nodeName, err) + return + } + } + }(pod) + } + wg.Wait() + close(errCh) + + for err := range errCh { + retErrs = append(retErrs, err) + } + + return retErrs +} + +// doesPodClaimVolume checks if the volume is mounted by the pod +func doesPodClaimVolume(pod corev1.Pod, pvcName string) bool { + for _, volumeMount := range pod.Spec.Volumes { + if volumeMount.PersistentVolumeClaim != nil && volumeMount.PersistentVolumeClaim.ClaimName == pvcName { + return true + } + } + return false +} From 72f156a2c9c7857e29bd019f740e7742e94a9cc3 Mon Sep 17 00:00:00 2001 From: Sankalp Rangare Date: Mon, 14 Mar 2022 19:39:08 +0100 Subject: [PATCH 088/489] update in kubervirt rawconfig (#1143) * update in kubervirt rawconfig Signed-off-by: Sankalp Rangare * rawConfig field renaming Signed-off-by: Sankalp Rangare * update kubevirt provider Config Signed-off-by: Sankalp Rangare * add namespace option in CloudConfig Signed-off-by: Sankalp Rangare * create VM in dedicated ns Signed-off-by: Sankalp Rangare * remove vmName from rawConfig Signed-off-by: Sankalp Rangare --- .../provider/kubevirt/provider.go | 298 ++++++++++++------ .../provider/kubevirt/types/cloudconfig.go | 2 + .../provider/kubevirt/types/types.go | 54 +++- .../testdata/machinedeployment-kubevirt.yaml | 27 +- 4 files changed, 266 insertions(+), 115 deletions(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 5dd26ff1a..282916f28 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "os" "strconv" "strings" "time" @@ -35,7 +36,6 @@ import ( cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -46,6 +46,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/klog" utilpointer "k8s.io/utils/pointer" + "net/url" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -79,9 +80,21 @@ type Config struct { CPUs string Memory string Namespace string - SourceURL string + OsImage OSImage StorageClassName string PVCSize resource.Quantity + FlavorName string + SecondaryDisks []SecondaryDisks +} + +type SecondaryDisks struct { + Size resource.Quantity + StorageClassName string +} + +type OSImage struct { + URL string + DataVolumeName string } type kubeVirtServer struct { @@ -135,34 +148,36 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } config := Config{} - config.Kubeconfig, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Kubeconfig, "KUBEVIRT_KUBECONFIG") + config.Kubeconfig, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Auth.Kubeconfig, "KUBEVIRT_KUBECONFIG") if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "kubeconfig" field: %v`, err) } - config.CPUs, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.CPUs) + config.CPUs, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.CPUs) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "cpus" field: %v`, err) } - config.Memory, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Memory) + config.Memory, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.Memory) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "memory" field: %v`, err) } - config.Namespace, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Namespace) - if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "namespace" field: %v`, err) - } - config.SourceURL, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.SourceURL) + config.Namespace = getNamespace() + osImage, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.OsImage) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "sourceURL" field: %v`, err) } - pvcSize, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.PVCSize) + if _, err = url.ParseRequestURI(osImage); err == nil { + config.OsImage.URL = osImage + } else { + config.OsImage.DataVolumeName = osImage + } + pvcSize, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.Size) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "pvcSize" field: %v`, err) } if config.PVCSize, err = resource.ParseQuantity(pvcSize); err != nil { return nil, nil, fmt.Errorf(`failed to parse value of "pvcSize" field: %v`, err) } - config.StorageClassName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.StorageClassName) + config.StorageClassName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.StorageClassName) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "storageClassName" field: %v`, err) } @@ -170,8 +185,12 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if err != nil { return nil, nil, fmt.Errorf("failed to decode kubeconfig: %v", err) } + config.FlavorName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Flavor.Name) + if err != nil { + return nil, nil, fmt.Errorf(`failed to get value of "flavor.name" field: %v`, err) + } - dnsPolicyString, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.DNSPolicy) + dnsPolicyString, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.DNSPolicy) if err != nil { return nil, nil, fmt.Errorf(`failed to parse "dnsPolicy" field: %v`, err) } @@ -181,13 +200,38 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf("failed to get dns policy: %v", err) } } - if rawConfig.DNSConfig != nil { - config.DNSConfig = rawConfig.DNSConfig + if rawConfig.VirtualMachine.DNSConfig != nil { + config.DNSConfig = rawConfig.VirtualMachine.DNSConfig + } + if len(rawConfig.VirtualMachine.Template.SecondaryDisks) > 0 { + for _, sd := range rawConfig.VirtualMachine.Template.SecondaryDisks { + pvc, err := resource.ParseQuantity(sd.Size.Value) + if err != nil { + return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.size" field: %v`, err) + } + config.SecondaryDisks = append(config.SecondaryDisks, SecondaryDisks{ + Size: pvc, + StorageClassName: sd.StorageClassName.Value, + }) + } } return &config, pconfig, nil } +// getNamespace returns the namespace where the VM is created. +// VM is created in a dedicated namespace +// which is the namespace where the machine-controller pod is running. +// Defaults to `kube-system`. +func getNamespace() string { + ns := os.Getenv("POD_NAMESPACE") + if ns == "" { + // Useful especially for ci tests. + ns = metav1.NamespaceSystem + } + return ns +} + func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { @@ -254,9 +298,13 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { if err != nil { return fmt.Errorf("failed to parse config: %v", err) } - if _, err := parseResources(c.CPUs, c.Memory); err != nil { - return err + // If VMIPreset is specified, skip CPU and Memory validation + if c.FlavorName == "" { + if _, err := parseResources(c.CPUs, c.Memory); err != nil { + return err + } } + sigClient, err := client.New(c.RestConfig, client.Options{}) if err != nil { return fmt.Errorf("failed to get kubevirt client: %v", err) @@ -302,7 +350,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s if err == nil { labels["cpus"] = c.CPUs labels["memoryMIB"] = c.Memory - labels["sourceURL"] = c.SourceURL + labels["osImage"] = c.OsImage.URL } return labels, err @@ -322,16 +370,37 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, _ *cloudprovidertype // The secret has an ownerRef on the VMI so garbace collection will take care of cleaning up terminationGracePeriodSeconds := int64(30) userDataSecretName := fmt.Sprintf("userdata-%s-%s", machine.Name, strconv.Itoa(int(time.Now().Unix()))) - requestsAndLimits, err := parseResources(c.CPUs, c.Memory) + + resourceRequirements := kubevirtv1.ResourceRequirements{} + labels := map[string]string{"kubevirt.io/vm": machine.Name} + + sigClient, err := client.New(c.RestConfig, client.Options{}) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to get kubevirt client: %v", err) + } + ctx := context.Background() + + // Add VMIPreset label if specified + if c.FlavorName != "" { + vmiPreset := kubevirtv1.VirtualMachineInstancePreset{} + if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: c.FlavorName}, &vmiPreset); err != nil { + return nil, err + } + for key, val := range vmiPreset.Spec.Selector.MatchLabels { + labels[key] = val + } + } else { + requestsAndLimits, err := parseResources(c.CPUs, c.Memory) + if err != nil { + return nil, err + } + resourceRequirements.Requests = *requestsAndLimits + resourceRequirements.Limits = *requestsAndLimits } var ( - pvcRequest = corev1.ResourceList{corev1.ResourceStorage: c.PVCSize} dataVolumeName = machine.Name - - annotations map[string]string + annotations map[string]string ) if pc.OperatingSystem == providerconfigtypes.OperatingSystemFlatcar { @@ -350,95 +419,32 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, _ *cloudprovidertype ObjectMeta: metav1.ObjectMeta{ Name: machine.Name, Namespace: c.Namespace, - Labels: map[string]string{ - "kubevirt.io/vm": machine.Name, - }, + Labels: labels, }, Spec: kubevirtv1.VirtualMachineSpec{ Running: utilpointer.BoolPtr(true), Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: annotations, - Labels: map[string]string{ - "kubevirt.io/vm": machine.Name, - }, + Labels: labels, }, Spec: kubevirtv1.VirtualMachineInstanceSpec{ Domain: kubevirtv1.DomainSpec{ Devices: kubevirtv1.Devices{ - Disks: []kubevirtv1.Disk{ - { - Name: "datavolumedisk", - DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: "virtio"}}, - }, - { - Name: "cloudinitdisk", - DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: "virtio"}}, - }, - }, - }, - Resources: kubevirtv1.ResourceRequirements{ - Requests: *requestsAndLimits, - Limits: *requestsAndLimits, + Disks: getVMDisks(c), }, + Resources: resourceRequirements, }, TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, - Volumes: []kubevirtv1.Volume{ - { - Name: "datavolumedisk", - VolumeSource: kubevirtv1.VolumeSource{ - DataVolume: &kubevirtv1.DataVolumeSource{ - Name: dataVolumeName, - }, - }, - }, - { - Name: "cloudinitdisk", - VolumeSource: kubevirtv1.VolumeSource{ - CloudInitNoCloud: &kubevirtv1.CloudInitNoCloudSource{ - UserDataSecretRef: &corev1.LocalObjectReference{ - Name: userDataSecretName, - }, - }, - }, - }, - }, - DNSPolicy: c.DNSPolicy, - DNSConfig: c.DNSConfig, - }, - }, - DataVolumeTemplates: []kubevirtv1.DataVolumeTemplateSpec{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: dataVolumeName, - }, - Spec: cdiv1beta1.DataVolumeSpec{ - PVC: &corev1.PersistentVolumeClaimSpec{ - StorageClassName: utilpointer.StringPtr(c.StorageClassName), - AccessModes: []corev1.PersistentVolumeAccessMode{ - "ReadWriteOnce", - }, - Resources: corev1.ResourceRequirements{ - Requests: pvcRequest, - }, - }, - Source: &cdiv1beta1.DataVolumeSource{ - HTTP: &cdiv1beta1.DataVolumeSourceHTTP{ - URL: c.SourceURL, - }, - }, - }, + Volumes: getVMVolumes(c, dataVolumeName, userDataSecretName), + DNSPolicy: c.DNSPolicy, + DNSConfig: c.DNSConfig, }, }, + DataVolumeTemplates: getDataVolumeTemplates(c, dataVolumeName), }, } - sigClient, err := client.New(c.RestConfig, client.Options{}) - if err != nil { - return nil, fmt.Errorf("failed to get kubevirt client: %v", err) - } - ctx := context.Background() - if err := sigClient.Create(ctx, virtualMachine); err != nil { return nil, fmt.Errorf("failed to create vmi: %v", err) } @@ -517,3 +523,107 @@ func dnsPolicy(policy string) (corev1.DNSPolicy, error) { return "", fmt.Errorf("unknown dns policy: %s", policy) } + +func getVMDisks(config *Config) []kubevirtv1.Disk { + disks := []kubevirtv1.Disk{ + { + Name: "datavolumedisk", + DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: "virtio"}}, + }, + { + Name: "cloudinitdisk", + DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: "virtio"}}, + }, + } + for i := range config.SecondaryDisks { + disks = append(disks, kubevirtv1.Disk{ + Name: "secondarydisk" + strconv.Itoa(i), + DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: "virtio"}}, + }) + } + return disks +} + +func getVMVolumes(config *Config, dataVolumeName string, userDataSecretName string) []kubevirtv1.Volume { + volumes := []kubevirtv1.Volume{ + { + Name: "datavolumedisk", + VolumeSource: kubevirtv1.VolumeSource{ + DataVolume: &kubevirtv1.DataVolumeSource{ + Name: dataVolumeName, + }, + }, + }, + { + Name: "cloudinitdisk", + VolumeSource: kubevirtv1.VolumeSource{ + CloudInitNoCloud: &kubevirtv1.CloudInitNoCloudSource{ + UserDataSecretRef: &corev1.LocalObjectReference{ + Name: userDataSecretName, + }, + }, + }, + }, + } + for i := range config.SecondaryDisks { + volumes = append(volumes, kubevirtv1.Volume{ + Name: "secondarydisk" + strconv.Itoa(i), + VolumeSource: kubevirtv1.VolumeSource{ + DataVolume: &kubevirtv1.DataVolumeSource{ + Name: "secondarydisk" + strconv.Itoa(i), + }}, + }) + } + return volumes +} + +func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1.DataVolumeTemplateSpec { + pvcRequest := corev1.ResourceList{corev1.ResourceStorage: config.PVCSize} + dataVolumeTemplates := []kubevirtv1.DataVolumeTemplateSpec{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: dataVolumeName, + }, + Spec: cdiv1beta1.DataVolumeSpec{ + PVC: &corev1.PersistentVolumeClaimSpec{ + StorageClassName: utilpointer.StringPtr(config.StorageClassName), + AccessModes: []corev1.PersistentVolumeAccessMode{ + "ReadWriteOnce", + }, + Resources: corev1.ResourceRequirements{ + Requests: pvcRequest, + }, + }, + Source: &cdiv1beta1.DataVolumeSource{ + HTTP: &cdiv1beta1.DataVolumeSourceHTTP{ + URL: config.OsImage.URL, + }, + }, + }, + }, + } + for i, sd := range config.SecondaryDisks { + dataVolumeTemplates = append(dataVolumeTemplates, kubevirtv1.DataVolumeTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secondarydisk" + strconv.Itoa(i), + }, + Spec: cdiv1beta1.DataVolumeSpec{ + PVC: &corev1.PersistentVolumeClaimSpec{ + StorageClassName: utilpointer.StringPtr(sd.StorageClassName), + AccessModes: []corev1.PersistentVolumeAccessMode{ + "ReadWriteOnce", + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: sd.Size}, + }, + }, + Source: &cdiv1beta1.DataVolumeSource{ + HTTP: &cdiv1beta1.DataVolumeSourceHTTP{ + URL: config.OsImage.URL, + }, + }, + }, + }) + } + return dataVolumeTemplates +} diff --git a/pkg/cloudprovider/provider/kubevirt/types/cloudconfig.go b/pkg/cloudprovider/provider/kubevirt/types/cloudconfig.go index 91c40e1e8..8d41053e3 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/cloudconfig.go +++ b/pkg/cloudprovider/provider/kubevirt/types/cloudconfig.go @@ -23,6 +23,8 @@ import ( type CloudConfig struct { // Kubeconfig used to connect to the cluster that runs KubeVirt Kubeconfig string `yaml:"kubeconfig"` + // Namespace used in KubeVirt cloud-controller-manager as infra cluster namespace. + Namespace string `yaml:"namespace"` } func (c *CloudConfig) String() (string, error) { diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index 4924f8e49..be3413965 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -19,20 +19,56 @@ package types import ( "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - corev1 "k8s.io/api/core/v1" ) type RawConfig struct { - Kubeconfig providerconfigtypes.ConfigVarString `json:"kubeconfig,omitempty"` - CPUs providerconfigtypes.ConfigVarString `json:"cpus,omitempty"` - Memory providerconfigtypes.ConfigVarString `json:"memory,omitempty"` - Namespace providerconfigtypes.ConfigVarString `json:"namespace,omitempty"` - SourceURL providerconfigtypes.ConfigVarString `json:"sourceURL,omitempty"` - PVCSize providerconfigtypes.ConfigVarString `json:"pvcSize,omitempty"` + Auth Auth `json:"auth,omitempty"` + VirtualMachine VirtualMachine `json:"virtualMachine,omitempty"` +} + +// Auth +type Auth struct { + Kubeconfig providerconfigtypes.ConfigVarString `json:"kubeconfig,omitempty"` +} + +// VirtualMachine +type VirtualMachine struct { + Flavor Flavor `json:"flavor,omitempty"` + Template Template `json:"template,omitempty"` + DNSPolicy providerconfigtypes.ConfigVarString `json:"dnsPolicy,omitempty"` + DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` +} + +// Flavor +type Flavor struct { + Name providerconfigtypes.ConfigVarString `json:"name,omitempty"` + Profile providerconfigtypes.ConfigVarString `json:"profile,omitempty"` +} + +// Template +type Template struct { + CPUs providerconfigtypes.ConfigVarString `json:"cpus,omitempty"` + Memory providerconfigtypes.ConfigVarString `json:"memory,omitempty"` + PrimaryDisk PrimaryDisk `json:"primaryDisk,omitempty"` + SecondaryDisks []SecondaryDisks `json:"secondaryDisks,omitempty"` +} + +// PrimaryDisk +type PrimaryDisk struct { + Disk + OsImage providerconfigtypes.ConfigVarString `json:"osImage,omitempty"` +} + +// SecondaryDisks +type SecondaryDisks struct { + Disk +} + +// Disk +type Disk struct { + Size providerconfigtypes.ConfigVarString `json:"size,omitempty"` StorageClassName providerconfigtypes.ConfigVarString `json:"storageClassName,omitempty"` - DNSPolicy providerconfigtypes.ConfigVarString `json:"dnsPolicy,omitempty"` - DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` } func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index 936e74675..fe4f38f86 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -26,18 +26,21 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "kubevirt" cloudProviderSpec: - storageClassName: local-path - pvcSize: "10Gi" - sourceURL: http://10.107.208.71/<< OS_NAME >>.img - cpus: "1" - memory: "4096M" - dnsPolicy: "None" - dnsConfig: - nameservers: - - 8.8.8.8 - kubeconfig: - value: '<< KUBECONFIG >>' - namespace: kube-system + auth: + kubeconfig: + value: '<< KUBECONFIG >>' + virtualMachine: + template: + cpus: "1" + memory: "4096M" + primaryDisk: + osImage: http://10.107.208.71/<< OS_NAME >>.img + size: "10Gi" + storageClassName: local-path + dnsPolicy: "None" + dnsConfig: + nameservers: + - 8.8.8.8 operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false From 3d0a093193b3e82091f5ca40d911a2c4a922012b Mon Sep 17 00:00:00 2001 From: Mara Sophie Grosch Date: Fri, 18 Mar 2022 06:30:34 +0100 Subject: [PATCH 089/489] increase number of inotify instances per user (#1214) This defaulted to 128, meaning a given user ID can have a maximum of 128 inotify instances. Increasing to 8192. Signed-off-by: Mara Sophie Grosch --- pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml | 1 + pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml | 1 + pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml | 1 + pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml | 1 + pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml | 1 + pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml | 1 + pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml | 1 + pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml | 1 + pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml | 1 + pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml | 1 + pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml | 1 + pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml | 1 + pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml | 1 + pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml | 1 + pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml | 1 + pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml | 1 + pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml | 1 + pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml | 1 + pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml | 1 + pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml | 1 + pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml | 1 + pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml | 1 + pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml | 1 + pkg/userdata/flatcar/testdata/containerd.yaml | 1 + pkg/userdata/flatcar/testdata/ignition_v1.20.14.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.21.8.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.22.5.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.23.0.json | 2 +- pkg/userdata/helper/helper.go | 1 + pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml | 1 + pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml | 1 + pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml | 1 + pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml | 1 + pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml | 1 + pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml | 1 + pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml | 1 + pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml | 1 + pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml | 1 + pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml | 1 + pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml | 1 + .../sles/testdata/kubelet-version-without-v-prefix.yaml | 1 + pkg/userdata/sles/testdata/multiple-dns-servers.yaml | 1 + pkg/userdata/sles/testdata/multiple-ssh-keys.yaml | 1 + .../sles/testdata/openstack-overwrite-cloud-config.yaml | 1 + pkg/userdata/sles/testdata/openstack.yaml | 1 + pkg/userdata/sles/testdata/version-1.20.14.yaml | 1 + pkg/userdata/sles/testdata/version-1.21.8.yaml | 1 + pkg/userdata/sles/testdata/version-1.22.5.yaml | 1 + pkg/userdata/sles/testdata/version-1.23.0.yaml | 1 + pkg/userdata/sles/testdata/vsphere-mirrors.yaml | 1 + pkg/userdata/sles/testdata/vsphere-proxy.yaml | 1 + pkg/userdata/sles/testdata/vsphere.yaml | 1 + pkg/userdata/ubuntu/testdata/containerd.yaml | 1 + pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml | 1 + .../ubuntu/testdata/kubelet-version-without-v-prefix.yaml | 1 + pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml | 1 + pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml | 1 + pkg/userdata/ubuntu/testdata/nutanix.yaml | 1 + .../ubuntu/testdata/openstack-overwrite-cloud-config.yaml | 1 + pkg/userdata/ubuntu/testdata/openstack.yaml | 1 + pkg/userdata/ubuntu/testdata/version-1.20.14.yaml | 1 + pkg/userdata/ubuntu/testdata/version-1.21.8.yaml | 1 + pkg/userdata/ubuntu/testdata/version-1.22.5.yaml | 1 + pkg/userdata/ubuntu/testdata/version-1.23.0.yaml | 1 + pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml | 1 + pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml | 1 + pkg/userdata/ubuntu/testdata/vsphere.yaml | 1 + 67 files changed, 67 insertions(+), 4 deletions(-) diff --git a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml index 9b2714553..e50b4ca8d 100644 --- a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml @@ -38,6 +38,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml index c964cdfa6..b4bfa425b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml @@ -38,6 +38,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml index 2d0490115..352964c2a 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml @@ -38,6 +38,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml index 60269c6be..9ac68c8ec 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml @@ -38,6 +38,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml index f842297a2..3f7a6b4a8 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -48,6 +48,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml index 21d3d0b23..abfca5acd 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -48,6 +48,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml index b89c333f8..9dcff9302 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml @@ -40,6 +40,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml index 2c5ffec9d..16f27df60 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml @@ -38,6 +38,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml index ff061954d..ec84dcd6a 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml @@ -38,6 +38,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml index a2e8acb7f..5a27df9f1 100644 --- a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml @@ -38,6 +38,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml index 57a36fc33..314171060 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml @@ -38,6 +38,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml index 2ff821ad9..c8917f2ab 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml @@ -38,6 +38,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml index 98cccafbe..761198b5d 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml @@ -38,6 +38,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml index 84737013e..f3dea3c07 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml @@ -40,6 +40,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml index be65ea5c1..61fc1d084 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -48,6 +48,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml index ccd0349f8..9316a9672 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -48,6 +48,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml index 4b0bb7760..6bf322122 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml @@ -40,6 +40,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index 5a0ed8714..c81fb114d 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -38,6 +38,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index 839bd2ee6..b7f888022 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -38,6 +38,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml index 20b48a9b5..11301d193 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml @@ -251,6 +251,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup_net_env.sh" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml index bc9e03f0a..d3676d48b 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml @@ -251,6 +251,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup_net_env.sh" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml index 8e28e9949..e6d4c2d81 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml @@ -251,6 +251,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup_net_env.sh" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml index bf4d03c73..93ef4c5c4 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml @@ -249,6 +249,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup_net_env.sh" diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index a6f63ee8a..2582169da 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -234,6 +234,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup_net_env.sh" diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json b/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json index 8872fa45c..10cc21ece 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.14%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.14%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json b/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json index ebe926e02..a9a065094 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.8%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.8%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json b/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json index 0d4408f32..75137b5e8 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json index f6dd00901..39e5df380 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/helper.go b/pkg/userdata/helper/helper.go index fbf9e1657..32069503c 100644 --- a/pkg/userdata/helper/helper.go +++ b/pkg/userdata/helper/helper.go @@ -94,6 +94,7 @@ kernel.panic = 10 net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 +fs.inotify.max_user_instances = 8192 ` } diff --git a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml index 79685ec63..91c579b9b 100644 --- a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml @@ -40,6 +40,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml index bf5edafb4..f8633301e 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml @@ -40,6 +40,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml index a08bf51df..df9b1aa54 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml @@ -40,6 +40,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index fc62de7de..5129bf6e1 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -40,6 +40,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml index 24e55ecd8..acd97d9de 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml @@ -43,6 +43,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index 353463c22..692053c9e 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -40,6 +40,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index 75e193f4d..caf615754 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -40,6 +40,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index f56a3f063..23869b747 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -51,6 +51,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index 04a47939c..033e25220 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -51,6 +51,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index 173e97e39..1e6e2479d 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -43,6 +43,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: /etc/selinux/config diff --git a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml index b8549a686..0d3130e2f 100644 --- a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml @@ -44,6 +44,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup" diff --git a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml index 84b045912..c4f5d3d7f 100644 --- a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup" diff --git a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml index d2875d8fe..4697c9633 100644 --- a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup" diff --git a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml index ed16832ef..386253a7d 100644 --- a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml @@ -44,6 +44,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup" diff --git a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml index 72a7bfc94..0a99e47eb 100644 --- a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup" diff --git a/pkg/userdata/sles/testdata/openstack.yaml b/pkg/userdata/sles/testdata/openstack.yaml index 1fa4cfbf8..d9f22a305 100644 --- a/pkg/userdata/sles/testdata/openstack.yaml +++ b/pkg/userdata/sles/testdata/openstack.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup" diff --git a/pkg/userdata/sles/testdata/version-1.20.14.yaml b/pkg/userdata/sles/testdata/version-1.20.14.yaml index 5355db2b3..8255e4295 100644 --- a/pkg/userdata/sles/testdata/version-1.20.14.yaml +++ b/pkg/userdata/sles/testdata/version-1.20.14.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup" diff --git a/pkg/userdata/sles/testdata/version-1.21.8.yaml b/pkg/userdata/sles/testdata/version-1.21.8.yaml index 7fc71dbe6..416bc52bc 100644 --- a/pkg/userdata/sles/testdata/version-1.21.8.yaml +++ b/pkg/userdata/sles/testdata/version-1.21.8.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup" diff --git a/pkg/userdata/sles/testdata/version-1.22.5.yaml b/pkg/userdata/sles/testdata/version-1.22.5.yaml index 84b045912..c4f5d3d7f 100644 --- a/pkg/userdata/sles/testdata/version-1.22.5.yaml +++ b/pkg/userdata/sles/testdata/version-1.22.5.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup" diff --git a/pkg/userdata/sles/testdata/version-1.23.0.yaml b/pkg/userdata/sles/testdata/version-1.23.0.yaml index 3f4c181a5..e22ecb368 100644 --- a/pkg/userdata/sles/testdata/version-1.23.0.yaml +++ b/pkg/userdata/sles/testdata/version-1.23.0.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup" diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml index e92b70481..7d233ab75 100644 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml @@ -51,6 +51,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup" diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml index 937369f93..281d09b70 100644 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/sles/testdata/vsphere-proxy.yaml @@ -51,6 +51,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup" diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml index 337322d8f..ec6d9379f 100644 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ b/pkg/userdata/sles/testdata/vsphere.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/opt/bin/setup" diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index cf7fdbb2a..06b8533f8 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -44,6 +44,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/etc/default/grub.d/60-swap-accounting.cfg" diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index d08f60028..ce329c09b 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -44,6 +44,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/etc/default/grub.d/60-swap-accounting.cfg" diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index c31cbd8d0..de0a10f09 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/etc/default/grub.d/60-swap-accounting.cfg" diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index e96c48b0c..536e72499 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/etc/default/grub.d/60-swap-accounting.cfg" diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 82f61d628..06e7f29ff 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -44,6 +44,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/etc/default/grub.d/60-swap-accounting.cfg" diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 6880fca57..362abc7ef 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/etc/default/grub.d/60-swap-accounting.cfg" diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index 13196ae4d..e6a02b1c0 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/etc/default/grub.d/60-swap-accounting.cfg" diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 9ff6357ed..ad6055793 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/etc/default/grub.d/60-swap-accounting.cfg" diff --git a/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml b/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml index 4374b4550..70d5d896f 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/etc/default/grub.d/60-swap-accounting.cfg" diff --git a/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml b/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml index a3598fc76..85313d6c1 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/etc/default/grub.d/60-swap-accounting.cfg" diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml index c31cbd8d0..de0a10f09 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/etc/default/grub.d/60-swap-accounting.cfg" diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml index 9f080bb13..eacd59338 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/etc/default/grub.d/60-swap-accounting.cfg" diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index 58c4fcccd..d69464826 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -51,6 +51,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/etc/default/grub.d/60-swap-accounting.cfg" diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 89fbdc265..1add0b5ea 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -51,6 +51,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/etc/default/grub.d/60-swap-accounting.cfg" diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 1f260b7f5..108cd9c93 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -42,6 +42,7 @@ write_files: net.ipv4.ip_forward = 1 vm.overcommit_memory = 1 fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 - path: "/etc/default/grub.d/60-swap-accounting.cfg" From 4fad0db83ab81fd55307e0fe297b8bea5a6fdca1 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Fri, 18 Mar 2022 08:46:35 +0100 Subject: [PATCH 090/489] Update Azure SDK and API version (#1201) Signed-off-by: Marvin Beckers --- go.mod | 2 +- go.sum | 3 ++- .../provider/azure/create_delete_resources.go | 14 +++++++------- pkg/cloudprovider/provider/azure/get_client.go | 4 ++-- pkg/cloudprovider/provider/azure/provider.go | 8 ++++---- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index fd23411b7..abd9200be 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( cloud.google.com/go v0.73.0 cloud.google.com/go/logging v1.1.2 - github.com/Azure/azure-sdk-for-go v49.0.0+incompatible + github.com/Azure/azure-sdk-for-go v62.0.0+incompatible github.com/Azure/go-autorest/autorest/azure/auth v0.5.5 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/BurntSushi/toml v0.3.1 diff --git a/go.sum b/go.sum index 1cf1d49de..5cb0cbda2 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,9 @@ github.com/Azure/azure-sdk-for-go v28.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9mo github.com/Azure/azure-sdk-for-go v35.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v38.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v46.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v49.0.0+incompatible h1:rvYYNgKNBwoxUaBFmd/+TpW3qrd805EHBBvUp5FmFso= github.com/Azure/azure-sdk-for-go v49.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v62.0.0+incompatible h1:8N2k27SYtc12qj5nTsuFMFJPZn5CGmgMWqTy4y9I7Jw= +github.com/Azure/azure-sdk-for-go v62.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= diff --git a/pkg/cloudprovider/provider/azure/create_delete_resources.go b/pkg/cloudprovider/provider/azure/create_delete_resources.go index 6a02f328a..3c66064c3 100644 --- a/pkg/cloudprovider/provider/azure/create_delete_resources.go +++ b/pkg/cloudprovider/provider/azure/create_delete_resources.go @@ -20,8 +20,8 @@ import ( "context" "fmt" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" "github.com/Azure/go-autorest/autorest/azure/auth" "github.com/Azure/go-autorest/autorest/to" @@ -113,7 +113,7 @@ func deleteVMsByMachineUID(ctx context.Context, c *config, machineUID types.UID) return err } - list, err := vmClient.ListAll(ctx) + list, err := vmClient.ListAll(ctx, "", "") if err != nil { return err } @@ -129,7 +129,7 @@ func deleteVMsByMachineUID(ctx context.Context, c *config, machineUID types.UID) for _, vm := range allServers { if vm.Tags != nil && vm.Tags[machineUIDTag] != nil && *vm.Tags[machineUIDTag] == string(machineUID) { - future, err := vmClient.Delete(ctx, c.ResourceGroup, *vm.Name) + future, err := vmClient.Delete(ctx, c.ResourceGroup, *vm.Name, nil) if err != nil { return err } @@ -203,8 +203,8 @@ func createOrUpdatePublicIPAddress(ctx context.Context, ipName string, machineUI Name: to.StringPtr(ipName), Location: to.StringPtr(c.Location), PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - PublicIPAddressVersion: network.IPv4, - PublicIPAllocationMethod: network.Static, + PublicIPAddressVersion: network.IPVersionIPv4, + PublicIPAllocationMethod: network.IPAllocationMethodStatic, }, Tags: map[string]*string{machineUIDTag: to.StringPtr(string(machineUID))}, Zones: &c.Zones, @@ -279,7 +279,7 @@ func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineU Name: to.StringPtr("ip-config-1"), InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ Subnet: &subnet, - PrivateIPAllocationMethod: network.Dynamic, + PrivateIPAllocationMethod: network.IPAllocationMethodDynamic, PublicIPAddress: publicIP, }, }, diff --git a/pkg/cloudprovider/provider/azure/get_client.go b/pkg/cloudprovider/provider/azure/get_client.go index c5fb7357c..9b3f1aaae 100644 --- a/pkg/cloudprovider/provider/azure/get_client.go +++ b/pkg/cloudprovider/provider/azure/get_client.go @@ -19,8 +19,8 @@ package azure import ( "fmt" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" "github.com/Azure/go-autorest/autorest/azure/auth" ) diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 5ff671e11..402f54258 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -23,8 +23,8 @@ import ( "fmt" "strings" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-06-01/compute" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2018-06-01/network" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" "github.com/Azure/go-autorest/autorest/to" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" @@ -699,7 +699,7 @@ func getVMByUID(ctx context.Context, c *config, uid types.UID) (*compute.Virtual return nil, err } - list, err := vmClient.ListAll(ctx) + list, err := vmClient.ListAll(ctx, "", "") if err != nil { return nil, err } @@ -890,7 +890,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return fmt.Errorf("failed to (create) vm client: %v", err.Error()) } - _, err = vmClient.ListAll(context.TODO()) + _, err = vmClient.ListAll(context.TODO(), "", "") if err != nil { return fmt.Errorf("failed to list all: %v", err.Error()) } From bf24dfec47bca351505797201258df6310cfbf30 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 18 Mar 2022 15:29:47 +0500 Subject: [PATCH 091/489] Enhance OpenStack app credentials support (#1210) * Enhance openstack app credentials support Signed-off-by: Waleed Malik * Revert validation in webhook Signed-off-by: Waleed Malik --- docs/cloud-provider.md | 5 +++++ pkg/cloudprovider/provider/openstack/provider.go | 7 +++---- pkg/cloudprovider/provider/openstack/types/cloudconfig.go | 2 +- .../use-application-credentials-ignore-userpass.golden | 1 - .../types/testdata/use-application-credentials.golden | 1 - 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index bdf56c9eb..8a54fdb87 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -95,6 +95,11 @@ tags: ```yaml # identity endpoint of your openstack installation identityEndpoint: "" +# application Credential ID and Secret can be used in place of username, password, tenantName/tenantID, and domainName. +# application credentials ID +applicationCredentialID: "" +# application credentials secret +applicationCredentialSecret: "" # your openstack username username: "" # your openstack password diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 64007d86b..f86f119fb 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -457,16 +457,15 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { if c.ProjectID == "" && c.ProjectName == "" { return errors.New("either projectID / tenantID or projectName / tenantName must be configured") } + if c.DomainName == "" { + return errors.New("domainName must be configured") + } } else { if c.ApplicationCredentialSecret == "" { return errors.New("applicationCredentialSecret must be configured in conjunction with applicationCredentialID") } } - if c.DomainName == "" { - return errors.New("domainName must be configured") - } - if c.Image == "" { return errors.New("image must be configured") } diff --git a/pkg/cloudprovider/provider/openstack/types/cloudconfig.go b/pkg/cloudprovider/provider/openstack/types/cloudconfig.go index 917665d9f..cdcc64121 100644 --- a/pkg/cloudprovider/provider/openstack/types/cloudconfig.go +++ b/pkg/cloudprovider/provider/openstack/types/cloudconfig.go @@ -41,8 +41,8 @@ username = {{ .Global.Username | iniEscape }} password = {{ .Global.Password | iniEscape }} tenant-name = {{ .Global.ProjectName | iniEscape }} tenant-id = {{ .Global.ProjectID | iniEscape }} -{{- end }} domain-name = {{ .Global.DomainName | iniEscape }} +{{- end }} region = {{ .Global.Region | iniEscape }} [LoadBalancer] diff --git a/pkg/cloudprovider/provider/openstack/types/testdata/use-application-credentials-ignore-userpass.golden b/pkg/cloudprovider/provider/openstack/types/testdata/use-application-credentials-ignore-userpass.golden index 2f880fa00..34e1f16b9 100644 --- a/pkg/cloudprovider/provider/openstack/types/testdata/use-application-credentials-ignore-userpass.golden +++ b/pkg/cloudprovider/provider/openstack/types/testdata/use-application-credentials-ignore-userpass.golden @@ -2,7 +2,6 @@ auth-url = "https://127.0.0.1:8443/" application-credential-id = "app-cred-id" application-credential-secret = "app-cred-secret" -domain-name = "Default" region = "eu-central1" [LoadBalancer] diff --git a/pkg/cloudprovider/provider/openstack/types/testdata/use-application-credentials.golden b/pkg/cloudprovider/provider/openstack/types/testdata/use-application-credentials.golden index 2f880fa00..34e1f16b9 100644 --- a/pkg/cloudprovider/provider/openstack/types/testdata/use-application-credentials.golden +++ b/pkg/cloudprovider/provider/openstack/types/testdata/use-application-credentials.golden @@ -2,7 +2,6 @@ auth-url = "https://127.0.0.1:8443/" application-credential-id = "app-cred-id" application-credential-secret = "app-cred-secret" -domain-name = "Default" region = "eu-central1" [LoadBalancer] From d116755d6b3952d5e255e6d9f2afc4312f55e668 Mon Sep 17 00:00:00 2001 From: Helene Durand <90329641+hdurand0710@users.noreply.github.com> Date: Sat, 19 Mar 2022 14:46:47 +0100 Subject: [PATCH 092/489] Alibaba Images names (#1215) * Alibaba Images names Signed-off-by: Helene Durand * test only ubuntu for Alibaba Signed-off-by: Helene Durand --- pkg/cloudprovider/provider/alibaba/provider.go | 2 +- test/e2e/provisioning/all_e2e_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index a98be284a..0687b8a28 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -42,7 +42,7 @@ import ( const ( machineUIDTag = "machine_uid" - centosImageName = "CentOS 7.7 64 bit" + centosImageName = "CentOS 7.9 64 bit" ubuntuImageName = "Ubuntu 20.04 64 bit" finalizerInstance = "kubermatic.io/cleanup-alibaba-instance" diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 8b517f3b9..f20401849 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -753,7 +753,7 @@ func TestAlibabaProvisioningE2E(t *testing.T) { t.Fatal("unable to run the test suite, ALIBABA_ACCESS_KEY_SECRET environment variable cannot be empty") } - selector := Not(OsSelector("sles", "rhel", "flatcar")) + selector := OsSelector("ubuntu") // act params := []string{ From a86a982f0f6d2e892df7d120b3e4e5d97058dab6 Mon Sep 17 00:00:00 2001 From: Pratik Deoghare Date: Mon, 21 Mar 2022 10:02:30 +0100 Subject: [PATCH 093/489] consider pod cidr information during instance creation (#1207) * accept pod cidrs from command line args * pass pod cidrs for instance creation * add comments on NetworkConfig * . * pass networkconfig everywhere * pass network config pointer * . * . * adjust azure rhel template for multiple pod cidrs * fix yamllint * fix generated rhel user data --- cmd/machine-controller/main.go | 10 +- pkg/apis/plugin/plugin.go | 2 +- .../provider/alibaba/provider.go | 2 +- pkg/cloudprovider/provider/anexia/provider.go | 8 +- pkg/cloudprovider/provider/aws/provider.go | 2 +- pkg/cloudprovider/provider/azure/provider.go | 2 +- .../provider/baremetal/provider.go | 2 +- .../provider/digitalocean/provider.go | 2 +- .../provider/equinixmetal/provider.go | 2 +- pkg/cloudprovider/provider/fake/provider.go | 2 +- pkg/cloudprovider/provider/gce/provider.go | 6 +- .../provider/hetzner/provider.go | 2 +- .../provider/kubevirt/provider.go | 2 +- pkg/cloudprovider/provider/linode/provider.go | 2 +- .../provider/nutanix/provider.go | 2 +- .../provider/openstack/provider.go | 2 +- .../provider/openstack/provider_test.go | 2 +- .../provider/scaleway/provider.go | 2 +- .../provider/vsphere/provider.go | 2 +- pkg/cloudprovider/types/types.go | 7 +- pkg/cloudprovider/validationwrapper.go | 4 +- pkg/controller/machine/machine_controller.go | 18 +- pkg/userdata/rhel/provider.go | 4 +- pkg/userdata/rhel/provider_test.go | 11 + .../rhel/testdata/pod-cidr-azure-rhel.yaml | 476 ++++++++++++++++++ test/e2e/provisioning/migrateuidscenario.go | 2 +- 26 files changed, 536 insertions(+), 42 deletions(-) create mode 100644 pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 0fa532d20..93f5caa76 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -79,7 +79,7 @@ var ( nodeRegistryMirrors string nodePauseImage string nodeContainerRuntime string - podCidr string + podCIDRs string nodePortRange string nodeRegistryCredentialsSecret string nodeContainerdRegistryMirrors = containerruntime.RegistryMirrorsFlags{} @@ -128,7 +128,7 @@ type controllerRunOptions struct { useOSM bool // Assigns the POD networks that will be allocated. - podCidr string + podCIDRs []string // A port range to reserve for services with NodePort visibility nodePortRange string @@ -166,7 +166,7 @@ func main() { flag.Var(&nodeContainerdRegistryMirrors, "node-containerd-registry-mirrors", "Configure registry mirrors endpoints. Can be used multiple times to specify multiple mirrors") flag.StringVar(&caBundleFile, "ca-bundle", "", "path to a file containing all PEM-encoded CA certificates (will be used instead of the host's certificates if set)") flag.BoolVar(&nodeCSRApprover, "node-csr-approver", true, "Enable NodeCSRApprover controller to automatically approve node serving certificate requests") - flag.StringVar(&podCidr, "pod-cidr", "172.25.0.0/16", "The network ranges from which POD networks are allocated") + flag.StringVar(&podCIDRs, "pod-cidr", "172.25.0.0/16", "Comma separated network ranges from which POD networks are allocated. Example: 172.25.0.0/16,fd00::/104") flag.StringVar(&nodePortRange, "node-port-range", "30000-32767", "A port range to reserve for services with NodePort visibility") flag.StringVar(&nodeRegistryCredentialsSecret, "node-registry-credentials-secret", "", "A Secret object reference, that containt auth info for image registry in namespace/secret-name form, example: kube-system/registry-credentials. See doc at https://github.com/kubermaric/machine-controller/blob/master/docs/registry-authentication.md") flag.BoolVar(&useOSM, "use-osm", false, "use osm controller for node bootstrap") @@ -266,7 +266,7 @@ func main() { ContainerRuntime: containerRuntimeConfig, }, useOSM: useOSM, - podCidr: podCidr, + podCIDRs: strings.Split(podCIDRs, ","), nodePortRange: nodePortRange, } @@ -402,7 +402,7 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { bs.opt.skipEvictionAfter, bs.opt.node, bs.opt.useOSM, - bs.opt.podCidr, + bs.opt.podCIDRs, bs.opt.nodePortRange, ); err != nil { return fmt.Errorf("failed to add Machine controller to manager: %v", err) diff --git a/pkg/apis/plugin/plugin.go b/pkg/apis/plugin/plugin.go index ce0a75c0f..0cc81ac6d 100644 --- a/pkg/apis/plugin/plugin.go +++ b/pkg/apis/plugin/plugin.go @@ -54,7 +54,7 @@ type UserDataRequest struct { KubeletFeatureGates map[string]bool KubeletConfigs map[string]string ContainerRuntime containerruntime.Config - PodCIDR string + PodCIDRs []string NodePortRange string } diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index 0687b8a28..f9a71bb13 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -198,7 +198,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri return "", "", nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 8d53160ed..646882e2a 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -190,7 +190,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (string, str } // Create creates a cloud instance according to the given machine -func (p *provider) Create(machine *clusterv1alpha1.Machine, providerData *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, "failed to parse MachineSpec: %v", err) @@ -255,7 +255,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, providerData *cloudp } status.ProvisioningID = provisionResponse.Identifier - if err := updateStatus(machine, status, providerData.Update); err != nil { + if err := updateStatus(machine, status, data.Update); err != nil { return nil, newError(common.UpdateMachineError, "machine status update failed: %v", err) } } @@ -266,11 +266,11 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, providerData *cloudp } status.InstanceID = instanceID - if err := updateStatus(machine, status, providerData.Update); err != nil { + if err := updateStatus(machine, status, data.Update); err != nil { return nil, newError(common.UpdateMachineError, "machine status update failed: %v", err) } - return p.Get(machine, providerData) + return p.Get(machine, data) } func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index c9e243043..91f46c724 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -660,7 +660,7 @@ func getVpc(client *ec2.EC2, id string) (*ec2.Vpc, error) { return vpcOut.Vpcs[0], nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 402f54258..1c9bbdf76 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -480,7 +480,7 @@ func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) return sp, nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { config, providerCfg, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/baremetal/provider.go b/pkg/cloudprovider/provider/baremetal/provider.go index 3956a12b1..61bd0e1cc 100644 --- a/pkg/cloudprovider/provider/baremetal/provider.go +++ b/pkg/cloudprovider/provider/baremetal/provider.go @@ -210,7 +210,7 @@ func (p provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config strin return "", "", nil } -func (p provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index 15a889ff4..93ab072c7 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -263,7 +263,7 @@ func uploadRandomSSHPublicKey(ctx context.Context, service godo.KeysService) (st return newDoKey.Fingerprint, nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index 81b492ea1..23dea42d6 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -214,7 +214,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { c, _, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/fake/provider.go b/pkg/cloudprovider/provider/fake/provider.go index 48f6dd240..da9f2a60d 100644 --- a/pkg/cloudprovider/provider/fake/provider.go +++ b/pkg/cloudprovider/provider/fake/provider.go @@ -94,7 +94,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (string, str } // Create creates a cloud instance according to the given machine -func (p *provider) Create(_ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, _ string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { return CloudProviderInstance{}, nil } diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 0297843c4..2a51fbf5a 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -195,11 +195,7 @@ func (p *Provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri } // Create inserts a cloud instance according to the given machine. -func (p *Provider) Create( - machine *clusterv1alpha1.Machine, - data *cloudprovidertypes.ProviderData, - userdata string, -) (instance.Instance, error) { +func (p *Provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { // Read configuration. cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) if err != nil { diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index f066d7290..61a4f2541 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -245,7 +245,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 282916f28..1b69254cd 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -356,7 +356,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } -func (p *provider) Create(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index 32ce14863..1e3688f6d 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -212,7 +212,7 @@ func createRandomPassword() (string, error) { return rootPass, nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 5ad6b0212..54025577f 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -241,7 +241,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { vm, err := p.create(machine, userdata) if err != nil { _, cleanupErr := p.Cleanup(machine, data) diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index f86f119fb..829d3f859 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -553,7 +553,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { cfg, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index 8bebddd7e..a0a8c2cd1 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -295,7 +295,7 @@ func TestCreateServer(t *testing.T) { // It only verifies that the content of the create request matches // the expectation // TODO(irozzo) check the returned instance too - _, err := p.Create(m, tt.data, tt.userdata) + _, err := p.Create(m, tt.data, tt.userdata, nil) if (err != nil) != tt.wantErr { t.Errorf("provider.Create() or = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go index 59c4a6942..d6f6f27b8 100644 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ b/pkg/cloudprovider/provider/scaleway/provider.go @@ -172,7 +172,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (cloudInstance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (cloudInstance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 9cfd9c986..d6c1d73f6 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -262,7 +262,7 @@ func machineInvalidConfigurationTerminalError(err error) error { } } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { vm, err := p.create(machine, userdata) if err != nil { _, cleanupErr := p.Cleanup(machine, data) diff --git a/pkg/cloudprovider/types/types.go b/pkg/cloudprovider/types/types.go index fdc756fe7..2e7249fb3 100644 --- a/pkg/cloudprovider/types/types.go +++ b/pkg/cloudprovider/types/types.go @@ -52,7 +52,7 @@ type Provider interface { GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) // Create creates a cloud instance according to the given machine - Create(machine *clusterv1alpha1.Machine, data *ProviderData, userdata string) (instance.Instance, error) + Create(machine *clusterv1alpha1.Machine, data *ProviderData, userdata string, networkConfig *NetworkConfig) (instance.Instance, error) // Cleanup will delete the instance associated with the machine and all associated resources. // If all resources have been cleaned up, true will be returned. @@ -76,6 +76,11 @@ type Provider interface { SetMetricsForMachines(machines clusterv1alpha1.MachineList) error } +// NetworkConfig holds information about cluster networking. +type NetworkConfig struct { + PodCIDRs []string `json:"podCIDRs"` // PodCIDRs fields is used to choose IPv4, IPv6 or dual-stack modes. +} + // MachineModifier defines a function to modify a machine type MachineModifier func(*clusterv1alpha1.Machine) diff --git a/pkg/cloudprovider/validationwrapper.go b/pkg/cloudprovider/validationwrapper.go index 9a5ae5c3c..b4426fead 100644 --- a/pkg/cloudprovider/validationwrapper.go +++ b/pkg/cloudprovider/validationwrapper.go @@ -73,8 +73,8 @@ func (w *cachingValidationWrapper) GetCloudConfig(spec v1alpha1.MachineSpec) (st } // Create just calls the underlying cloudproviders Create -func (w *cachingValidationWrapper) Create(m *v1alpha1.Machine, mcd *cloudprovidertypes.ProviderData, cloudConfig string) (instance.Instance, error) { - return w.actualProvider.Create(m, mcd, cloudConfig) +func (w *cachingValidationWrapper) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { + return w.actualProvider.Create(machine, data, userdata, networkConfig) } // Cleanup just calls the underlying cloudproviders Cleanup diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index d7977bf4d..3ae4e1406 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -117,7 +117,7 @@ type Reconciler struct { satelliteSubscriptionManager rhsm.SatelliteSubscriptionManager useOSM bool - podCIDR string + podCIDRs []string nodePortRange string } @@ -175,7 +175,7 @@ func Add( skipEvictionAfter time.Duration, nodeSettings NodeSettings, useOSM bool, - podCIDR string, + podCIDRs []string, nodePortRange string, ) error { reconciler := &Reconciler{ @@ -194,7 +194,7 @@ func Add( satelliteSubscriptionManager: rhsm.NewSatelliteSubscriptionManager(), useOSM: useOSM, - podCIDR: podCIDR, + podCIDRs: podCIDRs, nodePortRange: nodePortRange, } m, err := userdatamanager.New() @@ -340,13 +340,13 @@ func (r *Reconciler) updateMachineErrorIfTerminalError(machine *clusterv1alpha1. return fmt.Errorf("%s, due to %v", errMsg, err) } -func (r *Reconciler) createProviderInstance(prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { +func (r *Reconciler) createProviderInstance(prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { // Ensure finalizer is there _, err := r.ensureDeleteFinalizerExists(machine) if err != nil { return nil, fmt.Errorf("failed to add %q finalizer: %v", FinalizerDeleteInstance, err) } - instance, err := prov.Create(machine, r.providerData, userdata) + instance, err := prov.Create(machine, r.providerData, userdata, networkConfig) if err != nil { return nil, err } @@ -802,7 +802,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( NoProxy: r.nodeSettings.NoProxy, HTTPProxy: r.nodeSettings.HTTPProxy, ContainerRuntime: crRuntime, - PodCIDR: r.podCIDR, + PodCIDRs: r.podCIDRs, NodePortRange: r.nodePortRange, } @@ -844,8 +844,12 @@ func (r *Reconciler) ensureInstanceExistsForMachine( } } + networkConfig := &cloudprovidertypes.NetworkConfig{ + PodCIDRs: r.podCIDRs, + } + // Create the instance - if _, err = r.createProviderInstance(prov, machine, userdata); err != nil { + if _, err = r.createProviderInstance(prov, machine, userdata, networkConfig); err != nil { message := fmt.Sprintf("%v. Unable to create a machine.", err) return nil, r.updateMachineErrorIfTerminalError(machine, common.CreateMachineError, message, err, "failed to create machine at cloudprovider") } diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 306f863f1..13bf1bee0 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -241,7 +241,9 @@ write_files: /opt/bin/setup_net_env.sh {{ if eq .CloudProviderName "azure" }} - firewall-cmd --permanent --zone=trusted --add-source={{ .PodCIDR }} + {{- range $idx, $podCIDR := .PodCIDRs }} + firewall-cmd --permanent --zone=trusted --add-source={{ $podCIDR}} + {{ end }} firewall-cmd --permanent --add-port=8472/udp firewall-cmd --permanent --add-port={{ .NodePortRange }}/tcp firewall-cmd --permanent --add-port={{ .NodePortRange }}/udp diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index 3c0ac70f1..fc48cec11 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -203,6 +203,16 @@ func TestUserDataGeneration(t *testing.T) { }, cloudProviderName: stringPtr("nutanix"), }, + { + name: "pod-cidr-azure-rhel", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.22.2", + }, + }, + cloudProviderName: stringPtr("azure"), + }, } defaultCloudProvider := &fakeCloudConfigProvider{ @@ -273,6 +283,7 @@ func TestUserDataGeneration(t *testing.T) { PauseImage: test.pauseImage, KubeletFeatureGates: kubeletFeatureGates, ContainerRuntime: containerRuntimeConfig, + PodCIDRs: []string{"172.25.0.0/16", "fd00::/104"}, } s, err := provider.UserData(req) if err != nil { diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml new file mode 100644 index 000000000..d1023e0af --- /dev/null +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -0,0 +1,476 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + +hostname: node1 +fqdn: node1 + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + hostnamectl set-hostname node1 + + + yum update -y --disablerepo='*' --enablerepo='*microsoft*' + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + + firewall-cmd --permanent --zone=trusted --add-source=172.25.0.0/16 + + firewall-cmd --permanent --zone=trusted --add-source=fd00::/104 + + firewall-cmd --permanent --add-port=8472/udp + firewall-cmd --permanent --add-port=/tcp + firewall-cmd --permanent --add-port=/udp + firewall-cmd --reload + systemctl restart firewalld + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=azure \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: {} + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + +rh_subscription: + username: "" + password: "" + auto-attach: false + +runcmd: +- systemctl start setup.service +- systemctl start disable-nm-cloud-setup.service diff --git a/test/e2e/provisioning/migrateuidscenario.go b/test/e2e/provisioning/migrateuidscenario.go index 8f1e20f60..7cf6745c7 100644 --- a/test/e2e/provisioning/migrateuidscenario.go +++ b/test/e2e/provisioning/migrateuidscenario.go @@ -96,7 +96,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time } return fmt.Errorf("failed to get machine %s before creating it: %v", machine.Name, err) } - _, err := prov.Create(machine, providerData, "#cloud-config\n") + _, err := prov.Create(machine, providerData, "#cloud-config\n", nil) if err != nil { if i < maxTries-1 { time.Sleep(10 * time.Second) From ec55fb7876369e87a20cc1702e22c6987c5e17c5 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 22 Mar 2022 12:56:56 +0100 Subject: [PATCH 094/489] refactor controller utilities (#1218) Signed-off-by: Moath Qasim --- pkg/controller/machine/machine_controller.go | 40 ++----------- pkg/controller/machinedeployment/rolling.go | 2 +- pkg/controller/machinedeployment/sync.go | 2 +- pkg/controller/util/machine.go | 58 +++++++++++++++++++ .../util.go => util/machine_deployment.go} | 5 +- 5 files changed, 66 insertions(+), 41 deletions(-) create mode 100644 pkg/controller/util/machine.go rename pkg/controller/{machinedeployment/util/util.go => util/machine_deployment.go} (99%) diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 3ae4e1406..2e8a50584 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -37,6 +37,7 @@ import ( cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/containerruntime" + controllerutil "github.com/kubermatic/machine-controller/pkg/controller/util" kuberneteshelper "github.com/kubermatic/machine-controller/pkg/kubernetes" "github.com/kubermatic/machine-controller/pkg/node/eviction" "github.com/kubermatic/machine-controller/pkg/node/poddeletion" @@ -223,7 +224,7 @@ func Add( handler.EnqueueRequestsFromMapFunc(func(node client.Object) (result []reconcile.Request) { machinesList := &clusterv1alpha1.MachineList{} if err := mgr.GetClient().List(ctx, machinesList); err != nil { - utilruntime.HandleError(fmt.Errorf("Failed to list machines in lister: %v", err)) + utilruntime.HandleError(fmt.Errorf("failed to list machines in lister: %v", err)) return } @@ -346,11 +347,11 @@ func (r *Reconciler) createProviderInstance(prov cloudprovidertypes.Provider, ma if err != nil { return nil, fmt.Errorf("failed to add %q finalizer: %v", FinalizerDeleteInstance, err) } - instance, err := prov.Create(machine, r.providerData, userdata, networkConfig) + i, err := prov.Create(machine, r.providerData, userdata, networkConfig) if err != nil { return nil, err } - return instance, nil + return i, nil } func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { @@ -810,7 +811,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( var userdata string if r.useOSM { - referencedMachineDeployment, err := r.getMachineDeploymentNameForMachine(ctx, machine) + referencedMachineDeployment, err := controllerutil.GetMachineDeploymentNameForMachine(ctx, machine, r.client) if err != nil { return nil, fmt.Errorf("failed to find machine's MachineDployment: %v", err) } @@ -1152,34 +1153,3 @@ func (r *Reconciler) updateNode(ctx context.Context, node *corev1.Node, modifier return r.client.Update(ctx, node) }) } - -func (r *Reconciler) getMachineDeploymentNameForMachine(ctx context.Context, machine *clusterv1alpha1.Machine) (string, error) { - var ( - machineSetName string - machineDeploymentName string - ) - for _, ownerRef := range machine.OwnerReferences { - if ownerRef.Kind == "MachineSet" { - machineSetName = ownerRef.Name - } - } - - if machineSetName != "" { - machineSet := &clusterv1alpha1.MachineSet{} - if err := r.client.Get(ctx, types.NamespacedName{Name: machineSetName, Namespace: "kube-system"}, machineSet); err != nil { - return "", err - } - - for _, ownerRef := range machineSet.OwnerReferences { - if ownerRef.Kind == "MachineDeployment" { - machineDeploymentName = ownerRef.Name - } - } - - if machineDeploymentName != "" { - return machineDeploymentName, nil - } - } - - return "", fmt.Errorf("failed to find machine deployment reference for the machine %s", machine.Name) -} diff --git a/pkg/controller/machinedeployment/rolling.go b/pkg/controller/machinedeployment/rolling.go index 809ffa9e2..4ca3496e8 100644 --- a/pkg/controller/machinedeployment/rolling.go +++ b/pkg/controller/machinedeployment/rolling.go @@ -23,7 +23,7 @@ import ( "github.com/pkg/errors" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - dutil "github.com/kubermatic/machine-controller/pkg/controller/machinedeployment/util" + dutil "github.com/kubermatic/machine-controller/pkg/controller/util" "k8s.io/apimachinery/pkg/types" "k8s.io/klog" diff --git a/pkg/controller/machinedeployment/sync.go b/pkg/controller/machinedeployment/sync.go index e9c14203e..51ede6fa7 100644 --- a/pkg/controller/machinedeployment/sync.go +++ b/pkg/controller/machinedeployment/sync.go @@ -26,7 +26,7 @@ import ( "github.com/pkg/errors" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - dutil "github.com/kubermatic/machine-controller/pkg/controller/machinedeployment/util" + dutil "github.com/kubermatic/machine-controller/pkg/controller/util" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/controller/util/machine.go b/pkg/controller/util/machine.go new file mode 100644 index 000000000..85a6c5c21 --- /dev/null +++ b/pkg/controller/util/machine.go @@ -0,0 +1,58 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "context" + "fmt" + + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func GetMachineDeploymentNameForMachine(ctx context.Context, machine *clusterv1alpha1.Machine, c client.Client) (string, error) { + var ( + machineSetName string + machineDeploymentName string + ) + for _, ownerRef := range machine.OwnerReferences { + if ownerRef.Kind == "MachineSet" { + machineSetName = ownerRef.Name + } + } + + if machineSetName != "" { + machineSet := &clusterv1alpha1.MachineSet{} + if err := c.Get(ctx, types.NamespacedName{Name: machineSetName, Namespace: "kube-system"}, machineSet); err != nil { + return "", err + } + + for _, ownerRef := range machineSet.OwnerReferences { + if ownerRef.Kind == "MachineDeployment" { + machineDeploymentName = ownerRef.Name + } + } + + if machineDeploymentName != "" { + return machineDeploymentName, nil + } + } + + return "", fmt.Errorf("failed to find machine deployment reference for the machine %s", machine.Name) +} diff --git a/pkg/controller/machinedeployment/util/util.go b/pkg/controller/util/machine_deployment.go similarity index 99% rename from pkg/controller/machinedeployment/util/util.go rename to pkg/controller/util/machine_deployment.go index 42cb11521..48074279c 100644 --- a/pkg/controller/machinedeployment/util/util.go +++ b/pkg/controller/util/machine_deployment.go @@ -1,5 +1,5 @@ /* -Copyright 2019 The Machine Controller Authors. +Copyright 2022 The Machine Controller Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -63,9 +63,6 @@ const ( // estimated once a deployment is paused. PausedDeployReason = "DeploymentPaused" - // - // Available: - // // MinimumReplicasAvailable is added in a deployment when it has its minimum replicas required available. MinimumReplicasAvailable = "MinimumReplicasAvailable" // MinimumReplicasUnavailable is added in a deployment when it doesn't have the minimum required replicas From b067fb6226c1a3ef334d0feee94360460d5e2439 Mon Sep 17 00:00:00 2001 From: Kevin Stiehl Date: Tue, 22 Mar 2022 19:17:59 +0100 Subject: [PATCH 095/489] Make anexia provider more resilient against errors (#1175) * add conditions & tests Original commit by @kstiehl, conflicts resolved by @LittleFox94, mainly in pkg/cloudprovider/provider/anexia/provider.go Signed-off-by: kstiehl Signed-off-by: Mara Sophie Grosch * Fix various linter errors Signed-off-by: Mara Sophie Grosch * Add missing boilerplate to files Signed-off-by: Mara Sophie Grosch * Anexia provider: add dummy updater for tests Signed-off-by: Mara Sophie Grosch Co-authored-by: Mara Sophie Grosch --- go.mod | 2 +- go.sum | 12 +- .../provider/anexia/helper_test.go | 91 ++++ pkg/cloudprovider/provider/anexia/provider.go | 405 +++++++++++++----- .../provider/anexia/provider_test.go | 333 ++++++++++++++ .../provider/anexia/types/errors.go | 49 +++ .../provider/anexia/types/types.go | 30 +- .../provider/anexia/utils/utils.go | 49 +++ 8 files changed, 843 insertions(+), 128 deletions(-) create mode 100644 pkg/cloudprovider/provider/anexia/helper_test.go create mode 100644 pkg/cloudprovider/provider/anexia/provider_test.go create mode 100644 pkg/cloudprovider/provider/anexia/types/errors.go create mode 100644 pkg/cloudprovider/provider/anexia/utils/utils.go diff --git a/go.mod b/go.mod index abd9200be..7fb3d9fd0 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/Masterminds/sprig/v3 v3.2.2 github.com/aliyun/alibaba-cloud-sdk-go v1.61.751 - github.com/anexia-it/go-anxcloud v0.3.8 + github.com/anexia-it/go-anxcloud v0.3.26 github.com/aws/aws-sdk-go v1.36.2 github.com/coreos/container-linux-config-transpiler v0.9.0 github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index 5cb0cbda2..a078d09dd 100644 --- a/go.sum +++ b/go.sum @@ -173,8 +173,9 @@ github.com/aliyun/alibaba-cloud-sdk-go v1.61.751 h1:PX0jCn9kBBgaybsFltpmQ8F7O74h github.com/aliyun/alibaba-cloud-sdk-go v1.61.751/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andygrunwald/go-gerrit v0.0.0-20190120104749-174420ebee6c/go.mod h1:0iuRQp6WJ44ts+iihy5E/WlPqfg5RNeQxOmzRkxCdtk= -github.com/anexia-it/go-anxcloud v0.3.8 h1:+ZOVqUHwINTm9Q68GPVh+Q/c794Fe+2GahIVagNLjDg= github.com/anexia-it/go-anxcloud v0.3.8/go.mod h1:cevqezsbOJ4GBlAWaztfLKl9w4VzxJBt4ipgHORi3gw= +github.com/anexia-it/go-anxcloud v0.3.26 h1:uStosj8srS6OA1OsPsMJBFqd4Znzl6fEhUv8b3+G8FU= +github.com/anexia-it/go-anxcloud v0.3.26/go.mod h1:fiEBxEtBXx78/OWBJvL7+2o4TESrnEcrDYjLeonGkDw= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -320,6 +321,7 @@ github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -1004,6 +1006,7 @@ github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= @@ -1182,7 +1185,6 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= @@ -1606,7 +1608,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1987,7 +1988,6 @@ k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= k8s.io/api v0.19.4/go.mod h1:SbtJ2aHCItirzdJ36YslycFNzWADYH3tgOhvBEFtZAk= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= -k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= k8s.io/api v0.22.2 h1:M8ZzAD0V6725Fjg53fKeTJxGsJvRbk4TEm/fexHMtfw= k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY= @@ -2026,7 +2026,6 @@ k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlm k8s.io/apimachinery v0.19.4/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.22.2 h1:ejz6y/zNma8clPVfNDLnPbleBo6MpoFy/HBiBqCouVk= k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg= @@ -2112,8 +2111,6 @@ k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-aggregator v0.19.0/go.mod h1:1Ln45PQggFAG8xOqWPIYMxUq8WNtpPnYsbUJ39DpF/A= k8s.io/kube-aggregator v0.19.4/go.mod h1:cTkvun110194d797AuThyydBBlgm+cKIFUeS2uzGJfU= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kubectl v0.19.0/go.mod h1:gPCjjsmE6unJzgaUNXIFGZGafiUp5jh0If3F/x7/rRg= k8s.io/kubectl v0.19.4/go.mod h1:XPmlu4DJEYgD83pvZFeKF8+MSvGnYGqunbFSrJsqHv0= @@ -2186,7 +2183,6 @@ sigs.k8s.io/structured-merge-diff v1.0.1/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8AP sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= diff --git a/pkg/cloudprovider/provider/anexia/helper_test.go b/pkg/cloudprovider/provider/anexia/helper_test.go new file mode 100644 index 000000000..2edc3ba32 --- /dev/null +++ b/pkg/cloudprovider/provider/anexia/helper_test.go @@ -0,0 +1,91 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package anexia + +import ( + "encoding/json" + "net/http" + "testing" + + "github.com/anexia-it/go-anxcloud/pkg/vsphere/search" + "github.com/gophercloud/gophercloud/testhelper" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" + "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8s.io/apimachinery/pkg/runtime" +) + +type ConfigTestCase struct { + Config anxtypes.RawConfig + Error error +} + +type ValidateCallTestCase struct { + Spec v1alpha1.MachineSpec + ExpectedError error +} + +func getSpecsForValidationTest(t *testing.T, configCases []ConfigTestCase) []ValidateCallTestCase { + + var testCases []ValidateCallTestCase + + for _, configCase := range configCases { + jsonConfig, err := json.Marshal(configCase.Config) + testhelper.AssertNoErr(t, err) + jsonProviderConfig, err := json.Marshal(types.Config{ + CloudProviderSpec: runtime.RawExtension{Raw: jsonConfig}, + OperatingSystemSpec: runtime.RawExtension{Raw: []byte("{}")}, + }) + testhelper.AssertNoErr(t, err) + testCases = append(testCases, ValidateCallTestCase{ + Spec: v1alpha1.MachineSpec{ + ProviderSpec: v1alpha1.ProviderSpec{ + Value: &runtime.RawExtension{Raw: jsonProviderConfig}, + }, + }, + ExpectedError: configCase.Error, + }) + } + return testCases +} + +func createSearchHandler(t *testing.T, iterations int) http.HandlerFunc { + counter := 0 + return func(writer http.ResponseWriter, request *http.Request) { + test := request.URL.Query().Get("name") + testhelper.AssertEquals(t, "%-TestMachine", test) + testhelper.TestMethod(t, request, http.MethodGet) + if iterations == counter { + encoder := json.NewEncoder(writer) + testhelper.AssertNoErr(t, encoder.Encode(map[string]interface{}{ + "data": []search.VM{ + { + Name: "543053-TestMachine", + Identifier: TestIdentifier, + }, + }, + })) + } + counter++ + } +} + +func newConfigVarString(str string) types.ConfigVarString { + return types.ConfigVarString{ + Value: str, + } +} diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 646882e2a..796a83336 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -23,10 +23,18 @@ import ( "errors" "fmt" "net/http" + "time" + + "github.com/anexia-it/go-anxcloud/pkg/vsphere/provisioning/progress" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/utils" + "k8s.io/apimachinery/pkg/api/meta" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" anxclient "github.com/anexia-it/go-anxcloud/pkg/client" anxaddr "github.com/anexia-it/go-anxcloud/pkg/ipam/address" - anxvsphere "github.com/anexia-it/go-anxcloud/pkg/vsphere" + "github.com/anexia-it/go-anxcloud/pkg/vsphere" anxvm "github.com/anexia-it/go-anxcloud/pkg/vsphere/provisioning/vm" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" @@ -43,25 +51,239 @@ import ( k8stypes "k8s.io/apimachinery/pkg/types" ) -type Config struct { - Token string - VlanID string - LocationID string - TemplateID string - CPUs int - Memory int - DiskSize int -} +const ( + ProvisionedType = "Provisioned" +) type provider struct { configVarResolver *providerconfig.ConfigVarResolver } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, + userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance instance.Instance, retErr error) { + status := getProviderStatus(machine) + klog.V(3).Infof(fmt.Sprintf("'%s' has status %#v", machine.Name, status)) + + // ensure conditions are present on machine + ensureConditions(&status) + + config, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return nil, fmt.Errorf("unable to get provider config: %w", err) + } + + ctx := utils.CreateReconcileContext(utils.ReconcileContext{ + Status: &status, + UserData: userdata, + Config: config, + ProviderData: data, + Machine: machine, + }) + + client, err := getClient(config.Token) + if err != nil { + return nil, err + } + + // make sure status is reflected in Machine Object + defer func() { + // if error occurs during updating the machine object don't override the original error + retErr = anxtypes.NewMultiError(retErr, updateMachineStatus(machine, status, data.Update)) + }() + + // check whether machine is already provisioning + if isAlreadyProvisioning(ctx) && status.ProvisioningID == "" { + klog.Info("ongoing provisioning detected") + err := waitForVM(ctx, client) + if err != nil { + return nil, err + } + return p.Get(machine, data) + } + + // provision machine + err = provisionVM(ctx, client) + if err != nil { + return nil, err + } + return p.Get(machine, data) +} + +func waitForVM(ctx context.Context, client anxclient.Client) error { + reconcileContext := utils.GetReconcileContext(ctx) + api := vsphere.NewAPI(client) + var identifier string + err := wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) { + klog.V(2).Info("checking for VM with name ", reconcileContext.Machine.Name) + vms, err := api.Search().ByName(ctx, fmt.Sprintf("%%-%s", reconcileContext.Machine.Name)) + if err != nil { + return false, nil + } + if len(vms) < 1 { + return false, nil + } + if len(vms) > 1 { + return false, errors.New("too many VMs returned by search") + } + identifier = vms[0].Identifier + return true, nil + }) + if err != nil { + return err + } + + reconcileContext.Status.InstanceID = identifier + return updateMachineStatus(reconcileContext.Machine, *reconcileContext.Status, reconcileContext.ProviderData.Update) +} + +func provisionVM(ctx context.Context, client anxclient.Client) error { + reconcileContext := utils.GetReconcileContext(ctx) + vmAPI := vsphere.NewAPI(client) + + ctx, cancel := context.WithTimeout(ctx, anxtypes.CreateRequestTimeout) + defer cancel() + + status := reconcileContext.Status + if status.ProvisioningID == "" { + klog.V(2).Info(fmt.Sprintf("Machine '%s' does not contain a provisioningID yet. Starting to provision", + reconcileContext.Machine.Name)) + + config := reconcileContext.Config + reservedIP, err := getIPAddress(ctx, client) + if err != nil { + return newError(common.CreateMachineError, "failed to reserve IP: %v", err) + } + networkInterfaces := []anxvm.Network{{ + NICType: anxtypes.VmxNet3NIC, + IPs: []string{reservedIP}, + VLAN: config.VlanID, + }} + + vm := vmAPI.Provisioning().VM().NewDefinition( + config.LocationID, + "templates", + config.TemplateID, + reconcileContext.Machine.Name, + config.CPUs, + config.Memory, + config.DiskSize, + networkInterfaces, + ) + + vm.Script = base64.StdEncoding.EncodeToString([]byte(reconcileContext.UserData)) + + sshKey, err := ssh.NewKey() + if err != nil { + return newError(common.CreateMachineError, "failed to generate ssh key: %v", err) + } + vm.SSH = sshKey.PublicKey + + provisionResponse, err := vmAPI.Provisioning().VM().Provision(ctx, vm, false) + meta.SetStatusCondition(&status.Conditions, v1.Condition{ + Type: ProvisionedType, + Status: v1.ConditionFalse, + Reason: "Provisioning", + Message: "provisioning request was sent", + }) + if err != nil { + return newError(common.CreateMachineError, "instance provisioning failed: %v", err) + } + + // we successfully sent a VM provisioning request to the API, we consider the IP as 'Bound' now + status.IPState = anxtypes.IPStateBound + + status.ProvisioningID = provisionResponse.Identifier + err = updateMachineStatus(reconcileContext.Machine, *status, reconcileContext.ProviderData.Update) + if err != nil { + return err + } + } + + klog.V(2).Info(fmt.Sprintf("Using provisionID from machine '%s' to await completion", + reconcileContext.Machine.Name)) + + instanceID, err := vmAPI.Provisioning().Progress().AwaitCompletion(ctx, status.ProvisioningID) + if err != nil { + klog.Errorf("failed to await machine completion '%s'", reconcileContext.Machine.Name) + // something went wrong remove provisioning ID, so we can start from scratch + status.ProvisioningID = "" + return newError(common.CreateMachineError, "instance provisioning failed: %v", err) + } + + status.InstanceID = instanceID + meta.SetStatusCondition(&status.Conditions, v1.Condition{ + Type: ProvisionedType, + Status: v1.ConditionTrue, + Reason: "Provisioned", + Message: "Machine has been successfully created", + }) + + return updateMachineStatus(reconcileContext.Machine, *status, reconcileContext.ProviderData.Update) +} + +func getIPAddress(ctx context.Context, client anxclient.Client) (string, error) { + reconcileContext := utils.GetReconcileContext(ctx) + status := reconcileContext.Status + + // only use IP if it is still unbound + if status.ReservedIP != "" && status.IPState == anxtypes.IPStateUnbound { + klog.Info("reusing already provisioned ip", "IP", status.ReservedIP) + return status.ReservedIP, nil + } + klog.Info(fmt.Sprintf("Creating a new IP for machine ''%s", reconcileContext.Machine.Name)) + addrAPI := anxaddr.NewAPI(client) + config := reconcileContext.Config + res, err := addrAPI.ReserveRandom(ctx, anxaddr.ReserveRandom{ + LocationID: config.LocationID, + VlanID: config.VlanID, + Count: 1, + }) + if err != nil { + return "", newError(common.InvalidConfigurationMachineError, "failed to reserve an ip address: %v", err) + } + if len(res.Data) < 1 { + return "", newError(common.InsufficientResourcesMachineError, "no ip address is available for this machine") + } + + ip := res.Data[0].Address + status.ReservedIP = ip + status.IPState = anxtypes.IPStateUnbound + + return ip, nil +} + +func isAlreadyProvisioning(ctx context.Context) bool { + status := utils.GetReconcileContext(ctx).Status + condition := meta.FindStatusCondition(status.Conditions, ProvisionedType) + lastChange := condition.LastTransitionTime.Time + const reasonInProvisioning = "InProvisioning" + if condition.Reason == reasonInProvisioning && time.Since(lastChange) > 5*time.Minute { + meta.SetStatusCondition(&status.Conditions, v1.Condition{ + Type: ProvisionedType, + Reason: "ReInitialising", + Message: "Could not find ongoing VM provisioning", + Status: v1.ConditionFalse, + }) + } + + return condition.Status == v1.ConditionFalse && condition.Reason == reasonInProvisioning +} + +func ensureConditions(status *anxtypes.ProviderStatus) { + conditions := [...]v1.Condition{ + {Type: ProvisionedType, Message: "", Status: v1.ConditionUnknown, Reason: "Initialising"}, + } + for _, condition := range conditions { + if meta.FindStatusCondition(status.Conditions, condition.Type) == nil { + meta.SetStatusCondition(&status.Conditions, condition) + } + } +} + +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*anxtypes.Config, *providerconfigtypes.Config, error) { if provSpec.Value == nil { return nil, nil, fmt.Errorf("machine.spec.providerSpec.value is nil") } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err @@ -76,7 +298,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, err } - c := Config{} + c := anxtypes.Config{} c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, anxtypes.AnxTokenEnv) if err != nil { return nil, nil, fmt.Errorf("failed to get 'token': %v", err) @@ -162,9 +384,9 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P if err != nil { return nil, newError(common.InvalidConfigurationMachineError, "failed to create Anexia client: %v", err) } - vsphere := anxvsphere.NewAPI(cli) + vsphereAPI := vsphere.NewAPI(cli) - status, err := getStatus(machine.Status.ProviderStatus) + status := getProviderStatus(machine) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, "failed to get machine status: %v", err) } @@ -175,7 +397,7 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P ctx, cancel := context.WithTimeout(context.Background(), anxtypes.GetRequestTimeout) defer cancel() - info, err := vsphere.Info().Get(ctx, status.InstanceID) + info, err := vsphereAPI.Info().Get(ctx, status.InstanceID) if err != nil { return nil, fmt.Errorf("failed get machine info: %w", err) } @@ -185,151 +407,99 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P }, nil } -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (string, string, error) { +func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string, error) { return "", "", nil } -// Create creates a cloud instance according to the given machine -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { +func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (isDeleted bool, retErr error) { + status := getProviderStatus(machine) + // make sure status is reflected in Machine Object + defer func() { + // if error occurs during updating the machine object don't override the original error + retErr = anxtypes.NewMultiError(retErr, updateMachineStatus(machine, status, data.Update)) + }() + + ensureConditions(&status) config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, "failed to parse MachineSpec: %v", err) + return false, newError(common.InvalidConfigurationMachineError, "failed to parse MachineSpec: %v", err) } cli, err := getClient(config.Token) if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, "failed to create Anexia client: %v", err) + return false, newError(common.InvalidConfigurationMachineError, "failed to create Anexia client: %v", err) } - vsphere := anxvsphere.NewAPI(cli) - addr := anxaddr.NewAPI(cli) - - ctx, cancel := context.WithTimeout(context.Background(), anxtypes.CreateRequestTimeout) - defer cancel() + vsphereAPI := vsphere.NewAPI(cli) - status, err := getStatus(machine.Status.ProviderStatus) if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, "failed to get machine status: %v", err) + return false, newError(common.InvalidConfigurationMachineError, "failed to get machine status: %v", err) } - if status.ProvisioningID == "" { - res, err := addr.ReserveRandom(ctx, anxaddr.ReserveRandom{ - LocationID: config.LocationID, - VlanID: config.VlanID, - Count: 1, - }) - if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, "failed to reserve an ip address: %v", err) - } - if len(res.Data) < 1 { - return nil, newError(common.InsufficientResourcesMachineError, "no ip address is available for this machine") - } - - networkInterfaces := []anxvm.Network{{ - NICType: anxtypes.VmxNet3NIC, - IPs: []string{res.Data[0].Address}, - VLAN: config.VlanID, - }} - - vm := vsphere.Provisioning().VM().NewDefinition( - config.LocationID, - "templates", - config.TemplateID, - machine.ObjectMeta.Name, - config.CPUs, - config.Memory, - config.DiskSize, - networkInterfaces, - ) - - vm.Script = base64.StdEncoding.EncodeToString([]byte(userdata)) - - sshKey, err := ssh.NewKey() - if err != nil { - return nil, newError(common.CreateMachineError, "failed to generate ssh key: %v", err) - } - vm.SSH = sshKey.PublicKey + ctx, cancel := context.WithTimeout(context.Background(), anxtypes.DeleteRequestTimeout) + defer cancel() - provisionResponse, err := vsphere.Provisioning().VM().Provision(ctx, vm) + // first check whether there is an provisioning ongoing + if status.DeprovisioningID == "" { + response, err := vsphereAPI.Provisioning().VM().Deprovision(ctx, status.InstanceID, false) if err != nil { - return nil, newError(common.CreateMachineError, "instance provisioning failed: %v", err) - } - - status.ProvisioningID = provisionResponse.Identifier - if err := updateStatus(machine, status, data.Update); err != nil { - return nil, newError(common.UpdateMachineError, "machine status update failed: %v", err) + var respErr *anxclient.ResponseError + // Only error if the error was not "not found" + if !(errors.As(err, &respErr) && respErr.ErrorData.Code == http.StatusNotFound) { + return false, newError(common.DeleteMachineError, "failed to delete machine: %v", err) + } } + status.DeprovisioningID = response.Identifier } - instanceID, err := vsphere.Provisioning().Progress().AwaitCompletion(ctx, status.ProvisioningID) - if err != nil { - return nil, newError(common.CreateMachineError, "instance provisioning failed: %v", err) - } - - status.InstanceID = instanceID - if err := updateStatus(machine, status, data.Update); err != nil { - return nil, newError(common.UpdateMachineError, "machine status update failed: %v", err) - } - - return p.Get(machine, data) + return isTaskDone(ctx, cli, status.DeprovisioningID) } -func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { - config, _, err := p.getConfig(machine.Spec.ProviderSpec) - if err != nil { - return false, newError(common.InvalidConfigurationMachineError, "failed to parse MachineSpec: %v", err) - } - - cli, err := getClient(config.Token) +func isTaskDone(ctx context.Context, cli anxclient.Client, progressIdentifier string) (bool, error) { + response, err := progress.NewAPI(cli).Get(ctx, progressIdentifier) if err != nil { - return false, newError(common.InvalidConfigurationMachineError, "failed to create Anexia client: %v", err) + return false, err } - vsphere := anxvsphere.NewAPI(cli) - status, err := getStatus(machine.Status.ProviderStatus) - if err != nil { - return false, newError(common.InvalidConfigurationMachineError, "failed to get machine status: %v", err) + if len(response.Errors) != 0 { + taskErrors, _ := json.Marshal(response.Errors) + return true, fmt.Errorf("task failed with: %s", taskErrors) } - ctx, cancel := context.WithTimeout(context.Background(), anxtypes.DeleteRequestTimeout) - defer cancel() - - err = vsphere.Provisioning().VM().Deprovision(ctx, status.InstanceID, false) - if err != nil { - var respErr *anxclient.ResponseError - // Only error if the error was not "not found" - if !(errors.As(err, &respErr) && respErr.ErrorData.Code == http.StatusNotFound) { - return false, newError(common.DeleteMachineError, "failed to delete machine: %v", err) - } + if response.Progress == 100 { + return true, nil } - return true, nil + return false, nil } func (p *provider) MigrateUID(_ *clusterv1alpha1.Machine, _ k8stypes.UID) error { return nil } -func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { +func (p *provider) MachineMetricsLabels(_ *clusterv1alpha1.Machine) (map[string]string, error) { return map[string]string{}, nil } -func (p *provider) SetMetricsForMachines(machine clusterv1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } func getClient(token string) (anxclient.Client, error) { tokenOpt := anxclient.TokenFromString(token) - return anxclient.New(tokenOpt) + client := anxclient.HTTPClient(&http.Client{Timeout: 30 * time.Second}) + return anxclient.New(tokenOpt, client) } -func getStatus(rawStatus *runtime.RawExtension) (*anxtypes.ProviderStatus, error) { - var status anxtypes.ProviderStatus - if rawStatus != nil && rawStatus.Raw != nil { - if err := json.Unmarshal(rawStatus.Raw, &status); err != nil { - return nil, err +func getProviderStatus(machine *clusterv1alpha1.Machine) anxtypes.ProviderStatus { + var providerStatus anxtypes.ProviderStatus + status := machine.Status.ProviderStatus + if status != nil && status.Raw != nil { + if err := json.Unmarshal(status.Raw, &providerStatus); err != nil { + klog.Warningf("Unable to parse status from machine object. status was discarded for machine") + return anxtypes.ProviderStatus{} } } - return &status, nil + return providerStatus } // newError creates a terminal error matching to the provider interface. @@ -340,7 +510,9 @@ func newError(reason common.MachineStatusError, msg string, args ...interface{}) } } -func updateStatus(machine *clusterv1alpha1.Machine, status *anxtypes.ProviderStatus, updater cloudprovidertypes.MachineUpdater) error { +// updateMachineStatus tries to update the machine status by any means +// an error will lead to a panic +func updateMachineStatus(machine *clusterv1alpha1.Machine, status anxtypes.ProviderStatus, updater cloudprovidertypes.MachineUpdater) error { rawStatus, err := json.Marshal(status) if err != nil { return err @@ -350,6 +522,7 @@ func updateStatus(machine *clusterv1alpha1.Machine, status *anxtypes.ProviderSta Raw: rawStatus, } }) + if err != nil { return err } diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go new file mode 100644 index 000000000..4fe746704 --- /dev/null +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -0,0 +1,333 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package anexia + +import ( + "encoding/json" + "errors" + "net/http" + "testing" + "time" + + anxclient "github.com/anexia-it/go-anxcloud/pkg/client" + "github.com/anexia-it/go-anxcloud/pkg/ipam/address" + "github.com/anexia-it/go-anxcloud/pkg/vsphere/provisioning/progress" + "github.com/anexia-it/go-anxcloud/pkg/vsphere/provisioning/vm" + "github.com/gophercloud/gophercloud/testhelper" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/utils" + cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +const TestIdentifier = "TestIdent" + +func TestAnexiaProvider(t *testing.T) { + testhelper.SetupHTTP() + client, server := anxclient.NewTestClient(nil, testhelper.Mux) + t.Cleanup(func() { + testhelper.TeardownHTTP() + server.Close() + }) + + t.Run("Test waiting for VM", func(t *testing.T) { + t.Parallel() + + waitUntilVMIsFound := 2 + testhelper.Mux.HandleFunc("/api/vsphere/v1/search/by_name.json", createSearchHandler(t, waitUntilVMIsFound)) + + providerStatus := anxtypes.ProviderStatus{} + ctx := utils.CreateReconcileContext(utils.ReconcileContext{ + Machine: &v1alpha1.Machine{ + ObjectMeta: metav1.ObjectMeta{Name: "TestMachine"}, + }, + Status: &providerStatus, + UserData: "", + Config: &anxtypes.Config{}, + + ProviderData: &cloudprovidertypes.ProviderData{ + Update: func(m *clusterv1alpha1.Machine, mod ...cloudprovidertypes.MachineModifier) error { + return nil + }, + }, + }) + + err := waitForVM(ctx, client) + if err != nil { + t.Fatal("No error was expected", err) + + } + + if providerStatus.InstanceID != TestIdentifier { + t.Errorf("Excpected InstanceID to be set") + } + }) + + t.Run("Test provision VM", func(t *testing.T) { + t.Parallel() + testhelper.Mux.HandleFunc("/api/ipam/v1/address/reserve/ip/count.json", func(writer http.ResponseWriter, request *http.Request) { + err := json.NewEncoder(writer).Encode(address.ReserveRandomSummary{ + Data: []address.ReservedIP{ + { + ID: "IP-ID", + Address: "8.8.8.8", + }, + }, + }) + testhelper.AssertNoErr(t, err) + }) + + testhelper.Mux.HandleFunc("/api/vsphere/v1/provisioning/vm.json/LOCATION-ID/templates/TEMPLATE-ID", func(writer http.ResponseWriter, request *http.Request) { + testhelper.TestMethod(t, request, http.MethodPost) + type jsonObject = map[string]interface{} + expectedJSON := map[string]interface{}{ + "cpu_performance_type": "performance", + "hostname": "TestMachine", + "memory_mb": json.Number("5"), + "network": []jsonObject{ + { + "vlan": "VLAN-ID", + "nic_type": "vmxnet3", + "ips": []interface{}{"8.8.8.8"}, + }, + }, + } + var jsonBody jsonObject + decoder := json.NewDecoder(request.Body) + decoder.UseNumber() + testhelper.AssertNoErr(t, decoder.Decode(&jsonBody)) + testhelper.AssertEquals(t, expectedJSON["cpu_performance_type"], jsonBody["cpu_performance_type"]) + testhelper.AssertEquals(t, expectedJSON["hostname"], jsonBody["hostname"]) + testhelper.AssertEquals(t, expectedJSON["memory_mb"], jsonBody["memory_mb"]) + testhelper.AssertEquals(t, expectedJSON["count"], jsonBody["count"]) + + expectedNetwork := expectedJSON["network"].([]jsonObject)[0] + bodyNetwork := jsonBody["network"].([]interface{})[0].(jsonObject) + testhelper.AssertEquals(t, expectedNetwork["vlan"], bodyNetwork["vlan"]) + testhelper.AssertEquals(t, expectedNetwork["nic_type"], bodyNetwork["nic_type"]) + testhelper.AssertEquals(t, expectedNetwork["ips"].([]interface{})[0], bodyNetwork["ips"].([]interface{})[0]) + + err := json.NewEncoder(writer).Encode(vm.ProvisioningResponse{ + Progress: 100, + Errors: nil, + Identifier: "TEST-IDENTIFIER", + Queued: false, + }) + testhelper.AssertNoErr(t, err) + }) + + testhelper.Mux.HandleFunc("/api/vsphere/v1/provisioning/progress.json/TEST-IDENTIFIER", func(writer http.ResponseWriter, request *http.Request) { + testhelper.TestMethod(t, request, http.MethodGet) + + err := json.NewEncoder(writer).Encode(progress.Progress{ + TaskIdentifier: "TEST-IDENTIFIER", + Queued: false, + Progress: 100, + VMIdentifier: "VM-IDENTIFIER", + Errors: nil, + }) + testhelper.AssertNoErr(t, err) + }) + + providerStatus := anxtypes.ProviderStatus{} + ctx := utils.CreateReconcileContext(utils.ReconcileContext{ + Machine: &v1alpha1.Machine{ + ObjectMeta: metav1.ObjectMeta{Name: "TestMachine"}, + }, + Status: &providerStatus, + UserData: "", + Config: &anxtypes.Config{ + VlanID: "VLAN-ID", + LocationID: "LOCATION-ID", + TemplateID: "TEMPLATE-ID", + CPUs: 5, + Memory: 5, + DiskSize: 5, + }, + ProviderData: &cloudprovidertypes.ProviderData{ + Update: func(m *clusterv1alpha1.Machine, mods ...cloudprovidertypes.MachineModifier) error { + return nil + }, + }, + }) + + err := provisionVM(ctx, client) + testhelper.AssertNoErr(t, err) + }) + + t.Run("Test is VM Provisioning", func(t *testing.T) { + t.Parallel() + providerStatus := anxtypes.ProviderStatus{ + Conditions: []metav1.Condition{ + { + Type: ProvisionedType, + Reason: "InProvisioning", + Status: metav1.ConditionFalse, + }, + }, + } + ctx := utils.CreateReconcileContext(utils.ReconcileContext{ + Status: &providerStatus, + UserData: "", + Config: nil, + ProviderData: nil, + }) + + condition := meta.FindStatusCondition(providerStatus.Conditions, ProvisionedType) + condition.LastTransitionTime = metav1.Time{Time: time.Now().Add(-1 * time.Minute)} + testhelper.AssertEquals(t, true, isAlreadyProvisioning(ctx)) + + condition.Reason = "Provisioned" + condition.Status = metav1.ConditionTrue + testhelper.AssertEquals(t, false, isAlreadyProvisioning(ctx)) + + condition.Reason = "InProvisioning" + condition.Status = metav1.ConditionFalse + condition.LastTransitionTime = metav1.Time{Time: time.Now().Add(-10 * time.Minute)} + testhelper.AssertEquals(t, false, isAlreadyProvisioning(ctx)) + testhelper.AssertEquals(t, condition.Reason, "ReInitialising") + }) + + t.Run("Test getIPAddress", func(t *testing.T) { + t.Parallel() + providerStatus := &anxtypes.ProviderStatus{ + ReservedIP: "", + IPState: "", + } + ctx := utils.CreateReconcileContext(utils.ReconcileContext{Status: providerStatus}) + + t.Run("with unbound reserved IP", func(t *testing.T) { + expectedIP := "8.8.8.8" + providerStatus.ReservedIP = expectedIP + providerStatus.IPState = anxtypes.IPStateUnbound + reservedIP, err := getIPAddress(ctx, client) + testhelper.AssertNoErr(t, err) + testhelper.AssertEquals(t, expectedIP, reservedIP) + }) + }) +} + +func TestValidate(t *testing.T) { + t.Parallel() + + var configCases []ConfigTestCase + configCases = append(configCases, + ConfigTestCase{ + Config: anxtypes.RawConfig{}, + Error: errors.New("token is missing"), + }, + ConfigTestCase{ + Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN")}, + Error: errors.New("cpu count is missing"), + }, + ConfigTestCase{ + Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1}, + Error: errors.New("disk size is missing"), + }, + ConfigTestCase{ + Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1, DiskSize: 5}, + Error: errors.New("memory size is missing"), + }, + ConfigTestCase{ + Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1, DiskSize: 5, Memory: 5}, + Error: errors.New("location id is missing"), + }, + ConfigTestCase{ + Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1, DiskSize: 5, Memory: 5, + LocationID: newConfigVarString("TLID")}, + Error: errors.New("template id is missing"), + }, + ConfigTestCase{ + Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1, DiskSize: 5, Memory: 5, + LocationID: newConfigVarString("LID"), TemplateID: newConfigVarString("TID")}, + Error: errors.New("vlan id is missing"), + }, + ConfigTestCase{ + Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1, DiskSize: 5, Memory: 5, + LocationID: newConfigVarString("LID"), TemplateID: newConfigVarString("TID"), VlanID: newConfigVarString("VLAN")}, + Error: nil, + }, + ) + + provider := New(nil) + for _, testCase := range getSpecsForValidationTest(t, configCases) { + err := provider.Validate(testCase.Spec) + if testCase.ExpectedError != nil { + testhelper.AssertEquals(t, testCase.ExpectedError.Error(), err.Error()) + } else { + testhelper.AssertEquals(t, testCase.ExpectedError, err) + } + } +} + +func TestEnsureConditions(t *testing.T) { + t.Parallel() + status := anxtypes.ProviderStatus{} + + ensureConditions(&status) + + condition := meta.FindStatusCondition(status.Conditions, ProvisionedType) + if condition == nil { + t.Fatal("condition should not be nil") + } + testhelper.AssertEquals(t, metav1.ConditionUnknown, condition.Status) + testhelper.AssertEquals(t, "Initialising", condition.Reason) +} + +func TestGetProviderStatus(t *testing.T) { + t.Parallel() + + machine := &v1alpha1.Machine{} + providerStatus := anxtypes.ProviderStatus{ + InstanceID: "InstanceID", + } + providerStatusJSON, err := json.Marshal(providerStatus) + testhelper.AssertNoErr(t, err) + machine.Status.ProviderStatus = &runtime.RawExtension{Raw: providerStatusJSON} + + returnedStatus := getProviderStatus(machine) + + testhelper.AssertEquals(t, "InstanceID", returnedStatus.InstanceID) + +} + +func TestUpdateStatus(t *testing.T) { + t.Parallel() + machine := &v1alpha1.Machine{} + providerStatus := anxtypes.ProviderStatus{ + InstanceID: "InstanceID", + } + providerStatusJSON, err := json.Marshal(providerStatus) + testhelper.AssertNoErr(t, err) + machine.Status.ProviderStatus = &runtime.RawExtension{Raw: providerStatusJSON} + + called := false + err = updateMachineStatus(machine, providerStatus, func(paramMachine *v1alpha1.Machine, modifier ...cloudprovidertypes.MachineModifier) error { + called = true + testhelper.AssertEquals(t, machine, paramMachine) + status := getProviderStatus(machine) + testhelper.AssertEquals(t, status.InstanceID, providerStatus.InstanceID) + return nil + }) + + testhelper.AssertEquals(t, true, called) + testhelper.AssertNoErr(t, err) +} diff --git a/pkg/cloudprovider/provider/anexia/types/errors.go b/pkg/cloudprovider/provider/anexia/types/errors.go new file mode 100644 index 000000000..6e73d232f --- /dev/null +++ b/pkg/cloudprovider/provider/anexia/types/errors.go @@ -0,0 +1,49 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "fmt" + "strings" +) + +// MultiError represent multiple errors at the same time. +type MultiError []error + +func (r MultiError) Error() string { + errString := make([]string, len(r)) + for i, err := range r { + errString[i] = fmt.Sprintf("Error %d: %s", i, err) + } + return fmt.Sprintf("Multiple errors occoured:\n%s", strings.Join(errString, "\n")) +} + +func NewMultiError(errs ...error) error { + var combinedErr []error + for _, err := range errs { + if err == nil { + continue + } + combinedErr = append(combinedErr, err) + } + + if len(combinedErr) > 0 { + return MultiError(combinedErr) + } + + return nil +} diff --git a/pkg/cloudprovider/provider/anexia/types/types.go b/pkg/cloudprovider/provider/anexia/types/types.go index 9bdc929fd..1046315fb 100644 --- a/pkg/cloudprovider/provider/anexia/types/types.go +++ b/pkg/cloudprovider/provider/anexia/types/types.go @@ -17,6 +17,9 @@ limitations under the License. package types import ( + "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" + cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "time" "github.com/kubermatic/machine-controller/pkg/jsonutil" @@ -30,10 +33,18 @@ const ( GetRequestTimeout = 1 * time.Minute DeleteRequestTimeout = 1 * time.Minute + IPStateBound = "Bound" + IPStateUnbound = "Unbound" + VmxNet3NIC = "vmxnet3" MachinePoweredOn = "poweredOn" ) +var StatusUpdateFailed = cloudprovidererrors.TerminalError{ + Reason: common.UpdateMachineError, + Message: "Unable to update the machine status", +} + type RawConfig struct { Token providerconfigtypes.ConfigVarString `json:"token,omitempty"` VlanID providerconfigtypes.ConfigVarString `json:"vlanID"` @@ -45,9 +56,22 @@ type RawConfig struct { } type ProviderStatus struct { - InstanceID string `json:"instanceID"` - ProvisioningID string `json:"provisioningID"` - // TODO: add conditions to track progress on the provider side + InstanceID string `json:"instanceID"` + ProvisioningID string `json:"provisioningID"` + DeprovisioningID string `json:"deprovisioningID"` + ReservedIP string `json:"reservedIP"` + IPState string `json:"ipState"` + Conditions []v1.Condition `json:"conditions,omitempty"` +} + +type Config struct { + Token string + VlanID string + LocationID string + TemplateID string + CPUs int + Memory int + DiskSize int } func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { diff --git a/pkg/cloudprovider/provider/anexia/utils/utils.go b/pkg/cloudprovider/provider/anexia/utils/utils.go new file mode 100644 index 000000000..97aa73d5e --- /dev/null +++ b/pkg/cloudprovider/provider/anexia/utils/utils.go @@ -0,0 +1,49 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + + "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" + cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" +) + +type contextKey byte + +const MachineReconcileContextKey contextKey = 0 + +type ReconcileContext struct { + Machine *v1alpha1.Machine + Status *anxtypes.ProviderStatus + UserData string + Config *anxtypes.Config + ProviderData *cloudprovidertypes.ProviderData +} + +func CreateReconcileContext(cc ReconcileContext) context.Context { + return context.WithValue(context.Background(), MachineReconcileContextKey, cc) +} + +func GetReconcileContext(ctx context.Context) ReconcileContext { + rawContext := ctx.Value(MachineReconcileContextKey) + if recContext, ok := rawContext.(ReconcileContext); ok { + return recContext + } + return ReconcileContext{} +} From 5a13855222704ee81ac52c93fe03ba280702bb73 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Wed, 23 Mar 2022 17:15:24 +0100 Subject: [PATCH 096/489] remvoe youssefazrak from the MC maintainers (#1221) Signed-off-by: Moath Qasim --- OWNERS_ALIASES | 1 - 1 file changed, 1 deletion(-) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 23f2f754c..42b4f9a99 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -11,7 +11,6 @@ aliases: - themue - xmudrii - xrstf - - youssefazrak # Temporary SIG to oversee changes in userdata and cloudprovider sub-directories # This SIG is responsible for ensuring that OSM and machine-controller are in sync From 29140d34e32e88354713725ed72e0feaf09dd080 Mon Sep 17 00:00:00 2001 From: Frank Mueller Date: Thu, 24 Mar 2022 13:13:12 +0100 Subject: [PATCH 097/489] Add Helene and Sankalp o the owners (#1223) Signed-off-by: themue --- .gitignore | 3 ++- OWNERS_ALIASES | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b782ea31f..b5a1dac67 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ examples/*.csr examples/*.srl /webhook /vendor -.vscode \ No newline at end of file +.vscode +.gitpod.yml \ No newline at end of file diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 42b4f9a99..1d4d78a34 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -4,10 +4,12 @@ aliases: machine-controller-maintainers: - ahmedwaleedmalik - embik + - hdurand0710 - kron4eg - mfranczy - moadqassem - moelsayed + - sankalp-r - themue - xmudrii - xrstf From 8291d4121307cae5f628af18c6973643fccfdcac Mon Sep 17 00:00:00 2001 From: Helene Durand <90329641+hdurand0710@users.noreply.github.com> Date: Thu, 24 Mar 2022 19:52:28 +0100 Subject: [PATCH 098/489] Kubevirt Add common label to all VMs linked to the same MachineDeployment (#1217) * Kubevirt Add common label to all VMs linked to the same MachineDeployment Signed-off-by: Helene Durand * Use the helper function to get the MD name Signed-off-by: Helene Durand --- examples/kubevirt-machinedeployment.yaml | 19 +++++++++++-------- .../provider/kubevirt/provider.go | 8 +++++++- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index f8bc5c295..20c423ca9 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -26,14 +26,17 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "kubevirt" cloudProviderSpec: - storageClassName: kubermatic-fast - pvcSize: "10Gi" - sourceURL: http://10.109.79.210/<< OS_NAME >>.img - cpus: "1" - memory: "2048M" - kubeconfig: - value: '<< KUBECONFIG >>' - namespace: kube-system + auth: + kubeconfig: + value: '<< KUBECONFIG >>' + virtualMachine: + template: + cpus: "1" + memory: "2048M" + primaryDisk: + osImage: http://10.109.79.210/<< OS_NAME >>.img + size: "10Gi" + storageClassName: kubermatic-fast # Can also be `centos`, must align with he configured registryImage above operatingSystem: "ubuntu" operatingSystemSpec: diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 1b69254cd..af171fbe5 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "net/url" "os" "strconv" "strings" @@ -34,8 +35,10 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" kubevirttypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/kubevirt/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + controllerutil "github.com/kubermatic/machine-controller/pkg/controller/util" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -46,7 +49,6 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/klog" utilpointer "k8s.io/utils/pointer" - "net/url" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -373,6 +375,10 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert resourceRequirements := kubevirtv1.ResourceRequirements{} labels := map[string]string{"kubevirt.io/vm": machine.Name} + // Add a common label to all VirtualMachines spawned by the same MachineDeployment (= MachineDeployment name) + if mdName, err := controllerutil.GetMachineDeploymentNameForMachine(context.Background(), machine, data.Client); err == nil { + labels["md"] = mdName + } sigClient, err := client.New(c.RestConfig, client.Options{}) if err != nil { From 2b4c9c56d2fca4b81a7cd2a24471588a3869032f Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Thu, 24 Mar 2022 21:20:43 +0200 Subject: [PATCH 099/489] New -kubernetes-version-constraints webhook flag (#1222) Signed-off-by: Artiom Diomin --- cmd/webhook/main.go | 20 +++++++++++++++++++- pkg/admission/admission.go | 38 ++++++++++++++++++++++---------------- pkg/admission/machines.go | 10 ++++++++++ 3 files changed, 51 insertions(+), 17 deletions(-) diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 871b8e897..61c2bc904 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -19,6 +19,8 @@ package main import ( "flag" + "github.com/Masterminds/semver/v3" + "github.com/kubermatic/machine-controller/pkg/admission" "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/node" @@ -41,6 +43,7 @@ type options struct { useOSM bool namespace string workerClusterKubeconfig string + versionConstraint string } func main() { @@ -60,6 +63,7 @@ func main() { flag.StringVar(&opt.caBundleFile, "ca-bundle", "", "path to a file containing all PEM-encoded CA certificates (will be used instead of the host's certificates if set)") flag.StringVar(&opt.namespace, "namespace", "kubermatic", "The namespace where the webhooks will run") flag.StringVar(&opt.workerClusterKubeconfig, "worker-cluster-kubeconfig", "", "Path to kubeconfig of worker/user cluster where machines and machinedeployments exist. If not specified, value from --kubeconfig or in-cluster config will be used") + flag.StringVar(&opt.versionConstraint, "kubernetes-version-constraints", ">=0.0.0", "") // OSM specific flags flag.BoolVar(&opt.useOSM, "use-osm", false, "osm controller is enabled for node bootstrap") @@ -89,6 +93,11 @@ func main() { klog.Fatalf("failed to build client: %v", err) } + constraint, err := semver.NewConstraint(opt.versionConstraint) + if err != nil { + klog.Fatalf("failed to validate kubernetes-version-constraints: %v", err) + } + // Start with assuming that current cluster will be used as worker cluster workerClient := client // Handing for worker client @@ -112,7 +121,16 @@ func main() { klog.Fatalf("error initialising userdata plugins: %v", err) } - srv, err := admission.New(opt.admissionListenAddress, client, workerClient, um, nodeFlags, opt.useOSM, opt.namespace) + srv, err := admission.Builder{ + ListenAddress: opt.admissionListenAddress, + Client: client, + WorkerClient: workerClient, + UserdataManager: um, + NodeFlags: nodeFlags, + UseOSM: opt.useOSM, + Namespace: opt.namespace, + VersionConstraints: constraint, + }.Build() if err != nil { klog.Fatalf("failed to create admission hook: %v", err) } diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go index bc355dfd9..c4799de7f 100644 --- a/pkg/admission/admission.go +++ b/pkg/admission/admission.go @@ -26,6 +26,7 @@ import ( "reflect" "time" + "github.com/Masterminds/semver/v3" "gomodules.xyz/jsonpatch/v2" machinecontroller "github.com/kubermatic/machine-controller/pkg/controller/machine" @@ -47,29 +48,34 @@ type admissionData struct { nodeSettings machinecontroller.NodeSettings useOSM bool namespace string + constraints *semver.Constraints } var jsonPatch = admissionv1.PatchTypeJSONPatch -func New( - listenAddress string, - client ctrlruntimeclient.Client, - workerClient ctrlruntimeclient.Client, - um *userdatamanager.Manager, - nodeFlags *node.Flags, - useOSM bool, - namespace string, -) (*http.Server, error) { +type Builder struct { + ListenAddress string + Client ctrlruntimeclient.Client + WorkerClient ctrlruntimeclient.Client + UserdataManager *userdatamanager.Manager + NodeFlags *node.Flags + UseOSM bool + Namespace string + VersionConstraints *semver.Constraints +} + +func (build Builder) Build() (*http.Server, error) { mux := http.NewServeMux() ad := &admissionData{ - client: client, - workerClient: workerClient, - userDataManager: um, - useOSM: useOSM, - namespace: namespace, + client: build.Client, + workerClient: build.WorkerClient, + userDataManager: build.UserdataManager, + useOSM: build.UseOSM, + namespace: build.Namespace, + constraints: build.VersionConstraints, } - if err := nodeFlags.UpdateNodeSettings(&ad.nodeSettings); err != nil { + if err := build.NodeFlags.UpdateNodeSettings(&ad.nodeSettings); err != nil { return nil, fmt.Errorf("error updating nodeSettings, %w", err) } @@ -78,7 +84,7 @@ func New( mux.HandleFunc("/healthz", healthZHandler) return &http.Server{ - Addr: listenAddress, + Addr: build.ListenAddress, Handler: http.TimeoutHandler(mux, 25*time.Second, "timeout"), }, nil } diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index eeefd66ff..2174fcb47 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" + "github.com/Masterminds/semver/v3" "golang.org/x/crypto/ssh" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" @@ -129,6 +130,15 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec return fmt.Errorf("Kubelet version must be set") } + kubeletVer, err := semver.NewVersion(spec.Versions.Kubelet) + if err != nil { + return fmt.Errorf("failed to parse kubelet version: %w", err) + } + + if !ad.constraints.Check(kubeletVer) { + return fmt.Errorf("kubernetes version constraint didn't allow %q kubelet version", kubeletVer) + } + // Validate SSH keys if err := validatePublicKeys(providerConfig.SSHPublicKeys); err != nil { return fmt.Errorf("Invalid public keys specified: %v", err) From f09a7c70f6ee9fb57fb1cdbf7e2fb95e801eefc8 Mon Sep 17 00:00:00 2001 From: Pratik Deoghare Date: Fri, 25 Mar 2022 11:54:00 +0100 Subject: [PATCH 100/489] assign ipv6 address to aws (#1208) * assign ipv6 address to aws * fix nil pointer * reorg imports * move ContainsCIDR to utils package * generalize ContainsCIDRs * org imports --- pkg/cloudprovider/provider/aws/provider.go | 5 ++++ .../provider/kubevirt/types/types.go | 1 + pkg/cloudprovider/util/net.go | 29 +++++++++++++++++++ pkg/node/poddeletion/pod_deletion.go | 1 + test/e2e/provisioning/migrateuidscenario.go | 2 +- 5 files changed, 37 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 91f46c724..e836e1879 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -41,6 +41,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" awstypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/aws/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" "github.com/kubermatic/machine-controller/pkg/userdata/convert" @@ -796,6 +797,10 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert }, } + if util.ContainsCIDR(networkConfig.PodCIDRs, util.IPv6) { + instanceRequest.NetworkInterfaces[0].Ipv6AddressCount = aws.Int64(1) + } + runOut, err := ec2Client.RunInstances(instanceRequest) if err != nil { return nil, awsErrorToTerminalError(err, "failed create instance at aws") diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index be3413965..e574eb329 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -19,6 +19,7 @@ package types import ( "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + corev1 "k8s.io/api/core/v1" ) diff --git a/pkg/cloudprovider/util/net.go b/pkg/cloudprovider/util/net.go index a179aa94c..bcf71608c 100644 --- a/pkg/cloudprovider/util/net.go +++ b/pkg/cloudprovider/util/net.go @@ -20,6 +20,8 @@ import ( "errors" "fmt" "net" + + net2 "k8s.io/utils/net" ) func CIDRToIPAndNetMask(ipv4 string) (string, string, int, error) { @@ -36,3 +38,30 @@ func CIDRToIPAndNetMask(ipv4 string) (string, string, int, error) { netmask := fmt.Sprintf("%d.%d.%d.%d", ipNet.Mask[0], ipNet.Mask[1], ipNet.Mask[2], ipNet.Mask[3]) return ip.String(), netmask, size, nil } + +type IPVersion int + +const ( + IPv4 = iota + IPv6 +) + +// ContainsCIDR checks if cidrs contains a CIDR block of given version (IPv4 or IPv6). +func ContainsCIDR(cidrs []string, version IPVersion) bool { + f := func(string) bool { return false } + + switch version { + case IPv4: + f = net2.IsIPv4CIDRString + case IPv6: + f = net2.IsIPv6CIDRString + } + + for _, cidr := range cidrs { + if f(cidr) { + return true + } + } + + return false +} diff --git a/pkg/node/poddeletion/pod_deletion.go b/pkg/node/poddeletion/pod_deletion.go index 1b9874aa8..9781b8864 100644 --- a/pkg/node/poddeletion/pod_deletion.go +++ b/pkg/node/poddeletion/pod_deletion.go @@ -22,6 +22,7 @@ import ( "sync" "github.com/kubermatic/machine-controller/pkg/node/nodemanager" + corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/e2e/provisioning/migrateuidscenario.go b/test/e2e/provisioning/migrateuidscenario.go index 7cf6745c7..7a98c2de0 100644 --- a/test/e2e/provisioning/migrateuidscenario.go +++ b/test/e2e/provisioning/migrateuidscenario.go @@ -96,7 +96,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time } return fmt.Errorf("failed to get machine %s before creating it: %v", machine.Name, err) } - _, err := prov.Create(machine, providerData, "#cloud-config\n", nil) + _, err := prov.Create(machine, providerData, "#cloud-config\n", new(cloudprovidertypes.NetworkConfig)) if err != nil { if i < maxTries-1 { time.Sleep(10 * time.Second) From 1c742006ead08ca3fa7af0ce27ff04bfb48ecc48 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Mon, 28 Mar 2022 11:05:47 +0300 Subject: [PATCH 101/489] Disable firewalld on supported OSes (#1226) * Disable firewalld on supported OSes Signed-off-by: Artiom Diomin * Update fixtures Signed-off-by: Artiom Diomin --- pkg/userdata/amzn2/provider.go | 1 + .../amzn2/testdata/containerd-kubelet-v1.20-aws.yaml | 1 + pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml | 1 + .../amzn2/testdata/kubelet-v1.21-aws-external.yaml | 1 + pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml | 1 + .../amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml | 1 + .../amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml | 1 + .../amzn2/testdata/kubelet-v1.21-vsphere.yaml | 1 + pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml | 1 + pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml | 1 + pkg/userdata/centos/provider.go | 1 + .../centos/testdata/kubelet-containerd-v1.20-aws.yaml | 1 + pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml | 1 + .../centos/testdata/kubelet-v1.21-aws-external.yaml | 1 + pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml | 1 + .../centos/testdata/kubelet-v1.21-nutanix.yaml | 1 + .../testdata/kubelet-v1.21-vsphere-mirrors.yaml | 1 + .../centos/testdata/kubelet-v1.21-vsphere-proxy.yaml | 1 + .../centos/testdata/kubelet-v1.21-vsphere.yaml | 1 + pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml | 1 + pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml | 1 + pkg/userdata/rhel/provider.go | 11 +---------- .../rhel/testdata/kubelet-containerd-v1.20-aws.yaml | 1 + pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml | 1 + pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml | 1 + pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml | 1 + pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml | 1 + .../rhel/testdata/kubelet-v1.23-aws-external.yaml | 1 + pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml | 1 + .../rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml | 1 + .../rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml | 1 + pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml | 1 + pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml | 11 +---------- 33 files changed, 33 insertions(+), 20 deletions(-) diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index 5e0685022..7a2738c20 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -229,6 +229,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true {{ if eq .CloudProviderName "vsphere" }} systemctl enable --now vmtoolsd.service {{ end -}} diff --git a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml index e50b4ca8d..f117fab45 100644 --- a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml @@ -172,6 +172,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml index b4bfa425b..c64ffeb01 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml @@ -169,6 +169,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml index 352964c2a..33612520b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml @@ -169,6 +169,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml index 9ac68c8ec..771ab850a 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml @@ -169,6 +169,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 3f7a6b4a8..ec3dada07 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -182,6 +182,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now vmtoolsd.service systemctl enable --now kubelet diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml index abfca5acd..6a88add1a 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -182,6 +182,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now vmtoolsd.service systemctl enable --now kubelet diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml index 9dcff9302..3160341c3 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml @@ -174,6 +174,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now vmtoolsd.service systemctl enable --now kubelet diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml index 16f27df60..c0a527620 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml @@ -169,6 +169,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml index ec84dcd6a..ce9f0bbfd 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml @@ -169,6 +169,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index eac197eb5..5a9da4033 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -243,6 +243,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true {{ if eq .CloudProviderName "vsphere" }} systemctl enable --now vmtoolsd.service {{ end -}} diff --git a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml index 5a27df9f1..77b9778eb 100644 --- a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml @@ -177,6 +177,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml index 314171060..e313aaec6 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml @@ -178,6 +178,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml index c8917f2ab..c61119733 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml @@ -178,6 +178,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml index 761198b5d..0ea0405b6 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml @@ -178,6 +178,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml index f3dea3c07..42c8a2281 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml @@ -185,6 +185,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 61fc1d084..38cb4db42 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -191,6 +191,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now vmtoolsd.service systemctl enable --now kubelet diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml index 9316a9672..2be22f741 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -191,6 +191,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now vmtoolsd.service systemctl enable --now kubelet diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml index 6bf322122..e706071af 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml @@ -183,6 +183,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now vmtoolsd.service systemctl enable --now kubelet diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index c81fb114d..63e68d4c3 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -178,6 +178,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index b7f888022..0669410fb 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -178,6 +178,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 13bf1bee0..140c065b1 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -240,16 +240,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh - {{ if eq .CloudProviderName "azure" }} - {{- range $idx, $podCIDR := .PodCIDRs }} - firewall-cmd --permanent --zone=trusted --add-source={{ $podCIDR}} - {{ end }} - firewall-cmd --permanent --add-port=8472/udp - firewall-cmd --permanent --add-port={{ .NodePortRange }}/tcp - firewall-cmd --permanent --add-port={{ .NodePortRange }}/udp - firewall-cmd --reload - systemctl restart firewalld - {{ end -}} + systemctl disable --now firewalld || true {{ if eq .CloudProviderName "vsphere" }} systemctl enable --now vmtoolsd.service {{ end -}} diff --git a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml index 91c579b9b..6a89d2961 100644 --- a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml @@ -173,6 +173,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml index f8633301e..a83e35482 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml @@ -174,6 +174,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml index df9b1aa54..6d90d2366 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml @@ -174,6 +174,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index 5129bf6e1..027713804 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -174,6 +174,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml index acd97d9de..ff8bfc891 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml @@ -182,6 +182,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index 692053c9e..b563dfc60 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -174,6 +174,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index caf615754..6280f54b3 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -174,6 +174,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 23869b747..0da895b52 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -188,6 +188,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now vmtoolsd.service systemctl enable --now kubelet diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index 033e25220..1d52918bf 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -188,6 +188,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now vmtoolsd.service systemctl enable --now kubelet diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index 1e6e2479d..ea267147a 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -180,6 +180,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh + systemctl disable --now firewalld || true systemctl enable --now vmtoolsd.service systemctl enable --now kubelet diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index d1023e0af..4800ed5b0 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -181,16 +181,7 @@ write_files: mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh - - firewall-cmd --permanent --zone=trusted --add-source=172.25.0.0/16 - - firewall-cmd --permanent --zone=trusted --add-source=fd00::/104 - - firewall-cmd --permanent --add-port=8472/udp - firewall-cmd --permanent --add-port=/tcp - firewall-cmd --permanent --add-port=/udp - firewall-cmd --reload - systemctl restart firewalld + systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service From 471ed4a569e4b74873ee29c6c33912f8a1f8e2c9 Mon Sep 17 00:00:00 2001 From: Lukasz Zajaczkowski Date: Mon, 28 Mar 2022 10:48:33 +0200 Subject: [PATCH 102/489] Add tag support for vSphere provider (#1213) * Add tag support for vSphere provider * delete the tag when can't be attached to VM --- pkg/cloudprovider/provider/vsphere/client.go | 52 +++++++++++++++--- pkg/cloudprovider/provider/vsphere/helper.go | 55 +++++++++++++++++++ .../provider/vsphere/provider.go | 40 ++++++++++++++ .../provider/vsphere/types/types.go | 8 +++ 4 files changed, 147 insertions(+), 8 deletions(-) diff --git a/pkg/cloudprovider/provider/vsphere/client.go b/pkg/cloudprovider/provider/vsphere/client.go index b95178d83..b5e30afd0 100644 --- a/pkg/cloudprovider/provider/vsphere/client.go +++ b/pkg/cloudprovider/provider/vsphere/client.go @@ -25,6 +25,7 @@ import ( "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/session" + "github.com/vmware/govmomi/vapi/rest" "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/soap" @@ -39,19 +40,16 @@ type Session struct { Datacenter *object.Datacenter } +type RESTSession struct { + Client *rest.Client +} + // NewSession creates a vCenter client with initialized finder. func NewSession(ctx context.Context, config *Config) (*Session, error) { - clientURL, err := url.Parse(fmt.Sprintf("%s/sdk", config.VSphereURL)) + vim25Client, err := createVim25Client(ctx, config) if err != nil { return nil, err } - - // creating the govmoni Client in roundabout way because we need to set the proper CA bundle: reference https://github.com/vmware/govmomi/issues/1200 - soapClient := soap.NewClient(clientURL, config.AllowInsecure) - // set our CA bundle - soapClient.DefaultTransport().TLSClientConfig.RootCAs = util.CABundle - - vim25Client, err := vim25.NewClient(ctx, soapClient) if err != nil { return nil, err } @@ -85,3 +83,41 @@ func (s *Session) Logout() { utilruntime.HandleError(fmt.Errorf("vsphere client failed to logout: %s", err)) } } + +func NewRESTSession(ctx context.Context, config *Config) (*RESTSession, error) { + vim25Client, err := createVim25Client(ctx, config) + if err != nil { + return nil, err + } + + client := rest.NewClient(vim25Client) + + if err = client.Login(ctx, url.UserPassword(config.Username, config.Password)); err != nil { + return nil, fmt.Errorf("failed to login: %w", err) + } + + return &RESTSession{ + Client: client, + }, nil +} + +// Logout closes the idling vCenter connections +func (s *RESTSession) Logout(ctx context.Context) { + if err := s.Client.Logout(ctx); err != nil { + utilruntime.HandleError(fmt.Errorf("vsphere REST client failed to logout: %s", err)) + } +} + +func createVim25Client(ctx context.Context, config *Config) (*vim25.Client, error) { + clientURL, err := url.Parse(fmt.Sprintf("%s/sdk", config.VSphereURL)) + if err != nil { + return nil, err + } + + // creating the govmoni Client in roundabout way because we need to set the proper CA bundle: reference https://github.com/vmware/govmomi/issues/1200 + soapClient := soap.NewClient(clientURL, config.AllowInsecure) + // set our CA bundle + soapClient.DefaultTransport().TLSClientConfig.RootCAs = util.CABundle + + return vim25.NewClient(ctx, soapClient) +} diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go index f378bb5be..d69297451 100644 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ b/pkg/cloudprovider/provider/vsphere/helper.go @@ -29,6 +29,7 @@ import ( "text/template" "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vapi/tags" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/soap" "github.com/vmware/govmomi/vim25/types" @@ -455,3 +456,57 @@ func resolveResourcePoolRef(ctx context.Context, config *Config, session *Sessio } return nil, nil } + +func createAndAttachTags(ctx context.Context, config *Config, vm *object.VirtualMachine) error { + restAPISession, err := NewRESTSession(ctx, config) + if err != nil { + return fmt.Errorf("failed to create REST API session: %v", err) + } + defer restAPISession.Logout(ctx) + tagManager := tags.NewManager(restAPISession.Client) + klog.V(3).Info("Creating tags") + for _, tag := range config.Tags { + tagID, err := tagManager.CreateTag(ctx, &tag) + if err != nil { + return fmt.Errorf("failed to create tag: %v %v", tag, err) + } + + if err := tagManager.AttachTag(ctx, tagID, vm.Reference()); err != nil { + // If attaching the tag to VM failed then delete this tag. It prevents orphan tags. + if errDelete := tagManager.DeleteTag(ctx, &tags.Tag{ + ID: tagID, + Description: tag.Description, + Name: tag.Name, + CategoryID: tag.CategoryID, + }); errDelete != nil { + return fmt.Errorf("failed to attach tag to VM and delete the orphan tag: %v, attach error: %v, delete error: %v", tag, err, errDelete) + } + klog.V(3).Infof("Failed to attach tag %v. The tag was successfully deleted", tag) + return fmt.Errorf("failed to attach tag to VM: %v %v", tag, err) + } + } + return nil +} + +func deleteTags(ctx context.Context, config *Config, vm *object.VirtualMachine) error { + restAPISession, err := NewRESTSession(ctx, config) + if err != nil { + return fmt.Errorf("failed to create REST API session: %v", err) + } + defer restAPISession.Logout(ctx) + tagManager := tags.NewManager(restAPISession.Client) + + tags, err := tagManager.GetAttachedTags(ctx, vm.Reference()) + if err != nil { + return fmt.Errorf("failed to get attached tags for the VM: %s, %v", vm.Name(), err) + } + klog.V(3).Info("Deleting tags") + for _, tag := range tags { + err := tagManager.DeleteTag(ctx, &tag) + if err != nil { + return fmt.Errorf("failed to delete tag: %v %v", tag, err) + } + } + + return nil +} diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index d6c1d73f6..adb25a50d 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -27,6 +27,7 @@ import ( "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/property" + "github.com/vmware/govmomi/vapi/tags" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" @@ -71,6 +72,7 @@ type Config struct { CPUs int32 MemoryMB int64 DiskSizeGB *int64 + Tags []tags.Tag } // Ensures that Server implements Instance interface. @@ -186,6 +188,14 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c.MemoryMB = rawConfig.MemoryMB c.DiskSizeGB = rawConfig.DiskSizeGB + for _, tag := range rawConfig.Tags { + c.Tags = append(c.Tags, tags.Tag{ + Description: tag.Description, + Name: tag.Name, + CategoryID: tag.CategoryID, + }) + } + return &c, pconfig, rawConfig, nil } @@ -203,6 +213,28 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { } defer session.Logout() + if config.Tags != nil { + restAPISession, err := NewRESTSession(ctx, config) + if err != nil { + return fmt.Errorf("failed to create REST API session: %v", err) + } + defer restAPISession.Logout(ctx) + tagManager := tags.NewManager(restAPISession.Client) + klog.V(3).Info("Found tags") + for _, tag := range config.Tags { + if tag.Name == "" { + return fmt.Errorf("one of the tags name is empty") + } + if tag.CategoryID == "" { + return fmt.Errorf("one of the tags category is empty") + } + if _, err := tagManager.GetCategory(ctx, tag.CategoryID); err != nil { + return fmt.Errorf("can't get the category with ID %s, %w", tag.CategoryID, err) + } + } + klog.V(3).Info("Tag validation passed") + } + // Only and only one between datastore and datastre cluster should be // present, otherwise an error is raised. if config.DatastoreCluster != "" && config.Datastore == "" { @@ -304,6 +336,10 @@ func (p *provider) create(machine *clusterv1alpha1.Machine, userdata string) (in return nil, machineInvalidConfigurationTerminalError(fmt.Errorf("failed to create cloned vm: '%v'", err)) } + if err := createAndAttachTags(ctx, config, virtualMachine); err != nil { + return nil, fmt.Errorf("failed create and attach tags: %v", err) + } + if pc.OperatingSystem != providerconfigtypes.OperatingSystemFlatcar { localUserdataIsoFilePath, err := generateLocalUserdataISO(userdata, machine.Spec.Name) if err != nil { @@ -365,6 +401,10 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider return false, fmt.Errorf("failed to get instance from vSphere: %v", err) } + if err := deleteTags(ctx, config, virtualMachine); err != nil { + return false, fmt.Errorf("failed to delete tags: %v", err) + } + powerState, err := virtualMachine.PowerState(ctx) if err != nil { return false, fmt.Errorf("failed to get virtual machine power state: %v", err) diff --git a/pkg/cloudprovider/provider/vsphere/types/types.go b/pkg/cloudprovider/provider/vsphere/types/types.go index 29d54b6c8..2c1760de8 100644 --- a/pkg/cloudprovider/provider/vsphere/types/types.go +++ b/pkg/cloudprovider/provider/vsphere/types/types.go @@ -43,9 +43,17 @@ type RawConfig struct { CPUs int32 `json:"cpus"` MemoryMB int64 `json:"memoryMB"` DiskSizeGB *int64 `json:"diskSizeGB,omitempty"` + Tags []Tag `json:"tags,omitempty"` AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` } +// Tag represents vsphere tag +type Tag struct { + Description string `json:"description,omitempty"` + Name string `json:"name"` + CategoryID string `json:"categoryID"` +} + func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { rawConfig := &RawConfig{} From f4c562f10eb9b21087c143f19ec653e5ff8ac611 Mon Sep 17 00:00:00 2001 From: Helene Durand <90329641+hdurand0710@users.noreply.github.com> Date: Mon, 28 Mar 2022 17:09:04 +0200 Subject: [PATCH 103/489] Kubevirt VM affinity (#1224) * Kubevirt VM affinity Signed-off-by: Helene Durand * gimpsed the import Signed-off-by: Helene Durand --- examples/kubevirt-machinedeployment.yaml | 8 + .../provider/kubevirt/provider.go | 235 ++++++++++++++++-- .../provider/kubevirt/types/types.go | 15 ++ .../testdata/machinedeployment-kubevirt.yaml | 8 + 4 files changed, 243 insertions(+), 23 deletions(-) diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index 20c423ca9..49bb904cd 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -37,6 +37,14 @@ spec: osImage: http://10.109.79.210/<< OS_NAME >>.img size: "10Gi" storageClassName: kubermatic-fast + affinity: + podAffinityPreset: "" # Allowed values: "", "soft", "hard" + podAntiAffinityPreset: "" # Allowed values: "", "soft", "hard" + nodeAffinityPreset: + type: "" # Allowed values: "", "soft", "hard" + key: "foo" + values: + - bar # Can also be `centos`, must align with he configured registryImage above operatingSystem: "ubuntu" operatingSystemSpec: diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index af171fbe5..8ba1ab9d0 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -58,6 +58,14 @@ func init() { } } +const ( + // topologyKeyHostname defines the topology key for the node hostname. + topologyKeyHostname = "kubernetes.io/hostname" + // machineDeploymentLabelKey defines the label key used to contains as value the MachineDeployment name + // which machine comes from. + machineDeploymentLabelKey = "md" +) + var supportedOS = map[providerconfigtypes.OperatingSystem]*struct{}{ providerconfigtypes.OperatingSystemCentOS: nil, providerconfigtypes.OperatingSystemUbuntu: nil, @@ -75,18 +83,57 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes } type Config struct { - Kubeconfig string - RestConfig *rest.Config - DNSConfig *corev1.PodDNSConfig - DNSPolicy corev1.DNSPolicy - CPUs string - Memory string - Namespace string - OsImage OSImage - StorageClassName string - PVCSize resource.Quantity - FlavorName string - SecondaryDisks []SecondaryDisks + Kubeconfig string + RestConfig *rest.Config + DNSConfig *corev1.PodDNSConfig + DNSPolicy corev1.DNSPolicy + CPUs string + Memory string + Namespace string + OsImage OSImage + StorageClassName string + PVCSize resource.Quantity + FlavorName string + SecondaryDisks []SecondaryDisks + PodAffinityPreset AffinityType + PodAntiAffinityPreset AffinityType + NodeAffinityPreset NodeAffinityPreset +} + +type AffinityType string + +const ( + // Facade for podAffinity, podAntiAffinity, nodeAffinity, nodeAntiAffinity + // HardAffinityType: affinity will include requiredDuringSchedulingIgnoredDuringExecution + hardAffinityType = "hard" + // SoftAffinityType: affinity will include preferredDuringSchedulingIgnoredDuringExecution + softAffinityType = "soft" + // NoAffinityType: affinity section will not be preset + noAffinityType = "" +) + +func (p *provider) affinityType(affinityType providerconfigtypes.ConfigVarString) (AffinityType, error) { + podAffinityPresetString, err := p.configVarResolver.GetConfigVarStringValue(affinityType) + if err != nil { + return "", fmt.Errorf(`failed to parse "podAffinityPreset" field: %v`, err) + } + switch strings.ToLower(podAffinityPresetString) { + case string(hardAffinityType): + return hardAffinityType, nil + case string(softAffinityType): + return softAffinityType, nil + case string(noAffinityType): + return noAffinityType, nil + } + + return "", fmt.Errorf("unknown affinityType: %s", affinityType) +} + +// NodeAffinityPreset +type NodeAffinityPreset struct { + Type AffinityType + Key string + Values []string } type SecondaryDisks struct { @@ -205,22 +252,67 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if rawConfig.VirtualMachine.DNSConfig != nil { config.DNSConfig = rawConfig.VirtualMachine.DNSConfig } - if len(rawConfig.VirtualMachine.Template.SecondaryDisks) > 0 { - for _, sd := range rawConfig.VirtualMachine.Template.SecondaryDisks { - pvc, err := resource.ParseQuantity(sd.Size.Value) - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.size" field: %v`, err) - } - config.SecondaryDisks = append(config.SecondaryDisks, SecondaryDisks{ - Size: pvc, - StorageClassName: sd.StorageClassName.Value, - }) + config.SecondaryDisks = make([]SecondaryDisks, len(rawConfig.VirtualMachine.Template.SecondaryDisks)) + for _, sd := range rawConfig.VirtualMachine.Template.SecondaryDisks { + + sdSizeString, err := p.configVarResolver.GetConfigVarStringValue(sd.Size) + if err != nil { + return nil, nil, fmt.Errorf(`failed to parse "secondaryDisks.size" field: %v`, err) + } + pvc, err := resource.ParseQuantity(sdSizeString) + if err != nil { + return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.size" field: %v`, err) + } + + scString, err := p.configVarResolver.GetConfigVarStringValue(sd.StorageClassName) + if err != nil { + return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.storageClass" field: %v`, err) } + config.SecondaryDisks = append(config.SecondaryDisks, SecondaryDisks{ + Size: pvc, + StorageClassName: scString, + }) + } + + // Affinity/AntiAffinity + config.PodAffinityPreset, err = p.affinityType(rawConfig.Affinity.PodAffinityPreset) + if err != nil { + return nil, nil, fmt.Errorf(`failed to parse "podAffinityPreset" field: %v`, err) + } + config.PodAntiAffinityPreset, err = p.affinityType(rawConfig.Affinity.PodAntiAffinityPreset) + if err != nil { + return nil, nil, fmt.Errorf(`failed to parse "podAntiAffinityPreset" field: %v`, err) + } + config.NodeAffinityPreset, err = p.parseNodeAffinityPreset(rawConfig.Affinity.NodeAffinityPreset) + if err != nil { + return nil, nil, fmt.Errorf(`failed to parse "nodeAffinityPreset" field: %v`, err) } return &config, pconfig, nil } +func (p *provider) parseNodeAffinityPreset(nodeAffinityPreset kubevirttypes.NodeAffinityPreset) (NodeAffinityPreset, error) { + nodeAffinity := NodeAffinityPreset{} + var err error + nodeAffinity.Type, err = p.affinityType(nodeAffinityPreset.Type) + if err != nil { + return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.type" field: %v`, err) + } + nodeAffinity.Key, err = p.configVarResolver.GetConfigVarStringValue(nodeAffinityPreset.Key) + if err != nil { + return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.key" field: %v`, err) + } + nodeAffinity.Values = make([]string, len(nodeAffinityPreset.Values)) + for _, v := range nodeAffinityPreset.Values { + valueString, err := p.configVarResolver.GetConfigVarStringValue(v) + if err != nil { + return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.value" field: %v`, err) + } + nodeAffinity.Values = append(nodeAffinity.Values, valueString) + } + return nodeAffinity, nil +} + // getNamespace returns the namespace where the VM is created. // VM is created in a dedicated namespace // which is the namespace where the machine-controller pod is running. @@ -377,7 +469,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert labels := map[string]string{"kubevirt.io/vm": machine.Name} // Add a common label to all VirtualMachines spawned by the same MachineDeployment (= MachineDeployment name) if mdName, err := controllerutil.GetMachineDeploymentNameForMachine(context.Background(), machine, data.Client); err == nil { - labels["md"] = mdName + labels[machineDeploymentLabelKey] = mdName } sigClient, err := client.New(c.RestConfig, client.Options{}) @@ -441,6 +533,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert }, Resources: resourceRequirements, }, + Affinity: getAffinity(c, machineDeploymentLabelKey, labels[machineDeploymentLabelKey]), TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, Volumes: getVMVolumes(c, dataVolumeName, userDataSecretName), DNSPolicy: c.DNSPolicy, @@ -633,3 +726,99 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. } return dataVolumeTemplates } + +func getAffinity(config *Config, matchKey, matchValue string) *corev1.Affinity { + affinity := &corev1.Affinity{} + + // PodAffinity + switch config.PodAffinityPreset { + case softAffinityType: + affinity.PodAffinity = &corev1.PodAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: hostnameWeightedAffinityTerm(matchKey, matchValue), + } + case hardAffinityType: + affinity.PodAffinity = &corev1.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: hostnameAffinityTerm(matchKey, matchValue), + } + } + + // PodAntiAffinity + switch config.PodAntiAffinityPreset { + case softAffinityType: + affinity.PodAntiAffinity = &corev1.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: hostnameWeightedAffinityTerm(matchKey, matchValue), + } + case hardAffinityType: + affinity.PodAntiAffinity = &corev1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: hostnameAffinityTerm(matchKey, matchValue), + } + } + + // NodeAffinity + switch config.NodeAffinityPreset.Type { + case softAffinityType: + affinity.NodeAffinity = &corev1.NodeAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []corev1.PreferredSchedulingTerm{ + { + Weight: 1, + Preference: corev1.NodeSelectorTerm{ + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: config.NodeAffinityPreset.Key, + Values: config.NodeAffinityPreset.Values, + Operator: corev1.NodeSelectorOperator(metav1.LabelSelectorOpIn), + }, + }, + }, + }, + }, + } + case hardAffinityType: + affinity.NodeAffinity = &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: config.NodeAffinityPreset.Key, + Values: config.NodeAffinityPreset.Values, + Operator: corev1.NodeSelectorOperator(metav1.LabelSelectorOpIn), + }, + }, + }, + }, + }, + } + } + + return affinity +} + +func hostnameWeightedAffinityTerm(matchKey, matchValue string) []corev1.WeightedPodAffinityTerm { + return []corev1.WeightedPodAffinityTerm{ + { + Weight: 1, + PodAffinityTerm: corev1.PodAffinityTerm{ + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + matchKey: matchValue, + }, + }, + TopologyKey: topologyKeyHostname, + }, + }, + } +} + +func hostnameAffinityTerm(matchKey, matchValue string) []corev1.PodAffinityTerm { + return []corev1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + matchKey: matchValue, + }, + }, + TopologyKey: topologyKeyHostname, + }, + } +} diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index e574eb329..bd03203d1 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -26,6 +26,7 @@ import ( type RawConfig struct { Auth Auth `json:"auth,omitempty"` VirtualMachine VirtualMachine `json:"virtualMachine,omitempty"` + Affinity Affinity `json:"affinity,omitempty"` } // Auth @@ -72,6 +73,20 @@ type Disk struct { StorageClassName providerconfigtypes.ConfigVarString `json:"storageClassName,omitempty"` } +// Affinity +type Affinity struct { + PodAffinityPreset providerconfigtypes.ConfigVarString `json:"podAffinityPreset,omitempty"` + PodAntiAffinityPreset providerconfigtypes.ConfigVarString `json:"podAntiAffinityPreset,omitempty"` + NodeAffinityPreset NodeAffinityPreset `json:"nodeAffinityPreset,omitempty"` +} + +// NodeAffinityPreset +type NodeAffinityPreset struct { + Type providerconfigtypes.ConfigVarString `json:"type,omitempty"` + Key providerconfigtypes.ConfigVarString `json:"key,omitempty"` + Values []providerconfigtypes.ConfigVarString `json:"values,omitempty"` +} + func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { rawConfig := &RawConfig{} diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index fe4f38f86..23bb843f3 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -41,6 +41,14 @@ spec: dnsConfig: nameservers: - 8.8.8.8 + affinity: + podAffinityPreset: "" # Allowed values: "", "soft", "hard" + podAntiAffinityPreset: "" # Allowed values: "", "soft", "hard" + nodeAffinityPreset: + type: "" # Allowed values: "", "soft", "hard" + key: "foo" + values: + - bar operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false From b20fbea363d63baab193c53d85487797c6e09e44 Mon Sep 17 00:00:00 2001 From: Marvin Beckers <10295525+embik@users.noreply.github.com> Date: Wed, 30 Mar 2022 11:56:50 +0200 Subject: [PATCH 104/489] Remove 'alpha' status of Nutanix implementation (#1228) Signed-off-by: Marvin Beckers --- docs/nutanix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/nutanix.md b/docs/nutanix.md index 046beb63f..b3c87432d 100644 --- a/docs/nutanix.md +++ b/docs/nutanix.md @@ -1,6 +1,6 @@ # Nutanix Prism Central -This provider implementation is currently in **alpha** stage. Currently, the only supported API version is [Prism v3](https://www.nutanix.dev/reference/prism_central/v3/). +Currently the `machine-controller` implementation of Nutanix supports the [Prism v3 API](https://www.nutanix.dev/reference/prism_central/v3/) to create `Machines`. ## Prerequisites From e9a3b4b59ff38e5998e809e8401675539b76809d Mon Sep 17 00:00:00 2001 From: Hendrik Ferber <38665716+HaveFun83@users.noreply.github.com> Date: Wed, 30 Mar 2022 12:32:53 +0200 Subject: [PATCH 105/489] Flatcar stop disabled units (#1219) * flatcar/provider.go: added stop command to service mask Signed-off-by: Hendrik Ferber * Update test data Signed-off-by: Mara Sophie Grosch * added stop command to userDataIgnitionTemplate Signed-off-by: Hendrik Ferber * pkg/userdata/flatcar/provider.go: removed unsupported 'command: stop' from userDataIgnitionTemplate Signed-off-by: Hendrik Ferber Co-authored-by: Mara Sophie Grosch --- pkg/userdata/flatcar/provider.go | 2 ++ pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml | 2 ++ pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml | 2 ++ pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml | 2 ++ pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml | 2 ++ pkg/userdata/flatcar/testdata/containerd.yaml | 2 ++ 6 files changed, 12 insertions(+) diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index 45f93a5c8..c979ece67 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -450,10 +450,12 @@ coreos: {{- end }} {{- if .FlatcarConfig.DisableUpdateEngine }} - name: update-engine.service + command: stop mask: true {{- end }} {{- if .FlatcarConfig.DisableLocksmithD }} - name: locksmithd.service + command: stop mask: true {{- end }} {{- if .HTTPProxy }} diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml index 11301d193..9021dca2d 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml @@ -24,8 +24,10 @@ coreos: DNS=8.8.8.8 - name: update-engine.service + command: stop mask: true - name: locksmithd.service + command: stop mask: true - name: download-script.service enable: true diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml index d3676d48b..4bdc80d4f 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml @@ -24,8 +24,10 @@ coreos: DNS=8.8.8.8 - name: update-engine.service + command: stop mask: true - name: locksmithd.service + command: stop mask: true - name: download-script.service enable: true diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml index e6d4c2d81..4dadc5c3a 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml @@ -24,8 +24,10 @@ coreos: DNS=8.8.8.8 - name: update-engine.service + command: stop mask: true - name: locksmithd.service + command: stop mask: true - name: download-script.service enable: true diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml index 93ef4c5c4..9198d6f4e 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml @@ -24,8 +24,10 @@ coreos: DNS=8.8.8.8 - name: update-engine.service + command: stop mask: true - name: locksmithd.service + command: stop mask: true - name: download-script.service enable: true diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index 2582169da..41b76292c 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -9,8 +9,10 @@ users: coreos: units: - name: update-engine.service + command: stop mask: true - name: locksmithd.service + command: stop mask: true - name: download-script.service enable: true From 04cb11d7ab484c668d8da5e9a1fd9f6962ed148c Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Thu, 31 Mar 2022 18:00:42 +0200 Subject: [PATCH 106/489] Revert "assign ipv6 address to aws (#1208)" (#1229) This reverts commit f09a7c70f6ee9fb57fb1cdbf7e2fb95e801eefc8. --- pkg/cloudprovider/provider/aws/provider.go | 5 ---- .../provider/kubevirt/types/types.go | 1 - pkg/cloudprovider/util/net.go | 29 ------------------- pkg/node/poddeletion/pod_deletion.go | 1 - test/e2e/provisioning/migrateuidscenario.go | 2 +- 5 files changed, 1 insertion(+), 37 deletions(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index e836e1879..91f46c724 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -41,7 +41,6 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" awstypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/aws/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" "github.com/kubermatic/machine-controller/pkg/userdata/convert" @@ -797,10 +796,6 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert }, } - if util.ContainsCIDR(networkConfig.PodCIDRs, util.IPv6) { - instanceRequest.NetworkInterfaces[0].Ipv6AddressCount = aws.Int64(1) - } - runOut, err := ec2Client.RunInstances(instanceRequest) if err != nil { return nil, awsErrorToTerminalError(err, "failed create instance at aws") diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index bd03203d1..a2fff0ba6 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -19,7 +19,6 @@ package types import ( "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - corev1 "k8s.io/api/core/v1" ) diff --git a/pkg/cloudprovider/util/net.go b/pkg/cloudprovider/util/net.go index bcf71608c..a179aa94c 100644 --- a/pkg/cloudprovider/util/net.go +++ b/pkg/cloudprovider/util/net.go @@ -20,8 +20,6 @@ import ( "errors" "fmt" "net" - - net2 "k8s.io/utils/net" ) func CIDRToIPAndNetMask(ipv4 string) (string, string, int, error) { @@ -38,30 +36,3 @@ func CIDRToIPAndNetMask(ipv4 string) (string, string, int, error) { netmask := fmt.Sprintf("%d.%d.%d.%d", ipNet.Mask[0], ipNet.Mask[1], ipNet.Mask[2], ipNet.Mask[3]) return ip.String(), netmask, size, nil } - -type IPVersion int - -const ( - IPv4 = iota - IPv6 -) - -// ContainsCIDR checks if cidrs contains a CIDR block of given version (IPv4 or IPv6). -func ContainsCIDR(cidrs []string, version IPVersion) bool { - f := func(string) bool { return false } - - switch version { - case IPv4: - f = net2.IsIPv4CIDRString - case IPv6: - f = net2.IsIPv6CIDRString - } - - for _, cidr := range cidrs { - if f(cidr) { - return true - } - } - - return false -} diff --git a/pkg/node/poddeletion/pod_deletion.go b/pkg/node/poddeletion/pod_deletion.go index 9781b8864..1b9874aa8 100644 --- a/pkg/node/poddeletion/pod_deletion.go +++ b/pkg/node/poddeletion/pod_deletion.go @@ -22,7 +22,6 @@ import ( "sync" "github.com/kubermatic/machine-controller/pkg/node/nodemanager" - corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/test/e2e/provisioning/migrateuidscenario.go b/test/e2e/provisioning/migrateuidscenario.go index 7a98c2de0..7cf6745c7 100644 --- a/test/e2e/provisioning/migrateuidscenario.go +++ b/test/e2e/provisioning/migrateuidscenario.go @@ -96,7 +96,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time } return fmt.Errorf("failed to get machine %s before creating it: %v", machine.Name, err) } - _, err := prov.Create(machine, providerData, "#cloud-config\n", new(cloudprovidertypes.NetworkConfig)) + _, err := prov.Create(machine, providerData, "#cloud-config\n", nil) if err != nil { if i < maxTries-1 { time.Sleep(10 * time.Second) From e3b6519bc4cb823e919b2315d6477c288bc34418 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Thu, 31 Mar 2022 20:08:42 +0200 Subject: [PATCH 107/489] revert network config (#1234) Signed-off-by: Moath Qasim --- pkg/cloudprovider/provider/alibaba/provider.go | 2 +- pkg/cloudprovider/provider/anexia/provider.go | 3 +-- pkg/cloudprovider/provider/aws/provider.go | 2 +- pkg/cloudprovider/provider/azure/provider.go | 2 +- pkg/cloudprovider/provider/baremetal/provider.go | 2 +- pkg/cloudprovider/provider/digitalocean/provider.go | 2 +- pkg/cloudprovider/provider/equinixmetal/provider.go | 2 +- pkg/cloudprovider/provider/fake/provider.go | 2 +- pkg/cloudprovider/provider/gce/provider.go | 2 +- pkg/cloudprovider/provider/hetzner/provider.go | 2 +- pkg/cloudprovider/provider/kubevirt/provider.go | 2 +- pkg/cloudprovider/provider/linode/provider.go | 2 +- pkg/cloudprovider/provider/nutanix/provider.go | 2 +- pkg/cloudprovider/provider/openstack/provider.go | 2 +- pkg/cloudprovider/provider/openstack/provider_test.go | 2 +- pkg/cloudprovider/provider/scaleway/provider.go | 2 +- pkg/cloudprovider/provider/vsphere/provider.go | 2 +- pkg/cloudprovider/types/types.go | 7 +------ pkg/cloudprovider/validationwrapper.go | 4 ++-- pkg/controller/machine/machine_controller.go | 10 +++------- test/e2e/provisioning/migrateuidscenario.go | 2 +- 21 files changed, 24 insertions(+), 34 deletions(-) diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index f9a71bb13..0687b8a28 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -198,7 +198,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri return "", "", nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 796a83336..019f57a92 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -59,8 +59,7 @@ type provider struct { configVarResolver *providerconfig.ConfigVarResolver } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, - userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance instance.Instance, retErr error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance instance.Instance, retErr error) { status := getProviderStatus(machine) klog.V(3).Infof(fmt.Sprintf("'%s' has status %#v", machine.Name, status)) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 91f46c724..c9e243043 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -660,7 +660,7 @@ func getVpc(client *ec2.EC2, id string) (*ec2.Vpc, error) { return vpcOut.Vpcs[0], nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 1c9bbdf76..402f54258 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -480,7 +480,7 @@ func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) return sp, nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { config, providerCfg, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/baremetal/provider.go b/pkg/cloudprovider/provider/baremetal/provider.go index 61bd0e1cc..3956a12b1 100644 --- a/pkg/cloudprovider/provider/baremetal/provider.go +++ b/pkg/cloudprovider/provider/baremetal/provider.go @@ -210,7 +210,7 @@ func (p provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config strin return "", "", nil } -func (p provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { +func (p provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index 93ab072c7..14e6ce7a3 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -263,7 +263,7 @@ func uploadRandomSSHPublicKey(ctx context.Context, service godo.KeysService) (st return newDoKey.Fingerprint, nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index 23dea42d6..6b1704eb6 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -214,7 +214,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, _, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/fake/provider.go b/pkg/cloudprovider/provider/fake/provider.go index da9f2a60d..a0655d363 100644 --- a/pkg/cloudprovider/provider/fake/provider.go +++ b/pkg/cloudprovider/provider/fake/provider.go @@ -94,7 +94,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (string, str } // Create creates a cloud instance according to the given machine -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { return CloudProviderInstance{}, nil } diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 2a51fbf5a..c8b584932 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -195,7 +195,7 @@ func (p *Provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri } // Create inserts a cloud instance according to the given machine. -func (p *Provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { +func (p *Provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { // Read configuration. cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) if err != nil { diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 61a4f2541..382da7c8b 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -245,7 +245,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 8ba1ab9d0..dfcde1fbd 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -450,7 +450,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index 1e3688f6d..f6fca7fca 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -212,7 +212,7 @@ func createRandomPassword() (string, error) { return rootPass, nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 54025577f..5ad6b0212 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -241,7 +241,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { vm, err := p.create(machine, userdata) if err != nil { _, cleanupErr := p.Cleanup(machine, data) diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 829d3f859..f86f119fb 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -553,7 +553,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { cfg, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index a0a8c2cd1..8bebddd7e 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -295,7 +295,7 @@ func TestCreateServer(t *testing.T) { // It only verifies that the content of the create request matches // the expectation // TODO(irozzo) check the returned instance too - _, err := p.Create(m, tt.data, tt.userdata, nil) + _, err := p.Create(m, tt.data, tt.userdata) if (err != nil) != tt.wantErr { t.Errorf("provider.Create() or = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go index d6f6f27b8..b506210d6 100644 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ b/pkg/cloudprovider/provider/scaleway/provider.go @@ -172,7 +172,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (cloudInstance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (cloudInstance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index adb25a50d..cb1c7cef5 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -294,7 +294,7 @@ func machineInvalidConfigurationTerminalError(err error) error { } } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { +func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { vm, err := p.create(machine, userdata) if err != nil { _, cleanupErr := p.Cleanup(machine, data) diff --git a/pkg/cloudprovider/types/types.go b/pkg/cloudprovider/types/types.go index 2e7249fb3..fdc756fe7 100644 --- a/pkg/cloudprovider/types/types.go +++ b/pkg/cloudprovider/types/types.go @@ -52,7 +52,7 @@ type Provider interface { GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) // Create creates a cloud instance according to the given machine - Create(machine *clusterv1alpha1.Machine, data *ProviderData, userdata string, networkConfig *NetworkConfig) (instance.Instance, error) + Create(machine *clusterv1alpha1.Machine, data *ProviderData, userdata string) (instance.Instance, error) // Cleanup will delete the instance associated with the machine and all associated resources. // If all resources have been cleaned up, true will be returned. @@ -76,11 +76,6 @@ type Provider interface { SetMetricsForMachines(machines clusterv1alpha1.MachineList) error } -// NetworkConfig holds information about cluster networking. -type NetworkConfig struct { - PodCIDRs []string `json:"podCIDRs"` // PodCIDRs fields is used to choose IPv4, IPv6 or dual-stack modes. -} - // MachineModifier defines a function to modify a machine type MachineModifier func(*clusterv1alpha1.Machine) diff --git a/pkg/cloudprovider/validationwrapper.go b/pkg/cloudprovider/validationwrapper.go index b4426fead..976ba129e 100644 --- a/pkg/cloudprovider/validationwrapper.go +++ b/pkg/cloudprovider/validationwrapper.go @@ -73,8 +73,8 @@ func (w *cachingValidationWrapper) GetCloudConfig(spec v1alpha1.MachineSpec) (st } // Create just calls the underlying cloudproviders Create -func (w *cachingValidationWrapper) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { - return w.actualProvider.Create(machine, data, userdata, networkConfig) +func (w *cachingValidationWrapper) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { + return w.actualProvider.Create(machine, data, userdata) } // Cleanup just calls the underlying cloudproviders Cleanup diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 2e8a50584..9094c75a2 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -341,13 +341,13 @@ func (r *Reconciler) updateMachineErrorIfTerminalError(machine *clusterv1alpha1. return fmt.Errorf("%s, due to %v", errMsg, err) } -func (r *Reconciler) createProviderInstance(prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine, userdata string, networkConfig *cloudprovidertypes.NetworkConfig) (instance.Instance, error) { +func (r *Reconciler) createProviderInstance(prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { // Ensure finalizer is there _, err := r.ensureDeleteFinalizerExists(machine) if err != nil { return nil, fmt.Errorf("failed to add %q finalizer: %v", FinalizerDeleteInstance, err) } - i, err := prov.Create(machine, r.providerData, userdata, networkConfig) + i, err := prov.Create(machine, r.providerData, userdata) if err != nil { return nil, err } @@ -845,12 +845,8 @@ func (r *Reconciler) ensureInstanceExistsForMachine( } } - networkConfig := &cloudprovidertypes.NetworkConfig{ - PodCIDRs: r.podCIDRs, - } - // Create the instance - if _, err = r.createProviderInstance(prov, machine, userdata, networkConfig); err != nil { + if _, err = r.createProviderInstance(prov, machine, userdata); err != nil { message := fmt.Sprintf("%v. Unable to create a machine.", err) return nil, r.updateMachineErrorIfTerminalError(machine, common.CreateMachineError, message, err, "failed to create machine at cloudprovider") } diff --git a/test/e2e/provisioning/migrateuidscenario.go b/test/e2e/provisioning/migrateuidscenario.go index 7cf6745c7..8f1e20f60 100644 --- a/test/e2e/provisioning/migrateuidscenario.go +++ b/test/e2e/provisioning/migrateuidscenario.go @@ -96,7 +96,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time } return fmt.Errorf("failed to get machine %s before creating it: %v", machine.Name, err) } - _, err := prov.Create(machine, providerData, "#cloud-config\n", nil) + _, err := prov.Create(machine, providerData, "#cloud-config\n") if err != nil { if i < maxTries-1 { time.Sleep(10 * time.Second) From 20798f00b07f64bfa21e970ed9010245a748c5dc Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 5 Apr 2022 12:14:57 +0200 Subject: [PATCH 108/489] disable nutanix e2e tests when packages other than nutanix cloud provider have changed (#1237) Signed-off-by: Moath Qasim --- .prow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.prow.yaml b/.prow.yaml index 5c5c58db2..89c10d5d8 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -568,7 +568,7 @@ presubmits: - name: pull-machine-controller-e2e-nutanix optional: true always_run: false - run_if_changed: "(pkg/cloudprovider/provider/nutanix/|pkg/userdata/|test/e2e/provisioning/)" + run_if_changed: "(pkg/cloudprovider/provider/nutanix/)" decorate: true error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" From 53fee0531a371a94d7276d5abdd15439018841f8 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 5 Apr 2022 20:29:10 +0200 Subject: [PATCH 109/489] Add rocky linux support for AWS (#1235) * add rocky linux as a supported os in machine controller Signed-off-by: Moath Qasim * update fixtures Signed-off-by: Moath Qasim Signed-off-by: Moath Qasim * rockylinux support in aws Signed-off-by: Moath Qasim * add rockylinux support for aws Signed-off-by: Moath Qasim * fix rocky linux defaulting test Signed-off-by: Moath Qasim * enable rocky linux for aws only tests Signed-off-by: Moath Qasim * addressing pr reviews Signed-off-by: Moath Qasim * addressing pr reviews Signed-off-by: Moath Qasim * update fixtures Signed-off-by: Moath Qasim --- Makefile | 2 +- cmd/userdata/rockylinux/main.go | 46 ++ pkg/cloudprovider/provider/aws/provider.go | 14 + pkg/containerruntime/containerd.go | 2 +- pkg/containerruntime/docker.go | 2 +- pkg/providerconfig/types.go | 3 + pkg/providerconfig/types/types.go | 2 + pkg/userdata/manager/manager.go | 1 + pkg/userdata/rockylinux/provider.go | 329 +++++++++++++ pkg/userdata/rockylinux/provider_test.go | 296 ++++++++++++ pkg/userdata/rockylinux/rockylinux.go | 59 +++ .../kubelet-containerd-v1.20-aws.yaml | 440 +++++++++++++++++ .../testdata/kubelet-v1.20-aws.yaml | 424 +++++++++++++++++ .../testdata/kubelet-v1.21-aws-external.yaml | 424 +++++++++++++++++ .../testdata/kubelet-v1.21-aws.yaml | 424 +++++++++++++++++ .../testdata/kubelet-v1.21-nutanix.yaml | 432 +++++++++++++++++ .../kubelet-v1.21-vsphere-mirrors.yaml | 441 ++++++++++++++++++ .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 441 ++++++++++++++++++ .../testdata/kubelet-v1.21-vsphere.yaml | 432 +++++++++++++++++ .../testdata/kubelet-v1.22-aws.yaml | 424 +++++++++++++++++ .../testdata/kubelet-v1.23-aws.yaml | 422 +++++++++++++++++ test/e2e/provisioning/all_e2e_test.go | 10 +- test/e2e/provisioning/helper.go | 1 + 23 files changed, 5063 insertions(+), 8 deletions(-) create mode 100644 cmd/userdata/rockylinux/main.go create mode 100644 pkg/userdata/rockylinux/provider.go create mode 100644 pkg/userdata/rockylinux/provider_test.go create mode 100644 pkg/userdata/rockylinux/rockylinux.go create mode 100644 pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml create mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml create mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml create mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml create mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml create mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml create mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml create mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml create mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml create mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml diff --git a/Makefile b/Makefile index 0fa1477c1..547f98103 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ IMAGE_TAG = \ $(shell echo $$(git rev-parse HEAD && if [[ -n $$(git status --porcelain) ]]; then echo '-dirty'; fi)|tr -d ' ') IMAGE_NAME ?= $(REGISTRY)/$(REGISTRY_NAMESPACE)/machine-controller:$(IMAGE_TAG) -OS = amzn2 centos ubuntu sles rhel flatcar +OS = amzn2 centos ubuntu sles rhel flatcar rockylinux USERDATA_BIN = $(patsubst %, machine-controller-userdata-%, $(OS)) BASE64_ENC = \ diff --git a/cmd/userdata/rockylinux/main.go b/cmd/userdata/rockylinux/main.go new file mode 100644 index 000000000..204d38eb2 --- /dev/null +++ b/cmd/userdata/rockylinux/main.go @@ -0,0 +1,46 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +// UserData plugin for RockyLinux. +// + +package main + +import ( + "flag" + + userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" + "github.com/kubermatic/machine-controller/pkg/userdata/rockylinux" + + "k8s.io/klog" +) + +func main() { + // Parse flags. + var debug bool + + flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging") + flag.Parse() + + // Instantiate provider and start plugin. + var provider = &rockylinux.Provider{} + var p = userdataplugin.New(provider, debug) + + if err := p.Run(); err != nil { + klog.Fatalf("error running RockyLinux plugin: %v", err) + } +} diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index c9e243043..b41e8d186 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -111,6 +111,18 @@ var ( owner: "125523088429", }, }, + providerconfigtypes.OperatingSystemRockyLinux: { + awstypes.CPUArchitectureX86_64: { + description: "Rocky-8-ec2-8*.x86_64", + // The AWS marketplace ID from Rocky Linux Community Platform Engineering (CPE) + owner: "792107900819", + }, + awstypes.CPUArchitectureARM64: { + description: "Rocky-8-ec2-8*.aarch64", + // The AWS marketplace ID from Rocky Linux Community Platform Engineering (CPE) + owner: "792107900819", + }, + }, providerconfigtypes.OperatingSystemAmazonLinux2: { awstypes.CPUArchitectureX86_64: { description: "Amazon Linux 2 AMI * x86_64 HVM gp2", @@ -334,6 +346,8 @@ func getDefaultRootDevicePath(os providerconfigtypes.OperatingSystem) (string, e return rootDevicePathSDA, nil case providerconfigtypes.OperatingSystemCentOS: return rootDevicePathSDA, nil + case providerconfigtypes.OperatingSystemRockyLinux: + return rootDevicePathSDA, nil case providerconfigtypes.OperatingSystemSLES: return rootDevicePathXVDA, nil case providerconfigtypes.OperatingSystemRHEL: diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index 749ccd041..05599b23a 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -61,7 +61,7 @@ func (eng *Containerd) ScriptFor(os types.OperatingSystem) (string, error) { case types.OperatingSystemAmazonLinux2: err := containerdAmzn2Template.Execute(&buf, args) return buf.String(), err - case types.OperatingSystemCentOS, types.OperatingSystemRHEL: + case types.OperatingSystemCentOS, types.OperatingSystemRHEL, types.OperatingSystemRockyLinux: err := containerdYumTemplate.Execute(&buf, args) return buf.String(), err case types.OperatingSystemUbuntu: diff --git a/pkg/containerruntime/docker.go b/pkg/containerruntime/docker.go index c51f97cf0..cb3a2c9ca 100644 --- a/pkg/containerruntime/docker.go +++ b/pkg/containerruntime/docker.go @@ -67,7 +67,7 @@ func (eng *Docker) ScriptFor(os types.OperatingSystem) (string, error) { case types.OperatingSystemAmazonLinux2: err := dockerAmazonTemplate.Execute(&buf, args) return buf.String(), err - case types.OperatingSystemCentOS, types.OperatingSystemRHEL: + case types.OperatingSystemCentOS, types.OperatingSystemRHEL, types.OperatingSystemRockyLinux: err := dockerYumTemplate.Execute(&buf, args) return buf.String(), err case types.OperatingSystemUbuntu: diff --git a/pkg/providerconfig/types.go b/pkg/providerconfig/types.go index a984e21c1..b4e168adf 100644 --- a/pkg/providerconfig/types.go +++ b/pkg/providerconfig/types.go @@ -29,6 +29,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/userdata/centos" "github.com/kubermatic/machine-controller/pkg/userdata/flatcar" "github.com/kubermatic/machine-controller/pkg/userdata/rhel" + "github.com/kubermatic/machine-controller/pkg/userdata/rockylinux" "github.com/kubermatic/machine-controller/pkg/userdata/sles" "github.com/kubermatic/machine-controller/pkg/userdata/ubuntu" @@ -198,6 +199,8 @@ func DefaultOperatingSystemSpec( return sles.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemUbuntu: return ubuntu.DefaultConfig(operatingSystemSpec), nil + case providerconfigtypes.OperatingSystemRockyLinux: + return rockylinux.DefaultConfig(operatingSystemSpec), nil } return operatingSystemSpec, errors.New("unknown OperatingSystem") diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 925d1abdb..37f75ff24 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -39,6 +39,7 @@ const ( OperatingSystemSLES OperatingSystem = "sles" OperatingSystemRHEL OperatingSystem = "rhel" OperatingSystemFlatcar OperatingSystem = "flatcar" + OperatingSystemRockyLinux OperatingSystem = "rockylinux" ) type CloudProvider string @@ -75,6 +76,7 @@ var ( OperatingSystemSLES, OperatingSystemRHEL, OperatingSystemFlatcar, + OperatingSystemRockyLinux, } // AllCloudProviders is a slice containing all supported cloud providers. diff --git a/pkg/userdata/manager/manager.go b/pkg/userdata/manager/manager.go index a924cd1c3..ffa291944 100644 --- a/pkg/userdata/manager/manager.go +++ b/pkg/userdata/manager/manager.go @@ -50,6 +50,7 @@ var ( providerconfigtypes.OperatingSystemRHEL, providerconfigtypes.OperatingSystemSLES, providerconfigtypes.OperatingSystemUbuntu, + providerconfigtypes.OperatingSystemRockyLinux, } ) diff --git a/pkg/userdata/rockylinux/provider.go b/pkg/userdata/rockylinux/provider.go new file mode 100644 index 000000000..fc3db6849 --- /dev/null +++ b/pkg/userdata/rockylinux/provider.go @@ -0,0 +1,329 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +// UserData plugin for RockyLinux. +// + +package rockylinux + +import ( + "errors" + "fmt" + "strings" + "text/template" + + "github.com/Masterminds/semver/v3" + + "github.com/kubermatic/machine-controller/pkg/apis/plugin" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + userdatahelper "github.com/kubermatic/machine-controller/pkg/userdata/helper" +) + +// Provider is a pkg/userdata/plugin.Provider implementation. +type Provider struct{} + +// UserData renders user-data template to string. +func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { + tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(userDataTemplate) + if err != nil { + return "", fmt.Errorf("failed to parse user-data template: %w", err) + } + + kubeletVersion, err := semver.NewVersion(req.MachineSpec.Versions.Kubelet) + if err != nil { + return "", fmt.Errorf("invalid kubelet version: %w", err) + } + + pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) + if err != nil { + return "", fmt.Errorf("failed to get provider config: %w", err) + } + + if pconfig.OverwriteCloudConfig != nil { + req.CloudConfig = *pconfig.OverwriteCloudConfig + } + + if pconfig.Network != nil { + return "", errors.New("static IP config is not supported with RockyLinux") + } + + rockyLinuxConfig, err := LoadConfig(pconfig.OperatingSystemSpec) + if err != nil { + return "", fmt.Errorf("failed to parse OperatingSystemSpec: %w", err) + } + + serverAddr, err := userdatahelper.GetServerAddressFromKubeconfig(req.Kubeconfig) + if err != nil { + return "", fmt.Errorf("error extracting server address from kubeconfig: %w", err) + } + + kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) + if err != nil { + return "", err + } + + kubernetesCACert, err := userdatahelper.GetCACert(req.Kubeconfig) + if err != nil { + return "", fmt.Errorf("error extracting cacert: %w", err) + } + + crEngine := req.ContainerRuntime.Engine(kubeletVersion) + crScript, err := crEngine.ScriptFor(providerconfigtypes.OperatingSystemRockyLinux) + if err != nil { + return "", fmt.Errorf("failed to generate container runtime install script: %w", err) + } + + crConfig, err := crEngine.Config() + if err != nil { + return "", fmt.Errorf("failed to generate container runtime config: %w", err) + } + + data := struct { + plugin.UserDataRequest + ProviderSpec *providerconfigtypes.Config + OSConfig *Config + KubeletVersion string + ServerAddr string + Kubeconfig string + KubernetesCACert string + NodeIPScript string + ExtraKubeletFlags []string + ContainerRuntimeScript string + ContainerRuntimeConfigFileName string + ContainerRuntimeConfig string + ContainerRuntimeName string + }{ + UserDataRequest: req, + ProviderSpec: pconfig, + OSConfig: rockyLinuxConfig, + KubeletVersion: kubeletVersion.String(), + ServerAddr: serverAddr, + Kubeconfig: kubeconfigString, + KubernetesCACert: kubernetesCACert, + NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), + ExtraKubeletFlags: crEngine.KubeletFlags(), + ContainerRuntimeScript: crScript, + ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), + ContainerRuntimeConfig: crConfig, + ContainerRuntimeName: crEngine.String(), + } + + buf := strings.Builder{} + if err = tmpl.Execute(&buf, data); err != nil { + return "", fmt.Errorf("failed to execute user-data template: %w", err) + } + + return userdatahelper.CleanupTemplateOutput(buf.String()) +} + +// UserData template. +const userDataTemplate = `#cloud-config +bootcmd: +- modprobe ip_tables +{{ if ne .CloudProviderName "aws" }} +hostname: {{ .MachineSpec.Name }} +{{- /* Never set the hostname on AWS nodes. Kubernetes(kube-proxy) requires the hostname to be the private dns name */}} +{{ end }} + +{{- if .OSConfig.DistUpgradeOnBoot }} +package_upgrade: true +package_reboot_if_required: true +{{- end }} + +ssh_pwauth: false + +{{- if ne (len .ProviderSpec.SSHPublicKeys) 0 }} +ssh_authorized_keys: +{{- range .ProviderSpec.SSHPublicKeys }} + - "{{ . }}" +{{- end }} +{{- end }} + +write_files: +{{- if .HTTPProxy }} +- path: "/etc/environment" + content: | +{{ proxyEnvironment .HTTPProxy .NoProxy | indent 4 }} +{{- end }} + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | +{{ journalDConfig | indent 4 }} + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | +{{ kernelModulesScript | indent 4 }} + +- path: "/etc/sysctl.d/k8s.conf" + content: | +{{ kernelSettings | indent 4 }} + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + +{{- /* As we added some modules and don't want to reboot, restart the service */}} + systemctl restart systemd-modules-load.service + sysctl --system + +{{- /* Make sure we always disable swap - Otherwise the kubelet won't start */}} + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + {{ if ne .CloudProviderName "aws" }} +{{- /* The normal way of setting it via cloud-init is broken, see */}} +{{- /* https://bugs.launchpad.net/cloud-init/+bug/1662542 */}} + hostnamectl set-hostname {{ .MachineSpec.Name }} + {{ end -}} + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + {{- if eq .CloudProviderName "vsphere" }} + open-vm-tools \ + {{- end }} + {{- if eq .CloudProviderName "nutanix" }} + iscsi-initiator-utils \ + {{- end }} + ipvsadm + + {{- /* iscsid service is required on Nutanix machines for CSI driver to attach volumes. */}} + {{- if eq .CloudProviderName "nutanix" }} + systemctl enable --now iscsid + {{ end }} +{{ .ContainerRuntimeScript | indent 4 }} +{{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + {{ if eq .CloudProviderName "vsphere" }} + systemctl enable --now vmtoolsd.service + {{ end -}} + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | +{{ .CloudConfig | indent 4 }} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | +{{ .NodeIPScript | indent 4 }} + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | +{{ .Kubeconfig | indent 4 }} + +- path: "/etc/kubernetes/kubelet.conf" + content: | +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }} + +- path: "/etc/kubernetes/pki/ca.crt" + content: | +{{ .KubernetesCACert | indent 4 }} + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: {{ .ContainerRuntimeConfigFileName }} + permissions: "0644" + content: | +{{ .ContainerRuntimeConfig | indent 4 }} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | +{{ kubeletHealthCheckSystemdUnit | indent 4 }} + +{{- with .ProviderSpec.CAPublicKey }} + +- path: "/etc/ssh/trusted-user-ca-keys.pem" + content: | +{{ . | indent 4 }} + +- path: "/etc/ssh/sshd_config" + content: | +{{ sshConfigAddendum | indent 4 }} + append: true +{{- end }} + +runcmd: +- systemctl start setup.service +` diff --git a/pkg/userdata/rockylinux/provider_test.go b/pkg/userdata/rockylinux/provider_test.go new file mode 100644 index 000000000..ad23a2f1b --- /dev/null +++ b/pkg/userdata/rockylinux/provider_test.go @@ -0,0 +1,296 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +// UserData plugin for RockyLinux. +// + +package rockylinux + +import ( + "flag" + "net" + "testing" + + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "github.com/kubermatic/machine-controller/pkg/apis/plugin" + "github.com/kubermatic/machine-controller/pkg/containerruntime" + testhelper "github.com/kubermatic/machine-controller/pkg/test" + "github.com/kubermatic/machine-controller/pkg/userdata/convert" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +var ( + update = flag.Bool("update", false, "update testdata files") + + pemCertificate = `-----BEGIN CERTIFICATE----- +MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV +BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG +A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 +DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 +NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG +cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv +c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS +R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT +ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk +JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 +mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW +caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G +A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt +hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB +MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES +MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv +bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h +U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao +eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 +UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD +58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n +sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF +kPe6XoSbiLm/kxk32T0= +-----END CERTIFICATE-----` +) + +// fakeCloudConfigProvider simulates cloud config provider for test. +type fakeCloudConfigProvider struct { + config string + name string + err error +} + +func (p *fakeCloudConfigProvider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { + return p.config, p.name, p.err +} + +// userDataTestCase contains the data for a table-driven test. +type userDataTestCase struct { + name string + spec clusterv1alpha1.MachineSpec + clusterDNSIPs []net.IP + cloudProviderName *string + externalCloudProvider bool + httpProxy string + noProxy string + insecureRegistries string + registryMirrors string + pauseImage string + containerruntime string +} + +// TestUserDataGeneration runs the data generation for different +// environments. +func TestUserDataGeneration(t *testing.T) { + t.Parallel() + + tests := []userDataTestCase{ + { + name: "kubelet-v1.20-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.20.14", + }, + }, + }, + { + name: "kubelet-containerd-v1.20-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.20.14", + }, + }, + containerruntime: "containerd", + }, + { + name: "kubelet-v1.21-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.21.8", + }, + }, + }, + { + name: "kubelet-v1.21-aws-external", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.21.8", + }, + }, + externalCloudProvider: true, + }, + { + name: "kubelet-v1.21-vsphere", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.21.8", + }, + }, + cloudProviderName: stringPtr("vsphere"), + }, + { + name: "kubelet-v1.21-vsphere-proxy", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.21.8", + }, + }, + cloudProviderName: stringPtr("vsphere"), + httpProxy: "http://192.168.100.100:3128/", + noProxy: "192.168.1.0", + insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", + pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", + }, + { + name: "kubelet-v1.21-vsphere-mirrors", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.21.8", + }, + }, + cloudProviderName: stringPtr("vsphere"), + httpProxy: "http://192.168.100.100:3128/", + noProxy: "192.168.1.0", + registryMirrors: "https://registry.docker-cn.com/", + pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", + }, + { + name: "kubelet-v1.22-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.22.5", + }, + }, + }, + { + name: "kubelet-v1.23-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.23.0", + }, + }, + }, + { + name: "kubelet-v1.21-nutanix", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.21.5", + }, + }, + cloudProviderName: stringPtr("nutanix"), + }, + } + + defaultCloudProvider := &fakeCloudConfigProvider{ + name: "aws", + config: "{aws-config:true}", + err: nil, + } + kubeconfig := &clientcmdapi.Config{ + Clusters: map[string]*clientcmdapi.Cluster{ + "": { + Server: "https://server/", + CertificateAuthorityData: []byte(pemCertificate), + }, + }, + AuthInfos: map[string]*clientcmdapi.AuthInfo{ + "": { + Token: "my-token", + }, + }, + } + provider := Provider{} + + kubeletFeatureGates := map[string]bool{ + "RotateKubeletServerCertificate": true, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + emtpyProviderSpec := clusterv1alpha1.ProviderSpec{ + Value: &runtime.RawExtension{}, + } + test.spec.ProviderSpec = emtpyProviderSpec + var cloudProvider *fakeCloudConfigProvider + if test.cloudProviderName != nil { + cloudProvider = &fakeCloudConfigProvider{ + name: *test.cloudProviderName, + config: "{config:true}", + err: nil, + } + } else { + cloudProvider = defaultCloudProvider + } + cloudConfig, cloudProviderName, err := cloudProvider.GetCloudConfig(test.spec) + if err != nil { + t.Fatalf("failed to get cloud config: %v", err) + } + + containerRuntimeOpts := containerruntime.Opts{ + ContainerRuntime: test.containerruntime, + InsecureRegistries: test.insecureRegistries, + RegistryMirrors: test.registryMirrors, + } + containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) + if err != nil { + t.Fatalf("failed to generate container runtime config: %v", err) + } + + req := plugin.UserDataRequest{ + MachineSpec: test.spec, + Kubeconfig: kubeconfig, + CloudConfig: cloudConfig, + CloudProviderName: cloudProviderName, + KubeletCloudProviderName: cloudProviderName, + DNSIPs: test.clusterDNSIPs, + ExternalCloudProvider: test.externalCloudProvider, + HTTPProxy: test.httpProxy, + NoProxy: test.noProxy, + PauseImage: test.pauseImage, + KubeletFeatureGates: kubeletFeatureGates, + ContainerRuntime: containerRuntimeConfig, + } + + s, err := provider.UserData(req) + if err != nil { + t.Errorf("error getting userdata: '%v'", err) + } + + // Check if we can gzip it. + if _, err := convert.GzipString(s); err != nil { + t.Fatal(err) + } + goldenName := test.name + ".yaml" + testhelper.CompareOutput(t, goldenName, s, *update) + }) + } +} + +// stringPtr returns pointer to given string. +func stringPtr(a string) *string { + return &a +} diff --git a/pkg/userdata/rockylinux/rockylinux.go b/pkg/userdata/rockylinux/rockylinux.go new file mode 100644 index 000000000..f27b2c884 --- /dev/null +++ b/pkg/userdata/rockylinux/rockylinux.go @@ -0,0 +1,59 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rockylinux + +import ( + "encoding/json" + + "k8s.io/apimachinery/pkg/runtime" +) + +// Config contains specific configuration for RockyLinux. +type Config struct { + DistUpgradeOnBoot bool `json:"distUpgradeOnBoot"` +} + +func DefaultConfig(operatingSystemSpec runtime.RawExtension) runtime.RawExtension { + if operatingSystemSpec.Raw == nil { + operatingSystemSpec.Raw, _ = json.Marshal(Config{}) + } + + return operatingSystemSpec +} + +// LoadConfig retrieves the RockyLinux configuration from raw data. +func LoadConfig(r runtime.RawExtension) (*Config, error) { + r = DefaultConfig(r) + cfg := Config{} + + if err := json.Unmarshal(r.Raw, &cfg); err != nil { + return nil, err + } + return &cfg, nil +} + +// Spec return the configuration as raw data. +func (cfg *Config) Spec() (*runtime.RawExtension, error) { + ext := &runtime.RawExtension{} + b, err := json.Marshal(cfg) + if err != nil { + return nil, err + } + + ext.Raw = b + return ext, nil +} diff --git a/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml new file mode 100644 index 000000000..87f087801 --- /dev/null +++ b/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml @@ -0,0 +1,440 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: {} + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml new file mode 100644 index 000000000..71bd55592 --- /dev/null +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml @@ -0,0 +1,424 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: {} + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml new file mode 100644 index 000000000..83de75b32 --- /dev/null +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml @@ -0,0 +1,424 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=${KUBELET_HOSTNAME} \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: {} + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml new file mode 100644 index 000000000..e465d54d8 --- /dev/null +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml @@ -0,0 +1,424 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: {} + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml new file mode 100644 index 000000000..a71469379 --- /dev/null +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml @@ -0,0 +1,432 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + +hostname: node1 + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + + hostnamectl set-hostname node1 + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + iscsi-initiator-utils \ + ipvsadm + systemctl enable --now iscsid + + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=nutanix \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: {} + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml new file mode 100644 index 000000000..194aae181 --- /dev/null +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -0,0 +1,441 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + +hostname: node1 + + +ssh_pwauth: false + +write_files: +- path: "/etc/environment" + content: | + HTTP_PROXY=http://192.168.100.100:3128 + http_proxy=http://192.168.100.100:3128 + HTTPS_PROXY=http://192.168.100.100:3128 + https_proxy=http://192.168.100.100:3128 + NO_PROXY=192.168.1.0 + no_proxy=192.168.1.0 + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + + hostnamectl set-hostname node1 + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + open-vm-tools \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + + systemctl enable --now vmtoolsd.service + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=vsphere \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: {} + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"registry-mirrors":["https://registry.docker-cn.com/"]} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml new file mode 100644 index 000000000..e732a57a2 --- /dev/null +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -0,0 +1,441 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + +hostname: node1 + + +ssh_pwauth: false + +write_files: +- path: "/etc/environment" + content: | + HTTP_PROXY=http://192.168.100.100:3128 + http_proxy=http://192.168.100.100:3128 + HTTPS_PROXY=http://192.168.100.100:3128 + https_proxy=http://192.168.100.100:3128 + NO_PROXY=192.168.1.0 + no_proxy=192.168.1.0 + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + + hostnamectl set-hostname node1 + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + open-vm-tools \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + + systemctl enable --now vmtoolsd.service + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=vsphere \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: {} + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"insecure-registries":["192.168.100.100:5000","10.0.0.1:5000"]} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml new file mode 100644 index 000000000..623124e03 --- /dev/null +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml @@ -0,0 +1,432 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + +hostname: node1 + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + + hostnamectl set-hostname node1 + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + open-vm-tools \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + + systemctl enable --now vmtoolsd.service + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=vsphere \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: {} + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml new file mode 100644 index 000000000..e40bad813 --- /dev/null +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml @@ -0,0 +1,424 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: {} + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml new file mode 100644 index 000000000..1b491b4bd --- /dev/null +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml @@ -0,0 +1,422 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: {} + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index f20401849..6e1e497fc 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -321,7 +321,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { fmt.Sprintf("<< NETWORK_NAME >>=%s", osNetwork), } - selector := Not(OsSelector("sles", "rhel", "amzn2")) + selector := Not(OsSelector("sles", "rhel", "amzn2", "rockylinux")) runScenarios(t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -605,7 +605,7 @@ func TestAzureProvisioningE2E(t *testing.T) { t.Fatal("unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") } - selector := Not(OsSelector("sles", "amzn2")) + selector := Not(OsSelector("sles", "amzn2", "rockylinux")) // act params := []string{ fmt.Sprintf("<< AZURE_TENANT_ID >>=%s", azureTenantID), @@ -729,7 +729,7 @@ func TestEquinixMetalProvisioningE2E(t *testing.T) { t.Fatal("unable to run the test suite, METAL_PROJECT_ID environment variable cannot be empty") } - selector := Not(OsSelector("sles", "rhel", "amzn2")) + selector := Not(OsSelector("sles", "rhel", "amzn2", "rockylinux")) // act params := []string{ @@ -808,7 +808,7 @@ func getVSphereTestParams(t *testing.T) []string { func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() - selector := Not(OsSelector("sles", "amzn2")) + selector := Not(OsSelector("sles", "amzn2", "rockylinux")) params := getVSphereTestParams(t) runScenarios(t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) @@ -869,7 +869,7 @@ func TestScalewayProvisioningE2E(t *testing.T) { t.Fatal("unable to run the test suite, SCW_E2E_TEST_PROJECT_ID environment variable cannot be empty") } - selector := Not(OsSelector("sles", "rhel", "flatcar")) + selector := Not(OsSelector("sles", "rhel", "flatcar", "rockylinux")) // act params := []string{ fmt.Sprintf("<< SCW_ACCESS_KEY >>=%s", scwAccessKey), diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 8583dd074..aa356318d 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -46,6 +46,7 @@ var ( providerconfigtypes.OperatingSystemSLES, providerconfigtypes.OperatingSystemRHEL, providerconfigtypes.OperatingSystemFlatcar, + providerconfigtypes.OperatingSystemRockyLinux, } openStackImages = map[string]string{ From 3f004f8678df6e0803d451e0ed02efe5373e6014 Mon Sep 17 00:00:00 2001 From: Helene Durand <90329641+hdurand0710@users.noreply.github.com> Date: Wed, 6 Apr 2022 15:05:20 +0200 Subject: [PATCH 110/489] Kubevirt: fixed randomised MAC address for VM (#1238) * Kubevirt: Force fixed MAC address for VM Signed-off-by: Helene Durand * Error if rand MAC address failed Signed-off-by: Helene Durand --- .../provider/kubevirt/provider.go | 22 ++++++++++++++++++- pkg/cloudprovider/util/net.go | 18 +++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index dfcde1fbd..75f5ecfec 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -35,6 +35,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" kubevirttypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/kubevirt/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + netutil "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" controllerutil "github.com/kubermatic/machine-controller/pkg/controller/util" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -513,6 +514,11 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return nil, fmt.Errorf("dataVolumeName size %v, is bigger than 63 characters", len(dataVolumeName)) } + defaultBridgeNetwork, err := defaultBridgeNetwork() + if err != nil { + return nil, fmt.Errorf("could not compute a random MAC address") + } + virtualMachine := &kubevirtv1.VirtualMachine{ ObjectMeta: metav1.ObjectMeta{ Name: machine.Name, @@ -527,9 +533,13 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert Labels: labels, }, Spec: kubevirtv1.VirtualMachineInstanceSpec{ + Networks: []kubevirtv1.Network{ + *kubevirtv1.DefaultPodNetwork(), + }, Domain: kubevirtv1.DomainSpec{ Devices: kubevirtv1.Devices{ - Disks: getVMDisks(c), + Disks: getVMDisks(c), + Interfaces: []kubevirtv1.Interface{*defaultBridgeNetwork}, }, Resources: resourceRequirements, }, @@ -643,6 +653,16 @@ func getVMDisks(config *Config) []kubevirtv1.Disk { return disks } +func defaultBridgeNetwork() (*kubevirtv1.Interface, error) { + defaultBridgeNetwork := kubevirtv1.DefaultBridgeNetworkInterface() + mac, err := netutil.GenerateRandMAC() + if err != nil { + return nil, err + } + defaultBridgeNetwork.MacAddress = mac.String() + return defaultBridgeNetwork, nil +} + func getVMVolumes(config *Config, dataVolumeName string, userDataSecretName string) []kubevirtv1.Volume { volumes := []kubevirtv1.Volume{ { diff --git a/pkg/cloudprovider/util/net.go b/pkg/cloudprovider/util/net.go index a179aa94c..ec436bedb 100644 --- a/pkg/cloudprovider/util/net.go +++ b/pkg/cloudprovider/util/net.go @@ -17,6 +17,7 @@ limitations under the License. package util import ( + "crypto/rand" "errors" "fmt" "net" @@ -36,3 +37,20 @@ func CIDRToIPAndNetMask(ipv4 string) (string, string, int, error) { netmask := fmt.Sprintf("%d.%d.%d.%d", ipNet.Mask[0], ipNet.Mask[1], ipNet.Mask[2], ipNet.Mask[3]) return ip.String(), netmask, size, nil } + +// GenerateRandMAC generates a random unicast and locally administered MAC address. +func GenerateRandMAC() (net.HardwareAddr, error) { + buf := make([]byte, 6) + var mac net.HardwareAddr + + _, err := rand.Read(buf) + if err != nil { + return mac, err + } + + // Set locally administered addresses bit and reset multicast bit + buf[0] = (buf[0] | 0x02) & 0xfe + mac = append(mac, buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]) + + return mac, nil +} From 4f5fd92e6a297913840e43df2926d0b4bd352159 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Thu, 7 Apr 2022 00:44:31 +0200 Subject: [PATCH 111/489] Support rocky linux in azure cloud provider (#1241) * updating fixtures Signed-off-by: Moath Qasim * support rocky linux in azure Signed-off-by: Moath Qasim * refactor swap disabling Signed-off-by: Moath Qasim * update fixtures Signed-off-by: Moath Qasim * addressing PR review Signed-off-by: Moath Qasim --- pkg/cloudprovider/provider/azure/provider.go | 11 +++++++++++ pkg/userdata/amzn2/provider.go | 14 +++++++++----- .../testdata/containerd-kubelet-v1.20-aws.yaml | 13 +++++++++++-- .../amzn2/testdata/kubelet-v1.20-aws.yaml | 13 +++++++++++-- .../testdata/kubelet-v1.21-aws-external.yaml | 13 +++++++++++-- .../amzn2/testdata/kubelet-v1.21-aws.yaml | 13 +++++++++++-- .../kubelet-v1.21-vsphere-mirrors.yaml | 13 +++++++++++-- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 13 +++++++++++-- .../amzn2/testdata/kubelet-v1.21-vsphere.yaml | 13 +++++++++++-- .../amzn2/testdata/kubelet-v1.22-aws.yaml | 13 +++++++++++-- .../amzn2/testdata/kubelet-v1.23-aws.yaml | 13 +++++++++++-- pkg/userdata/centos/provider.go | 13 +++++++++---- .../testdata/kubelet-containerd-v1.20-aws.yaml | 14 ++++++++++++-- .../centos/testdata/kubelet-v1.20-aws.yaml | 14 ++++++++++++-- .../testdata/kubelet-v1.21-aws-external.yaml | 14 ++++++++++++-- .../centos/testdata/kubelet-v1.21-aws.yaml | 14 ++++++++++++-- .../centos/testdata/kubelet-v1.21-nutanix.yaml | 14 ++++++++++++-- .../kubelet-v1.21-vsphere-mirrors.yaml | 14 ++++++++++++-- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 14 ++++++++++++-- .../centos/testdata/kubelet-v1.21-vsphere.yaml | 14 ++++++++++++-- .../centos/testdata/kubelet-v1.22-aws.yaml | 14 ++++++++++++-- .../centos/testdata/kubelet-v1.23-aws.yaml | 14 ++++++++++++-- pkg/userdata/flatcar/provider.go | 4 ++-- .../flatcar/testdata/cloud-init_v1.20.14.yaml | 1 + .../flatcar/testdata/cloud-init_v1.21.8.yaml | 1 + .../flatcar/testdata/cloud-init_v1.22.5.yaml | 1 + .../flatcar/testdata/cloud-init_v1.23.0.yaml | 1 + pkg/userdata/flatcar/testdata/containerd.yaml | 1 + .../flatcar/testdata/ignition_v1.20.14.json | 2 +- .../flatcar/testdata/ignition_v1.21.8.json | 2 +- .../flatcar/testdata/ignition_v1.22.5.json | 2 +- .../flatcar/testdata/ignition_v1.23.0.json | 2 +- pkg/userdata/helper/kubelet.go | 7 ++++++- pkg/userdata/helper/kubelet_test.go | 1 + ...blet_systemd_unit_cloud-provider-set.golden | 3 +++ ...et_systemd_unit_multiple-dns-servers.golden | 3 +++ .../kublet_systemd_unit_pause-image-set.golden | 3 +++ .../kublet_systemd_unit_taints-set.golden | 3 +++ ...stemd_unit_version-v1.20.14-external.golden | 3 +++ ...kublet_systemd_unit_version-v1.20.14.golden | 3 +++ ...ystemd_unit_version-v1.21.8-external.golden | 3 +++ .../kublet_systemd_unit_version-v1.21.8.golden | 3 +++ ...ystemd_unit_version-v1.22.5-external.golden | 3 +++ .../kublet_systemd_unit_version-v1.22.5.golden | 3 +++ ...ystemd_unit_version-v1.23.0-external.golden | 3 +++ .../kublet_systemd_unit_version-v1.23.0.golden | 3 +++ pkg/userdata/rhel/provider.go | 14 +++++++++----- .../testdata/kubelet-containerd-v1.20-aws.yaml | 13 +++++++++++-- .../rhel/testdata/kubelet-v1.20-aws.yaml | 13 +++++++++++-- .../rhel/testdata/kubelet-v1.21-aws.yaml | 13 +++++++++++-- .../rhel/testdata/kubelet-v1.22-aws.yaml | 13 +++++++++++-- .../rhel/testdata/kubelet-v1.22-nutanix.yaml | 13 +++++++++++-- .../testdata/kubelet-v1.23-aws-external.yaml | 13 +++++++++++-- .../rhel/testdata/kubelet-v1.23-aws.yaml | 13 +++++++++++-- .../kubelet-v1.23-vsphere-mirrors.yaml | 13 +++++++++++-- .../testdata/kubelet-v1.23-vsphere-proxy.yaml | 13 +++++++++++-- .../rhel/testdata/kubelet-v1.23-vsphere.yaml | 13 +++++++++++-- .../rhel/testdata/pod-cidr-azure-rhel.yaml | 13 +++++++++++-- pkg/userdata/rockylinux/provider.go | 14 +++++++++----- .../testdata/kubelet-containerd-v1.20-aws.yaml | 13 +++++++++++-- .../rockylinux/testdata/kubelet-v1.20-aws.yaml | 13 +++++++++++-- .../testdata/kubelet-v1.21-aws-external.yaml | 13 +++++++++++-- .../rockylinux/testdata/kubelet-v1.21-aws.yaml | 13 +++++++++++-- .../testdata/kubelet-v1.21-nutanix.yaml | 13 +++++++++++-- .../kubelet-v1.21-vsphere-mirrors.yaml | 13 +++++++++++-- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 13 +++++++++++-- .../testdata/kubelet-v1.21-vsphere.yaml | 13 +++++++++++-- .../rockylinux/testdata/kubelet-v1.22-aws.yaml | 13 +++++++++++-- .../rockylinux/testdata/kubelet-v1.23-aws.yaml | 13 +++++++++++-- pkg/userdata/sles/provider.go | 18 +++++++++++------- .../sles/testdata/dist-upgrade-on-boot.yaml | 17 +++++++++++++---- .../kubelet-version-without-v-prefix.yaml | 17 +++++++++++++---- .../sles/testdata/multiple-dns-servers.yaml | 17 +++++++++++++---- .../sles/testdata/multiple-ssh-keys.yaml | 17 +++++++++++++---- .../openstack-overwrite-cloud-config.yaml | 17 +++++++++++++---- pkg/userdata/sles/testdata/openstack.yaml | 17 +++++++++++++---- .../sles/testdata/version-1.20.14.yaml | 17 +++++++++++++---- pkg/userdata/sles/testdata/version-1.21.8.yaml | 17 +++++++++++++---- pkg/userdata/sles/testdata/version-1.22.5.yaml | 17 +++++++++++++---- pkg/userdata/sles/testdata/version-1.23.0.yaml | 17 +++++++++++++---- .../sles/testdata/vsphere-mirrors.yaml | 17 +++++++++++++---- pkg/userdata/sles/testdata/vsphere-proxy.yaml | 17 +++++++++++++---- pkg/userdata/sles/testdata/vsphere.yaml | 17 +++++++++++++---- pkg/userdata/ubuntu/provider.go | 13 +++++++------ pkg/userdata/ubuntu/testdata/containerd.yaml | 12 +++++++++--- .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 12 +++++++++--- .../kubelet-version-without-v-prefix.yaml | 12 +++++++++--- .../ubuntu/testdata/multiple-dns-servers.yaml | 12 +++++++++--- .../ubuntu/testdata/multiple-ssh-keys.yaml | 12 +++++++++--- pkg/userdata/ubuntu/testdata/nutanix.yaml | 12 +++++++++--- .../openstack-overwrite-cloud-config.yaml | 12 +++++++++--- pkg/userdata/ubuntu/testdata/openstack.yaml | 12 +++++++++--- .../ubuntu/testdata/version-1.20.14.yaml | 12 +++++++++--- .../ubuntu/testdata/version-1.21.8.yaml | 12 +++++++++--- .../ubuntu/testdata/version-1.22.5.yaml | 12 +++++++++--- .../ubuntu/testdata/version-1.23.0.yaml | 12 +++++++++--- .../ubuntu/testdata/vsphere-mirrors.yaml | 12 +++++++++--- .../ubuntu/testdata/vsphere-proxy.yaml | 12 +++++++++--- pkg/userdata/ubuntu/testdata/vsphere.yaml | 12 +++++++++--- test/e2e/provisioning/all_e2e_test.go | 2 +- 100 files changed, 874 insertions(+), 217 deletions(-) diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 402f54258..852bfd1a1 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -141,6 +141,12 @@ var imageReferences = map[providerconfigtypes.OperatingSystem]compute.ImageRefer Sku: to.StringPtr("stable"), Version: to.StringPtr("2905.2.5"), }, + providerconfigtypes.OperatingSystemRockyLinux: { + Publisher: to.StringPtr("procomputers"), + Offer: to.StringPtr("rocky-linux-8-5"), + Sku: to.StringPtr("rocky-linux-8-5"), + Version: to.StringPtr("8.5.20211118"), + }, } var osPlans = map[providerconfigtypes.OperatingSystem]*compute.Plan{ @@ -154,6 +160,11 @@ var osPlans = map[providerconfigtypes.OperatingSystem]*compute.Plan{ Publisher: pointer.StringPtr("redhat"), Product: pointer.StringPtr("rhel-byos"), }, + providerconfigtypes.OperatingSystemRockyLinux: { + Name: pointer.StringPtr("rocky-linux-8-5"), + Publisher: pointer.StringPtr("procomputers"), + Product: pointer.StringPtr("rocky-linux-8-5"), + }, } func getOSImageReference(c *config, os providerconfigtypes.OperatingSystem) (*compute.ImageReference, error) { diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index 7a2738c20..27a22d0f8 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -196,10 +196,6 @@ write_files: {{- /* As we added some modules and don't want to reboot, restart the service */}} systemctl restart systemd-modules-load.service sysctl --system - -{{- /* Make sure we always disable swap - Otherwise the kubelet won't start */}} - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a {{ if ne .CloudProviderName "aws" }} {{- /* The normal way of setting it via cloud-init is broken, see */}} {{- /* https://bugs.launchpad.net/cloud-init/+bug/1662542 */}} @@ -245,9 +241,17 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" diff --git a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml index f117fab45..eb3c59a01 100644 --- a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml @@ -64,8 +64,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ @@ -185,6 +183,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -205,6 +211,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml index c64ffeb01..12f9ae2dc 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml @@ -64,8 +64,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ @@ -182,6 +180,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -202,6 +208,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml index 33612520b..bcee3bc4d 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml @@ -64,8 +64,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ @@ -182,6 +180,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -202,6 +208,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml index 771ab850a..fd0294c00 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml @@ -64,8 +64,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ @@ -182,6 +180,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -202,6 +208,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml index ec3dada07..de7f75839 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -74,8 +74,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a hostnamectl set-hostname node1 @@ -197,6 +195,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -217,6 +223,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml index 6a88add1a..e7bec561a 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -74,8 +74,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a hostnamectl set-hostname node1 @@ -197,6 +195,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -217,6 +223,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml index 3160341c3..cbac1694e 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml @@ -66,8 +66,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a hostnamectl set-hostname node1 @@ -189,6 +187,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -209,6 +215,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml index c0a527620..1040b8d0d 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml @@ -64,8 +64,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ @@ -182,6 +180,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -202,6 +208,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml index ce9f0bbfd..aa7340673 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml @@ -64,8 +64,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ @@ -182,6 +180,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -202,6 +208,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index 5a9da4033..85f8b74d1 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -197,9 +197,6 @@ write_files: systemctl restart systemd-modules-load.service sysctl --system -{{- /* Make sure we always disable swap - Otherwise the kubelet won't start */}} - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a {{ if ne .CloudProviderName "aws" }} {{- /* The normal way of setting it via cloud-init is broken, see */}} {{- /* https://bugs.launchpad.net/cloud-init/+bug/1662542 */}} @@ -259,9 +256,17 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" diff --git a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml index 77b9778eb..482b6fb28 100644 --- a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml @@ -64,8 +64,7 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a + source /etc/os-release if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then @@ -190,6 +189,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -210,6 +217,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml index e313aaec6..bda03b455 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml @@ -64,8 +64,7 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a + source /etc/os-release if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then @@ -191,6 +190,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -211,6 +218,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml index c61119733..07c0f2b8d 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml @@ -64,8 +64,7 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a + source /etc/os-release if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then @@ -191,6 +190,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -211,6 +218,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml index 0ea0405b6..94bed7d94 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml @@ -64,8 +64,7 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a + source /etc/os-release if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then @@ -191,6 +190,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -211,6 +218,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml index 42c8a2281..534abb608 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml @@ -66,8 +66,7 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a + hostnamectl set-hostname node1 @@ -198,6 +197,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -218,6 +225,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 38cb4db42..867c272f9 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -74,8 +74,7 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a + hostnamectl set-hostname node1 @@ -206,6 +205,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -226,6 +233,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml index 2be22f741..61d4bd4a9 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -74,8 +74,7 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a + hostnamectl set-hostname node1 @@ -206,6 +205,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -226,6 +233,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml index e706071af..6f7891bc8 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml @@ -66,8 +66,7 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a + hostnamectl set-hostname node1 @@ -198,6 +197,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -218,6 +225,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index 63e68d4c3..e7c800083 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -64,8 +64,7 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a + source /etc/os-release if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then @@ -191,6 +190,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -211,6 +218,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index 0669410fb..b98a62fd8 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -64,8 +64,7 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a + source /etc/os-release if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then @@ -191,6 +190,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -211,6 +218,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index c979ece67..85f4c3941 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -252,7 +252,7 @@ systemd: Requires=download-script.service After=download-script.service contents: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 8 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags false | indent 8 }} storage: files: @@ -522,7 +522,7 @@ coreos: Requires=download-script.service After=download-script.service content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 6 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags false | indent 6 }} - name: apply-sysctl-settings.service enable: true diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml index 9021dca2d..a1485c7cc 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml @@ -112,6 +112,7 @@ coreos: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml index 4bdc80d4f..f773b158f 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml @@ -112,6 +112,7 @@ coreos: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml index 4dadc5c3a..52ca397e4 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml @@ -112,6 +112,7 @@ coreos: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml index 9198d6f4e..78e44ad4f 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml @@ -112,6 +112,7 @@ coreos: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index 41b76292c..b24e425d1 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -97,6 +97,7 @@ coreos: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json b/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json index 10cc21ece..6c7dd9a99 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.14%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.14%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json b/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json index a9a065094..737e8c4f6 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.8%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.8%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json b/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json index 75137b5e8..d1a95c62a 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json index 39e5df380..9867446fa 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index b74c5ab84..d5f3f1b7b 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -84,6 +84,9 @@ Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bi EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh +{{ if .DisableSwap }} +ExecStartPre=/bin/bash /opt/disable-swap.sh +{{ end }} ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ {{ kubeletFlags .KubeletVersion .CloudProvider .Hostname .ClusterDNSIPs .IsExternal .PauseImage .InitialTaints .ExtraKubeletFlags | indent 2 }} @@ -133,7 +136,7 @@ func CloudProviderFlags(cpName string, external bool) (string, error) { } // KubeletSystemdUnit returns the systemd unit for the kubelet -func KubeletSystemdUnit(containerRuntime, kubeletVersion, cloudProvider, hostname string, dnsIPs []net.IP, external bool, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string) (string, error) { +func KubeletSystemdUnit(containerRuntime, kubeletVersion, cloudProvider, hostname string, dnsIPs []net.IP, external bool, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string, disableSwap bool) (string, error) { tmpl, err := template.New("kubelet-systemd-unit").Funcs(TxtFuncMap()).Parse(kubeletSystemdUnitTpl) if err != nil { return "", fmt.Errorf("failed to parse kubelet-systemd-unit template: %v", err) @@ -149,6 +152,7 @@ func KubeletSystemdUnit(containerRuntime, kubeletVersion, cloudProvider, hostnam PauseImage string InitialTaints []corev1.Taint ExtraKubeletFlags []string + DisableSwap bool }{ ContainerRuntime: containerRuntime, KubeletVersion: kubeletVersion, @@ -159,6 +163,7 @@ func KubeletSystemdUnit(containerRuntime, kubeletVersion, cloudProvider, hostnam PauseImage: pauseImage, InitialTaints: initialTaints, ExtraKubeletFlags: extraKubeletFlags, + DisableSwap: disableSwap, } var buf strings.Builder diff --git a/pkg/userdata/helper/kubelet_test.go b/pkg/userdata/helper/kubelet_test.go index e5e1b33f7..9e4f6acac 100644 --- a/pkg/userdata/helper/kubelet_test.go +++ b/pkg/userdata/helper/kubelet_test.go @@ -120,6 +120,7 @@ func TestKubeletSystemdUnit(t *testing.T) { test.pauseImage, test.initialTaints, test.extraFlags, + true, ) if err != nil { t.Error(err) diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden index 56147d951..42484e368 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden @@ -16,6 +16,9 @@ Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bi EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden index b05f41061..2c7c03898 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden @@ -16,6 +16,9 @@ Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bi EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_pause-image-set.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_pause-image-set.golden index aee30afcf..0d21c5dd6 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_pause-image-set.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_pause-image-set.golden @@ -16,6 +16,9 @@ Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bi EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_taints-set.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_taints-set.golden index 7c4722d6d..0db8bf8ae 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_taints-set.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_taints-set.golden @@ -16,6 +16,9 @@ Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bi EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14-external.golden index e595e9677..50c334831 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14-external.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14-external.golden @@ -16,6 +16,9 @@ Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bi EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14.golden index b05f41061..2c7c03898 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14.golden @@ -16,6 +16,9 @@ Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bi EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8-external.golden index e595e9677..50c334831 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8-external.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8-external.golden @@ -16,6 +16,9 @@ Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bi EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8.golden index b05f41061..2c7c03898 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8.golden @@ -16,6 +16,9 @@ Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bi EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5-external.golden index e595e9677..50c334831 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5-external.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5-external.golden @@ -16,6 +16,9 @@ Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bi EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5.golden index b05f41061..2c7c03898 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5.golden @@ -16,6 +16,9 @@ Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bi EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0-external.golden index 672b55ae0..62ee82ac5 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0-external.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0-external.golden @@ -16,6 +16,9 @@ Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bi EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0.golden index c7b334881..8cd283a84 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0.golden @@ -16,6 +16,9 @@ Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bi EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 140c065b1..52ea94964 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -199,10 +199,6 @@ write_files: {{- /* As we added some modules and don't want to reboot, restart the service */}} systemctl restart systemd-modules-load.service sysctl --system - -{{- /* Make sure we always disable swap - Otherwise the kubelet won't start */}} - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a {{ if ne .CloudProviderName "aws" }} {{- /* The normal way of setting it via cloud-init is broken, see */}} {{- /* https://bugs.launchpad.net/cloud-init/+bug/1662542 */}} @@ -257,9 +253,17 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" diff --git a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml index 6a89d2961..41d1fc4a5 100644 --- a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml @@ -66,8 +66,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ @@ -186,6 +184,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -206,6 +212,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml index a83e35482..7236a9128 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml @@ -66,8 +66,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ @@ -187,6 +185,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -207,6 +213,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml index 6d90d2366..a94fb37f5 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml @@ -66,8 +66,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ @@ -187,6 +185,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -207,6 +213,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index 027713804..e7e4c7751 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -66,8 +66,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ @@ -187,6 +185,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -207,6 +213,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml index ff8bfc891..af7856f57 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml @@ -69,8 +69,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a hostnamectl set-hostname node1 @@ -195,6 +193,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -215,6 +221,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index b563dfc60..30e6d0b07 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -66,8 +66,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ @@ -187,6 +185,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -207,6 +213,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index 6280f54b3..a9773509f 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -66,8 +66,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ @@ -187,6 +185,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -207,6 +213,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 0da895b52..e25b602bf 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -77,8 +77,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a hostnamectl set-hostname node1 @@ -203,6 +201,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -223,6 +229,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index 1d52918bf..d620aed6b 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -77,8 +77,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a hostnamectl set-hostname node1 @@ -203,6 +201,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -223,6 +229,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index ea267147a..a108bc213 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -69,8 +69,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a hostnamectl set-hostname node1 @@ -195,6 +193,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -215,6 +221,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index 4800ed5b0..f759a4f54 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -69,8 +69,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a hostnamectl set-hostname node1 @@ -194,6 +192,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -214,6 +220,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rockylinux/provider.go b/pkg/userdata/rockylinux/provider.go index fc3db6849..cc9671f74 100644 --- a/pkg/userdata/rockylinux/provider.go +++ b/pkg/userdata/rockylinux/provider.go @@ -199,10 +199,6 @@ write_files: systemctl restart systemd-modules-load.service sysctl --system -{{- /* Make sure we always disable swap - Otherwise the kubelet won't start */}} - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - {{ if ne .CloudProviderName "aws" }} {{- /* The normal way of setting it via cloud-init is broken, see */}} {{- /* https://bugs.launchpad.net/cloud-init/+bug/1662542 */}} @@ -254,9 +250,17 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml index 87f087801..f77f30de9 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml @@ -66,8 +66,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ device-mapper-persistent-data \ @@ -185,6 +183,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -205,6 +211,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml index 71bd55592..2ceaddece 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml @@ -66,8 +66,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ device-mapper-persistent-data \ @@ -186,6 +184,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -206,6 +212,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml index 83de75b32..e7d6e4f1a 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml @@ -66,8 +66,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ device-mapper-persistent-data \ @@ -186,6 +184,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -206,6 +212,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml index e465d54d8..aca5b1cc1 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml @@ -66,8 +66,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ device-mapper-persistent-data \ @@ -186,6 +184,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -206,6 +212,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml index a71469379..32615629c 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml @@ -68,8 +68,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a hostnamectl set-hostname node1 @@ -193,6 +191,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -213,6 +219,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 194aae181..5d7056570 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -76,8 +76,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a hostnamectl set-hostname node1 @@ -201,6 +199,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -221,6 +227,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml index e732a57a2..b7bbcad16 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -76,8 +76,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a hostnamectl set-hostname node1 @@ -201,6 +199,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -221,6 +227,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml index 623124e03..a13fb8cdd 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml @@ -68,8 +68,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a hostnamectl set-hostname node1 @@ -193,6 +191,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -213,6 +219,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml index e40bad813..34783e6e8 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml @@ -66,8 +66,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ device-mapper-persistent-data \ @@ -186,6 +184,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -206,6 +212,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml index 1b491b4bd..f650bcb1f 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml @@ -66,8 +66,6 @@ write_files: setenforce 0 || true systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a yum install -y \ device-mapper-persistent-data \ @@ -186,6 +184,14 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -206,6 +212,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/sles/provider.go b/pkg/userdata/sles/provider.go index 366ce072e..b2af70268 100644 --- a/pkg/userdata/sles/provider.go +++ b/pkg/userdata/sles/provider.go @@ -174,12 +174,6 @@ write_files: systemctl restart systemd-modules-load.service sysctl --system -{{- /* Make sure we always disable swap - Otherwise the kubelet won't start'. */}} - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - zypper --non-interactive --quiet --color install ebtables \ ceph-common \ e2fsprogs \ @@ -209,9 +203,19 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - path: "/etc/systemd/system/kubelet.service.d/extras.conf" content: | diff --git a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml index 0d3130e2f..faa9f488f 100644 --- a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml @@ -54,10 +54,6 @@ write_files: set -xeuo pipefail systemctl restart systemd-modules-load.service sysctl --system - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a zypper --non-interactive --quiet --color install ebtables \ ceph-common \ @@ -149,6 +145,16 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -169,6 +175,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml index c4f5d3d7f..deadd8625 100644 --- a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml @@ -52,10 +52,6 @@ write_files: set -xeuo pipefail systemctl restart systemd-modules-load.service sysctl --system - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a zypper --non-interactive --quiet --color install ebtables \ ceph-common \ @@ -147,6 +143,16 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -167,6 +173,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml index 4697c9633..f2c31b733 100644 --- a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml @@ -52,10 +52,6 @@ write_files: set -xeuo pipefail systemctl restart systemd-modules-load.service sysctl --system - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a zypper --non-interactive --quiet --color install ebtables \ ceph-common \ @@ -147,6 +143,16 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -167,6 +173,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml index 386253a7d..02ea10f5f 100644 --- a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml @@ -54,10 +54,6 @@ write_files: set -xeuo pipefail systemctl restart systemd-modules-load.service sysctl --system - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a zypper --non-interactive --quiet --color install ebtables \ ceph-common \ @@ -149,6 +145,16 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -169,6 +175,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml index 0a99e47eb..c3eb299b1 100644 --- a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml @@ -52,10 +52,6 @@ write_files: set -xeuo pipefail systemctl restart systemd-modules-load.service sysctl --system - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a zypper --non-interactive --quiet --color install ebtables \ ceph-common \ @@ -147,6 +143,16 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -167,6 +173,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/sles/testdata/openstack.yaml b/pkg/userdata/sles/testdata/openstack.yaml index d9f22a305..3cfb660d2 100644 --- a/pkg/userdata/sles/testdata/openstack.yaml +++ b/pkg/userdata/sles/testdata/openstack.yaml @@ -52,10 +52,6 @@ write_files: set -xeuo pipefail systemctl restart systemd-modules-load.service sysctl --system - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a zypper --non-interactive --quiet --color install ebtables \ ceph-common \ @@ -147,6 +143,16 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -167,6 +173,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/sles/testdata/version-1.20.14.yaml b/pkg/userdata/sles/testdata/version-1.20.14.yaml index 8255e4295..68903a0f9 100644 --- a/pkg/userdata/sles/testdata/version-1.20.14.yaml +++ b/pkg/userdata/sles/testdata/version-1.20.14.yaml @@ -52,10 +52,6 @@ write_files: set -xeuo pipefail systemctl restart systemd-modules-load.service sysctl --system - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a zypper --non-interactive --quiet --color install ebtables \ ceph-common \ @@ -147,6 +143,16 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -167,6 +173,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/sles/testdata/version-1.21.8.yaml b/pkg/userdata/sles/testdata/version-1.21.8.yaml index 416bc52bc..b4dbc3252 100644 --- a/pkg/userdata/sles/testdata/version-1.21.8.yaml +++ b/pkg/userdata/sles/testdata/version-1.21.8.yaml @@ -52,10 +52,6 @@ write_files: set -xeuo pipefail systemctl restart systemd-modules-load.service sysctl --system - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a zypper --non-interactive --quiet --color install ebtables \ ceph-common \ @@ -147,6 +143,16 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -167,6 +173,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/sles/testdata/version-1.22.5.yaml b/pkg/userdata/sles/testdata/version-1.22.5.yaml index c4f5d3d7f..deadd8625 100644 --- a/pkg/userdata/sles/testdata/version-1.22.5.yaml +++ b/pkg/userdata/sles/testdata/version-1.22.5.yaml @@ -52,10 +52,6 @@ write_files: set -xeuo pipefail systemctl restart systemd-modules-load.service sysctl --system - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a zypper --non-interactive --quiet --color install ebtables \ ceph-common \ @@ -147,6 +143,16 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -167,6 +173,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/sles/testdata/version-1.23.0.yaml b/pkg/userdata/sles/testdata/version-1.23.0.yaml index e22ecb368..966a28453 100644 --- a/pkg/userdata/sles/testdata/version-1.23.0.yaml +++ b/pkg/userdata/sles/testdata/version-1.23.0.yaml @@ -52,10 +52,6 @@ write_files: set -xeuo pipefail systemctl restart systemd-modules-load.service sysctl --system - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a zypper --non-interactive --quiet --color install ebtables \ ceph-common \ @@ -147,6 +143,16 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -167,6 +173,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml index 7d233ab75..260a60994 100644 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml @@ -61,10 +61,6 @@ write_files: set -xeuo pipefail systemctl restart systemd-modules-load.service sysctl --system - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a zypper --non-interactive --quiet --color install ebtables \ ceph-common \ @@ -157,6 +153,16 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -177,6 +183,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml index 281d09b70..88c66dd8b 100644 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/sles/testdata/vsphere-proxy.yaml @@ -61,10 +61,6 @@ write_files: set -xeuo pipefail systemctl restart systemd-modules-load.service sysctl --system - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a zypper --non-interactive --quiet --color install ebtables \ ceph-common \ @@ -157,6 +153,16 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -177,6 +183,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml index ec6d9379f..e4b0ff840 100644 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ b/pkg/userdata/sles/testdata/vsphere.yaml @@ -52,10 +52,6 @@ write_files: set -xeuo pipefail systemctl restart systemd-modules-load.service sysctl --system - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a zypper --non-interactive --quiet --color install ebtables \ ceph-common \ @@ -148,6 +144,16 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -168,6 +174,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index 3e435fed8..aa9399a78 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -188,11 +188,6 @@ write_files: {{- /* As we added some modules and don't want to reboot, restart the service */}} systemctl restart systemd-modules-load.service sysctl --system - -{{- /* Make sure we always disable swap - Otherwise the kubelet won't start'. */}} - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - apt-get update DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ @@ -253,9 +248,15 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - path: "/etc/systemd/system/kubelet.service.d/extras.conf" content: | diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index 06b8533f8..b7abc275e 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -62,9 +62,6 @@ write_files: systemctl mask ufw systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - apt-get update DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ @@ -193,6 +190,12 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -213,6 +216,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index ce329c09b..b9d497f8d 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -62,9 +62,6 @@ write_files: systemctl mask ufw systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - apt-get update DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ @@ -193,6 +190,12 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -213,6 +216,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index de0a10f09..cfab1a8dd 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -60,9 +60,6 @@ write_files: systemctl mask ufw systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - apt-get update DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ @@ -191,6 +188,12 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -211,6 +214,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 536e72499..f819a1ff7 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -60,9 +60,6 @@ write_files: systemctl mask ufw systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - apt-get update DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ @@ -191,6 +188,12 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -211,6 +214,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 06e7f29ff..655a9101a 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -62,9 +62,6 @@ write_files: systemctl mask ufw systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - apt-get update DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ @@ -193,6 +190,12 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -213,6 +216,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 362abc7ef..cd0bc21fa 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -60,9 +60,6 @@ write_files: systemctl mask ufw systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - apt-get update DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ @@ -194,6 +191,12 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -214,6 +217,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index e6a02b1c0..abea1da1e 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -60,9 +60,6 @@ write_files: systemctl mask ufw systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - apt-get update DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ @@ -191,6 +188,12 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -211,6 +214,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index ad6055793..4fb4d8f52 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -60,9 +60,6 @@ write_files: systemctl mask ufw systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - apt-get update DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ @@ -191,6 +188,12 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -211,6 +214,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml b/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml index 70d5d896f..fd1ee3dc2 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml @@ -60,9 +60,6 @@ write_files: systemctl mask ufw systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - apt-get update DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ @@ -191,6 +188,12 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -211,6 +214,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml b/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml index 85313d6c1..2019f39a8 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml @@ -60,9 +60,6 @@ write_files: systemctl mask ufw systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - apt-get update DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ @@ -191,6 +188,12 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -211,6 +214,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml index de0a10f09..cfab1a8dd 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml @@ -60,9 +60,6 @@ write_files: systemctl mask ufw systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - apt-get update DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ @@ -191,6 +188,12 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -211,6 +214,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml index eacd59338..2b0a1c8e5 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml @@ -60,9 +60,6 @@ write_files: systemctl mask ufw systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - apt-get update DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ @@ -191,6 +188,12 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -211,6 +214,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index d69464826..13d39f4b5 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -69,9 +69,6 @@ write_files: systemctl mask ufw systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - apt-get update DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ @@ -201,6 +198,12 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -221,6 +224,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 1add0b5ea..5422d4635 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -69,9 +69,6 @@ write_files: systemctl mask ufw systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - apt-get update DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ @@ -201,6 +198,12 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -221,6 +224,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 108cd9c93..ccfaf8ae8 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -60,9 +60,6 @@ write_files: systemctl mask ufw systemctl restart systemd-modules-load.service sysctl --system - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - apt-get update DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ @@ -192,6 +189,12 @@ write_files: sleep 1 done +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + - path: "/etc/systemd/system/kubelet.service" content: | [Unit] @@ -212,6 +215,9 @@ write_files: EnvironmentFile=-/etc/environment ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 6e1e497fc..16cd83c4f 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -605,7 +605,7 @@ func TestAzureProvisioningE2E(t *testing.T) { t.Fatal("unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") } - selector := Not(OsSelector("sles", "amzn2", "rockylinux")) + selector := Not(OsSelector("sles", "amzn2")) // act params := []string{ fmt.Sprintf("<< AZURE_TENANT_ID >>=%s", azureTenantID), From 1c8c76dfc2150b11e60ba82d69cd2de8ec9ecd33 Mon Sep 17 00:00:00 2001 From: Marvin Beckers <10295525+embik@users.noreply.github.com> Date: Thu, 7 Apr 2022 10:03:41 +0200 Subject: [PATCH 112/489] Add support for setting Azure disk types (OS and data) (#1199) * Add support for setting disk types (OS and data) Signed-off-by: Marvin Beckers * Use correct disk types to compare against Signed-off-by: Marvin Beckers * Rename diskType fields to diskSKU (Azure nomenclature) Signed-off-by: Marvin Beckers * Check if SKU supports premium storage Signed-off-by: Marvin Beckers * Cache SKU information Signed-off-by: Marvin Beckers * Include Azure disk SKUs in e2e testing Signed-off-by: Marvin Beckers * Implement disk SKU validation for ultra disks as well Signed-off-by: Marvin Beckers * CyClOmAtIc CoMpLeXitY Signed-off-by: Marvin Beckers * Update pkg/cloudprovider/provider/azure/create_delete_resources.go Co-authored-by: Moath Qasim * Use map instead of slice to store allowed SKUs Signed-off-by: Marvin Beckers * Remove label and adjust loop condition Signed-off-by: Marvin Beckers Co-authored-by: Moath Qasim --- .../provider/azure/create_delete_resources.go | 54 ++++++ .../provider/azure/get_client.go | 11 ++ pkg/cloudprovider/provider/azure/provider.go | 166 ++++++++++++++++++ .../provider/azure/types/types.go | 2 + test/e2e/provisioning/all_e2e_test.go | 6 + .../testdata/machinedeployment-azure.yaml | 2 + 6 files changed, 241 insertions(+) diff --git a/pkg/cloudprovider/provider/azure/create_delete_resources.go b/pkg/cloudprovider/provider/azure/create_delete_resources.go index 3c66064c3..01ee555f5 100644 --- a/pkg/cloudprovider/provider/azure/create_delete_resources.go +++ b/pkg/cloudprovider/provider/azure/create_delete_resources.go @@ -250,6 +250,60 @@ func getSubnet(ctx context.Context, c *config) (network.Subnet, error) { return subnetsClient.Get(ctx, c.VNetResourceGroup, c.VNetName, c.SubnetName, "") } +func getSKU(ctx context.Context, c *config) (compute.ResourceSku, error) { + cacheLock.Lock() + defer cacheLock.Unlock() + + cacheKey := fmt.Sprintf("%s-%s", c.Location, c.VMSize) + cacheSku, found := cache.Get(cacheKey) + if found { + klog.V(3).Info("found SKU in cache!") + return cacheSku.(compute.ResourceSku), nil + } + + skuClient, err := getSKUClient(c) + if err != nil { + return compute.ResourceSku{}, fmt.Errorf("failed to (create) SKU client: %w", err) + } + + skuPages, err := skuClient.List(ctx, fmt.Sprintf("location eq '%s'", c.Location), "false") + if err != nil { + return compute.ResourceSku{}, fmt.Errorf("failed to list available SKUs: %w", err) + } + + var sku *compute.ResourceSku + + for skuPages.NotDone() && sku == nil { + skus := skuPages.Values() + for _, skuResult := range skus { + // skip invalid SKU results so we don't trigger a nil pointer exception + if skuResult.ResourceType == nil || skuResult.Name == nil { + continue + } + + if *skuResult.ResourceType == "virtualMachines" && *skuResult.Name == c.VMSize { + sku = &skuResult + break + } + } + + // only fetch the next page if we haven't found our SKU yet + if sku == nil { + if err := skuPages.NextWithContext(ctx); err != nil { + return compute.ResourceSku{}, fmt.Errorf("failed to list available SKUs: %w", err) + } + } + } + + if sku == nil { + return compute.ResourceSku{}, fmt.Errorf("no VM SKU '%s' found for subscription '%s'", c.VMSize, c.SubscriptionID) + } + + cache.SetDefault(cacheKey, *sku) + + return *sku, nil +} + func getVirtualNetwork(ctx context.Context, c *config) (network.VirtualNetwork, error) { virtualNetworksClient, err := getVirtualNetworksClient(c) if err != nil { diff --git a/pkg/cloudprovider/provider/azure/get_client.go b/pkg/cloudprovider/provider/azure/get_client.go index 9b3f1aaae..9396178f8 100644 --- a/pkg/cloudprovider/provider/azure/get_client.go +++ b/pkg/cloudprovider/provider/azure/get_client.go @@ -78,6 +78,17 @@ func getVMClient(c *config) (*compute.VirtualMachinesClient, error) { return &vmClient, nil } +func getSKUClient(c *config) (*compute.ResourceSkusClient, error) { + var err error + skuClient := compute.NewResourceSkusClient(c.SubscriptionID) + skuClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() + if err != nil { + return nil, fmt.Errorf("failed to create authorizer: %w", err) + } + + return &skuClient, nil +} + func getInterfacesClient(c *config) (*network.InterfacesClient, error) { var err error ifClient := network.NewInterfacesClient(c.SubscriptionID) diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 852bfd1a1..fa7112887 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -22,10 +22,13 @@ import ( "errors" "fmt" "strings" + "sync" + "time" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" "github.com/Azure/go-autorest/autorest/to" + gocache "github.com/patrickmn/go-cache" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -46,6 +49,10 @@ import ( ) const ( + CapabilityPremiumIO = "PremiumIO" + CapabilityUltraSSD = "UltraSSDAvailable" + CapabilityValueTrue = "True" + machineUIDTag = "Machine-UID" finalizerPublicIP = "kubermatic.io/cleanup-azure-public-ip" @@ -88,7 +95,9 @@ type config struct { ImageReference *compute.ImageReference OSDiskSize int32 + OSDiskSKU *compute.StorageAccountTypes DataDiskSize int32 + DataDiskSKU *compute.StorageAccountTypes AssignPublicIP bool Tags map[string]string @@ -167,6 +176,26 @@ var osPlans = map[providerconfigtypes.OperatingSystem]*compute.Plan{ }, } +var osDiskSKUs = map[compute.StorageAccountTypes]string{ + compute.StorageAccountTypesStandardLRS: "", // Standard_LRS + compute.StorageAccountTypesStandardSSDLRS: "", // StandardSSD_LRS + compute.StorageAccountTypesPremiumLRS: "", // Premium_LRS +} + +var dataDiskSKUs = map[compute.StorageAccountTypes]string{ + compute.StorageAccountTypesStandardLRS: "", // Standard_LRS + compute.StorageAccountTypesStandardSSDLRS: "", // StandardSSD_LRS + compute.StorageAccountTypesPremiumLRS: "", // Premium_LRS + compute.StorageAccountTypesUltraSSDLRS: "", // UltraSSD_LRS +} + +var ( + // cacheLock protects concurrent cache misses against a single key. This usually happens when multiple machines get created simultaneously + // We lock so the first access updates/writes the data to the cache and afterwards everyone reads the cached data + cacheLock = &sync.Mutex{} + cache = gocache.New(10*time.Minute, 10*time.Minute) +) + func getOSImageReference(c *config, os providerconfigtypes.OperatingSystem) (*compute.ImageReference, error) { if c.ImageID != "" { return &compute.ImageReference{ @@ -302,6 +331,14 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p c.OSDiskSize = rawCfg.OSDiskSize c.DataDiskSize = rawCfg.DataDiskSize + if rawCfg.OSDiskSKU != nil { + c.OSDiskSKU = storageTypePtr(*rawCfg.OSDiskSKU) + } + + if rawCfg.DataDiskSKU != nil { + c.DataDiskSKU = storageTypePtr(*rawCfg.DataDiskSKU) + } + if rawCfg.ImagePlan != nil && rawCfg.ImagePlan.Name != "" { c.ImagePlan = &compute.Plan{ Name: pointer.StringPtr(rawCfg.ImagePlan.Name), @@ -476,6 +513,12 @@ func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) DiskSizeGB: pointer.Int32Ptr(config.OSDiskSize), CreateOption: compute.DiskCreateOptionTypesFromImage, } + + if config.OSDiskSKU != nil { + sp.OsDisk.ManagedDisk = &compute.ManagedDiskParameters{ + StorageAccountType: *config.OSDiskSKU, + } + } } if config.DataDiskSize != 0 { @@ -487,6 +530,13 @@ func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) CreateOption: compute.DiskCreateOptionTypesEmpty, }, } + + if config.DataDiskSKU != nil { + (*sp.DataDisks)[0].ManagedDisk = &compute.ManagedDiskParameters{ + StorageAccountType: *config.DataDiskSKU, + } + } + } return sp, nil } @@ -914,10 +964,51 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return fmt.Errorf("failed to get subnet: %v", err) } + if err := validateDiskSKUs(c); err != nil { + return fmt.Errorf("failed to validate disk SKUs: %w", err) + } + _, err = getOSImageReference(c, providerCfg.OperatingSystem) return err } +func validateDiskSKUs(c *config) error { + if c.OSDiskSKU != nil || c.DataDiskSKU != nil { + sku, err := getSKU(context.TODO(), c) + if err != nil { + return fmt.Errorf("failed to get VM SKU: %w", err) + } + + if c.OSDiskSKU != nil { + if _, ok := osDiskSKUs[*c.OSDiskSKU]; !ok { + return fmt.Errorf("invalid OS disk SKU '%s'", *c.OSDiskSKU) + } + + if err := supportsDiskSKU(sku, *c.OSDiskSKU, c.Zones); err != nil { + return err + } + } + + if c.DataDiskSKU != nil { + if _, ok := dataDiskSKUs[*c.DataDiskSKU]; !ok { + return fmt.Errorf("invalid data disk SKU '%s'", *c.DataDiskSKU) + } + + // Ultra SSDs do not support availability sets, see for reference: + // https://docs.microsoft.com/en-us/azure/virtual-machines/disks-enable-ultra-ssd#ga-scope-and-limitations + if *c.DataDiskSKU == compute.StorageAccountTypesUltraSSDLRS && ((c.AssignAvailabilitySet != nil && *c.AssignAvailabilitySet) || c.AvailabilitySet != "") { + return fmt.Errorf("data disk SKU '%s' does not support availability sets", *c.DataDiskSKU) + } + + if err := supportsDiskSKU(sku, *c.DataDiskSKU, c.Zones); err != nil { + return err + } + } + } + + return nil +} + func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -1019,3 +1110,78 @@ func getOSUsername(os providerconfigtypes.OperatingSystem) string { return string(os) } } + +func storageTypePtr(storageType string) *compute.StorageAccountTypes { + storage := compute.StorageAccountTypes(storageType) + return &storage +} + +// supportsDiskSKU validates some disk SKU types against the chosen VM SKU / VM type. +func supportsDiskSKU(vmSKU compute.ResourceSku, diskSKU compute.StorageAccountTypes, zones []string) error { + // sanity check to make sure the Azure API did not return something bad + if vmSKU.Name == nil || vmSKU.Capabilities == nil { + return fmt.Errorf("invalid VM SKU object") + } + + switch diskSKU { + case compute.StorageAccountTypesPremiumLRS: + found := false + for _, capability := range *vmSKU.Capabilities { + if *capability.Name == CapabilityPremiumIO && *capability.Value == CapabilityValueTrue { + found = true + break + } + } + + if !found { + return fmt.Errorf("VM SKU '%s' does not support disk SKU '%s'", *vmSKU.Name, diskSKU) + } + + case compute.StorageAccountTypesUltraSSDLRS: + if vmSKU.LocationInfo == nil || len(*vmSKU.LocationInfo) == 0 || (*vmSKU.LocationInfo)[0].Zones == nil || len(*(*vmSKU.LocationInfo)[0].Zones) == 0 { + // no zone information found, let's check for capability + found := false + for _, capability := range *vmSKU.Capabilities { + if *capability.Name == CapabilityUltraSSD && *capability.Value == CapabilityValueTrue { + found = true + break + } + } + + if !found { + return fmt.Errorf("VM SKU '%s' does not support disk SKU '%s'", *vmSKU.Name, diskSKU) + } + } else { + if (*vmSKU.LocationInfo)[0].ZoneDetails != nil { + for _, zone := range zones { + found := false + for _, details := range *(*vmSKU.LocationInfo)[0].ZoneDetails { + matchesZone := false + for _, zoneName := range *details.Name { + if zone == zoneName { + matchesZone = true + break + } + } + + // we only check this zone details for capabilities if it actually includes the zone we're checking for + if matchesZone { + for _, capability := range *details.Capabilities { + if *capability.Name == CapabilityUltraSSD && *capability.Value == CapabilityValueTrue { + found = true + break + } + } + } + } + + if !found { + return fmt.Errorf("VM SKU '%s' does not support disk SKU '%s' in zone '%s'", *vmSKU.Name, diskSKU, zone) + } + } + } + } + } + + return nil +} diff --git a/pkg/cloudprovider/provider/azure/types/types.go b/pkg/cloudprovider/provider/azure/types/types.go index 7c64c9cd7..10f881d28 100644 --- a/pkg/cloudprovider/provider/azure/types/types.go +++ b/pkg/cloudprovider/provider/azure/types/types.go @@ -45,7 +45,9 @@ type RawConfig struct { ImageID providerconfigtypes.ConfigVarString `json:"imageID"` OSDiskSize int32 `json:"osDiskSize"` + OSDiskSKU *string `json:"osDiskSKU,omitempty"` DataDiskSize int32 `json:"dataDiskSize"` + DataDiskSKU *string `json:"dataDiskSKU,omitempty"` AssignPublicIP providerconfigtypes.ConfigVarBool `json:"assignPublicIP"` Tags map[string]string `json:"tags,omitempty"` } diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 16cd83c4f..517835836 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -612,6 +612,8 @@ func TestAzureProvisioningE2E(t *testing.T) { fmt.Sprintf("<< AZURE_SUBSCRIPTION_ID >>=%s", azureSubscriptionID), fmt.Sprintf("<< AZURE_CLIENT_ID >>=%s", azureClientID), fmt.Sprintf("<< AZURE_CLIENT_SECRET >>=%s", azureClientSecret), + fmt.Sprintf("<< AZURE_OS_DISK_SKU >>=%s", "Standard_LRS"), + fmt.Sprintf("<< AZURE_DATA_DISK_SKU >>=%s", "Standard_LRS"), } runScenarios(t, selector, params, AzureManifest, fmt.Sprintf("azure-%s", *testRunIdentifier)) } @@ -637,6 +639,8 @@ func TestAzureCustomImageReferenceProvisioningE2E(t *testing.T) { fmt.Sprintf("<< AZURE_SUBSCRIPTION_ID >>=%s", azureSubscriptionID), fmt.Sprintf("<< AZURE_CLIENT_ID >>=%s", azureClientID), fmt.Sprintf("<< AZURE_CLIENT_SECRET >>=%s", azureClientSecret), + fmt.Sprintf("<< AZURE_OS_DISK_SKU >>=%s", "Standard_LRS"), + fmt.Sprintf("<< AZURE_DATA_DISK_SKU >>=%s", "Standard_LRS"), } runScenarios(t, selector, params, AzureCustomImageReferenceManifest, fmt.Sprintf("azure-%s", *testRunIdentifier)) } @@ -662,6 +666,8 @@ func TestAzureRedhatSatelliteProvisioningE2E(t *testing.T) { fmt.Sprintf("<< AZURE_SUBSCRIPTION_ID >>=%s", azureSubscriptionID), fmt.Sprintf("<< AZURE_CLIENT_ID >>=%s", azureClientID), fmt.Sprintf("<< AZURE_CLIENT_SECRET >>=%s", azureClientSecret), + fmt.Sprintf("<< AZURE_OS_DISK_SKU >>=%s", "Standard_LRS"), + fmt.Sprintf("<< AZURE_DATA_DISK_SKU >>=%s", "Standard_LRS"), } scenario := scenario{ diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure.yaml index f17dd9f21..ea6a910d7 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure.yaml @@ -34,7 +34,9 @@ spec: vmSize: "Standard_F2" # optional disk size values in GB. If not set, the defaults for the vmSize will be used. osDiskSize: << OS_DISK_SIZE >> + osDiskSKU: << AZURE_OS_DISK_SKU >> dataDiskSize: << DATA_DISK_SIZE >> + dataDiskSKU: << AZURE_DATA_DISK_SKU >> vnetName: "machine-controller-e2e" subnetName: "machine-controller-e2e" routeTableName: "machine-controller-e2e" From 3d52b73cc6b8d0c62cb50495079b6a815e0c7ee5 Mon Sep 17 00:00:00 2001 From: Pratik Deoghare Date: Thu, 7 Apr 2022 14:28:49 +0530 Subject: [PATCH 113/489] add dual stack support for gcp (#1236) * add dual stack support for gcp * add boilerplate * fix spelling * use pod network family instead of pod cidrs * fix log msg * rename podNetworkFamily to networkFamily * validate each network family * handle unspecified and unknown cases of network family * add test case for empty network family * use network family from provider config only * capitalize error message --- go.mod | 33 ++-- go.sum | 180 +++++++++++++++-- .../provider/anexia/helper_test.go | 2 + pkg/cloudprovider/provider/anexia/provider.go | 13 +- .../provider/anexia/provider_test.go | 2 + .../provider/anexia/types/types.go | 7 +- pkg/cloudprovider/provider/gce/instance.go | 3 + pkg/cloudprovider/provider/gce/provider.go | 39 +++- .../provider/gce/provider_test.go | 183 ++++++++++++++++++ pkg/cloudprovider/provider/gce/service.go | 36 +++- .../provider/gce/types/cloudconfig_test.go | 2 +- .../provider/kubevirt/types/types.go | 1 + pkg/cloudprovider/util/net.go | 10 + pkg/node/poddeletion/pod_deletion.go | 1 + pkg/providerconfig/types/types.go | 15 +- 15 files changed, 474 insertions(+), 53 deletions(-) create mode 100644 pkg/cloudprovider/provider/gce/provider_test.go diff --git a/go.mod b/go.mod index 7fb3d9fd0..3eb2471e3 100644 --- a/go.mod +++ b/go.mod @@ -3,8 +3,8 @@ module github.com/kubermatic/machine-controller go 1.17 require ( - cloud.google.com/go v0.73.0 cloud.google.com/go/logging v1.1.2 + cloud.google.com/go/monitoring v1.4.0 github.com/Azure/azure-sdk-for-go v62.0.0+incompatible github.com/Azure/go-autorest/autorest/azure/auth v0.5.5 github.com/Azure/go-autorest/autorest/to v0.4.0 @@ -36,10 +36,10 @@ require ( github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda github.com/vmware/govmomi v0.23.1 golang.org/x/crypto v0.0.0-20211202192323-5770296d904e - golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c + golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a gomodules.xyz/jsonpatch/v2 v2.2.0 - google.golang.org/api v0.36.0 - google.golang.org/grpc v1.38.0 + google.golang.org/api v0.74.0 + google.golang.org/grpc v1.45.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b k8c.io/operating-system-manager v0.4.0 @@ -57,6 +57,8 @@ require ( ) require ( + cloud.google.com/go v0.100.2 // indirect + cloud.google.com/go/compute v1.5.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.18 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect @@ -91,10 +93,10 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/go-cmp v0.5.6 // indirect + github.com/google/go-cmp v0.5.7 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/googleapis/gax-go/v2 v2.0.5 // indirect + github.com/googleapis/gax-go/v2 v2.2.0 // indirect github.com/googleapis/gnostic v0.5.5 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect @@ -102,7 +104,6 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.11 // indirect - github.com/jstemmer/go-junit-report v0.9.1 // indirect github.com/kr/pretty v0.2.1 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect @@ -121,21 +122,17 @@ require ( github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50 // indirect - go.opencensus.io v0.22.5 // indirect + go.opencensus.io v0.23.0 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect - golang.org/x/mod v0.4.2 // indirect - golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect + golang.org/x/net v0.0.0-20220325170049-de3da57026de // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 // indirect - golang.org/x/term v0.0.0-20210503060354-a79de5458b56 // indirect - golang.org/x/text v0.3.6 // indirect + golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 // indirect + golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect + golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect - golang.org/x/tools v0.1.2 // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect - google.golang.org/protobuf v1.26.0 // indirect + google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb // indirect + google.golang.org/protobuf v1.27.1 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.57.0 // indirect diff --git a/go.sum b/go.sum index a078d09dd..c1ae4c8b5 100644 --- a/go.sum +++ b/go.sum @@ -19,20 +19,39 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.71.0/go.mod h1:qZfY4Y7AEIQwG/fQYD3xrxLNkQZ0Xzf3HGeqCkA6LVM= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.73.0 h1:sGvc4e0Cmm4+DKQR76a9VwNukpacQK8TOl5pDl0Pcn0= cloud.google.com/go v0.73.0/go.mod h1:BkDh9dFvGjCitVw03TNjKbBxXNKULXXIq6orU6HrJ4Q= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.2 h1:t9Iw5QH5v4XtlEQaCtUY7x6sCABps8sW0acw7e2WQ6Y= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0 h1:b1zWmYuuHz7gO9kDcM/EpHGr06UgsYNRpNJzI2kFiLM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/logging v1.0.0/go.mod h1:V1cc3ogwobYzQq5f2R7DS/GvRIrI4FKj01Gs5glwAls= cloud.google.com/go/logging v1.1.2 h1:KNALX0NZn8UJhqKnqoHxhMqyoZfBZoh5wF7CQJZ5XrU= cloud.google.com/go/logging v1.1.2/go.mod h1:KrljuAHIw631j9+QXsnq9vDwsrwmdxfGpivMR68M7DY= +cloud.google.com/go/monitoring v1.4.0 h1:05+IuNMbh40hbxcqQ4SnynbwZbLG1Wc9dysIJxnfv7U= +cloud.google.com/go/monitoring v1.4.0/go.mod h1:y6xnxfwI3hTFWOdkOaD7nfJVlwuC3/mS/5kvtT131p4= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -243,6 +262,11 @@ github.com/cloudflare/cloudflare-go v0.13.2/go.mod h1:27kfc1apuifUmJhp069y0+hwlK github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= @@ -347,6 +371,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= github.com/etcd-io/gofail v0.0.0-20190801230047-ad7f989257ca/go.mod h1:49H/RkXP8pKaZy4h0d+NW16rSLhyVBt4o6VLJbmOqDE= @@ -590,6 +616,8 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v0.0.0-20181025225059-d3de96c4c28e/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -612,6 +640,7 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= @@ -628,10 +657,12 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-containerregistry v0.0.0-20200115214256-379933c9c22b/go.mod h1:Wtl/v6YdQxv397EREtzwgd9+Ud7Q5D8XMbi3Zazgkrs= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-licenses v0.0.0-20191112164736-212ea350c932/go.mod h1:16wa6pRqNDUIhOtwF0GcROVqMeXHZJ7H6eGDFUh5Pfk= @@ -646,6 +677,7 @@ github.com/google/licenseclassifier v0.0.0-20190926221455-842c0d70d702/go.mod h1 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -656,6 +688,12 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201117184057-ae444373da19/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v0.0.0-20170306145142-6a5e28554805/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -667,8 +705,11 @@ github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0 h1:s7jOdKSaksJVOxE0Y/S32otcfiP+UQ0cL8/GTKaONwE= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= @@ -816,7 +857,6 @@ github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMW github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jsonnet-bundler/jsonnet-bundler v0.1.0/go.mod h1:YKsSFc9VFhhLITkJS3X2PrRqWG9u2Jq99udTdDjQLfM= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -1321,8 +1361,9 @@ go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= @@ -1429,6 +1470,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= @@ -1441,7 +1483,7 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1499,12 +1541,20 @@ golang.org/x/net v0.0.0-20201026091529-146b70c837a4/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de h1:pZB1TWnKi+o4bENlbzAgLrEbY4RMYmUIRobMcSmfeYc= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1514,8 +1564,18 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c h1:pkQiBZBvdos9qq4wBAHqlzuZHEXo07pqV06ef90u1WI= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a h1:qfl7ob3DIEs3Ml9oLuPwY2N04gymzAW04WsUQHIClgM= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1610,25 +1670,44 @@ golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 h1:c8PlLMqBbOHoqtjteWm5/kbe6rNY2pbRfbIMVnepueo= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 h1:eJv7u3ksNXoLbGSKuv2s/SIO4tJVxc/A+MTpzxDgz/Q= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210503060354-a79de5458b56 h1:b8jxX3zqjpqb2LklXPzKSGJhzyxCOZSz8ncv8Nv+y7w= golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1638,8 +1717,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1732,11 +1812,17 @@ golang.org/x/tools v0.0.0-20201105220310-78b158585360/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.2 h1:kRBLX7v7Af8W7Gdbbc908OJcdgtK8bOz9Uaj8/F1ACA= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1772,8 +1858,25 @@ google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSr google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.34.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0 h1:l2Nfbl2GPXdWorv+dT2XfinX2jOOw4zv1VhLstx+6rE= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0 h1:ExR2D+5TYIrMphWgs5JCgwRhEDlPDXXrLwHHMgPHTXE= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1834,8 +1937,41 @@ google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201203001206-6486ece9c497/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb h1:0m9wktIpOxGw+SSKmydXWB3Z3GTfcPP6+q75HCQa6HI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= google.golang.org/grpc v1.13.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= @@ -1863,10 +1999,21 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1878,8 +2025,9 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= diff --git a/pkg/cloudprovider/provider/anexia/helper_test.go b/pkg/cloudprovider/provider/anexia/helper_test.go index 2edc3ba32..adc1b514e 100644 --- a/pkg/cloudprovider/provider/anexia/helper_test.go +++ b/pkg/cloudprovider/provider/anexia/helper_test.go @@ -23,9 +23,11 @@ import ( "github.com/anexia-it/go-anxcloud/pkg/vsphere/search" "github.com/gophercloud/gophercloud/testhelper" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8s.io/apimachinery/pkg/runtime" ) diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 019f57a92..22b8104b1 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -25,16 +25,10 @@ import ( "net/http" "time" - "github.com/anexia-it/go-anxcloud/pkg/vsphere/provisioning/progress" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/utils" - "k8s.io/apimachinery/pkg/api/meta" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog" - anxclient "github.com/anexia-it/go-anxcloud/pkg/client" anxaddr "github.com/anexia-it/go-anxcloud/pkg/ipam/address" "github.com/anexia-it/go-anxcloud/pkg/vsphere" + "github.com/anexia-it/go-anxcloud/pkg/vsphere/provisioning/progress" anxvm "github.com/anexia-it/go-anxcloud/pkg/vsphere/provisioning/vm" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" @@ -43,12 +37,17 @@ import ( cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/utils" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8s.io/apimachinery/pkg/api/meta" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" ) const ( diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index 4fe746704..bb23fb5ce 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -28,11 +28,13 @@ import ( "github.com/anexia-it/go-anxcloud/pkg/vsphere/provisioning/progress" "github.com/anexia-it/go-anxcloud/pkg/vsphere/provisioning/vm" "github.com/gophercloud/gophercloud/testhelper" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/utils" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/cloudprovider/provider/anexia/types/types.go b/pkg/cloudprovider/provider/anexia/types/types.go index 1046315fb..dd0faca84 100644 --- a/pkg/cloudprovider/provider/anexia/types/types.go +++ b/pkg/cloudprovider/provider/anexia/types/types.go @@ -17,13 +17,14 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "time" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" + cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( diff --git a/pkg/cloudprovider/provider/gce/instance.go b/pkg/cloudprovider/provider/gce/instance.go index 3426e2193..f53967ae4 100644 --- a/pkg/cloudprovider/provider/gce/instance.go +++ b/pkg/cloudprovider/provider/gce/instance.go @@ -68,6 +68,9 @@ func (gi *googleInstance) Addresses() map[string]v1.NodeAddressType { for _, ac := range ifc.AccessConfigs { addrs[ac.NatIP] = v1.NodeExternalIP } + for _, ac := range ifc.Ipv6AccessConfigs { + addrs[ac.ExternalIpv6] = v1.NodeExternalIP + } } // GCE has two types of the internal DNS, so we need to take both diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index c8b584932..61f80259b 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -24,10 +24,11 @@ import ( "fmt" "net/http" "strconv" + "strings" "cloud.google.com/go/logging" monitoring "cloud.google.com/go/monitoring/apiv3" - "google.golang.org/api/compute/v1" + compute "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" @@ -36,6 +37,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" gcetypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/gce/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/providerconfig" "k8s.io/apimachinery/pkg/types" @@ -47,6 +49,9 @@ const ( errOperatingSystem = "Invalid or not supported operating system specified %q: %v" errConnect = "Failed to connect: %v" errInvalidServiceAccount = "Service account is missing" + errIPv6UnsupportedZone = "IPv6 is not supported in zone: %s" + errUnknownNetworkFamily = "Unknown network family only IPv4,IPv6,IPv4+IPv6 are valid values, got: %q" + errIPv6OnlyUnsupported = "IPv6 only network family not supported yet" errInvalidZone = "Zone is missing" errInvalidMachineType = "Machine type is missing" errInvalidDiskSize = "Disk size must be a positive number" @@ -112,6 +117,20 @@ func (p *Provider) Validate(spec clusterv1alpha1.MachineSpec) error { if cfg.zone == "" { return newError(common.InvalidConfigurationMachineError, errInvalidZone) } + + switch cfg.providerConfig.Network.GetNetworkFamily() { + case util.Unspecified, util.IPv4: + // noop + case util.IPv6: + return newError(common.InvalidConfigurationMachineError, errIPv6OnlyUnsupported) + case util.DualStack: + if !isIPv6Supported(cfg.zone) { + return newError(common.InvalidConfigurationMachineError, errIPv6UnsupportedZone, cfg.zone) + } + default: + return newError(common.InvalidConfigurationMachineError, errUnknownNetworkFamily, cfg.providerConfig.Network.GetNetworkFamily()) + } + if cfg.machineType == "" { return newError(common.InvalidConfigurationMachineError, errInvalidMachineType) } @@ -128,6 +147,24 @@ func (p *Provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } +func isIPv6Supported(zone string) bool { + supportedRegions := []string{ + "asia-east1", + "asia-south1", + "europe-west2", + "us-west2", + } + + for _, region := range supportedRegions { + // this is fine since zones are constructed from region + zone suffix + if strings.HasPrefix(zone, region) { + return true + } + } + + return false +} + // Get retrieves a node instance that is associated with the given machine. func (p *Provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { return p.get(machine) diff --git a/pkg/cloudprovider/provider/gce/provider_test.go b/pkg/cloudprovider/provider/gce/provider_test.go new file mode 100644 index 000000000..f19965bfe --- /dev/null +++ b/pkg/cloudprovider/provider/gce/provider_test.go @@ -0,0 +1,183 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gce + +import ( + "context" + "encoding/base64" + "encoding/json" + "os" + "strings" + "testing" + + "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "github.com/kubermatic/machine-controller/pkg/providerconfig" + + "k8s.io/apimachinery/pkg/runtime" + fake2 "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func testProviderSpec() map[string]interface{} { + return map[string]interface{}{ + "caPublicKey": "", + "cloudProvider": "gce", + "cloudProviderSpec": map[string]interface{}{ + "assignPublicIPAddress": true, + "customImage": "", + "diskSize": 25, + "diskType": "pd-standard", + "machineType": "e2-highcpu-2", + "multizone": false, + "network": "global/networks/default", + "preemptible": false, + "regional": false, + "serviceAccount": "", + "subnetwork": "", + "tags": []string{ + "kubernetes-cluster-kdlj8sn58d", + "system-cluster-kdlj8sn58d", + "system-project-sszxpzjcnm", + }, + "zone": "europe-west2-a", + }, + "operatingSystem": "ubuntu", + "operatingSystemSpec": map[string]interface{}{ + "distUpgradeOnBoot": false, + }, + "sshPublicKeys": []string{}, + "network": map[string]interface{}{}, + } +} + +func testServiceAccount() string { + return base64.StdEncoding.EncodeToString([]byte(`{ + "type": "service_account", + "project_id": "test-dev", + "private_key_id": "testprivatekeyid", + "private_key": "-----BEGIN PRIVATE KEY-----\n\nMIICXQIBAAKBgQCU6DqY/hqXOOjomOuf3ESiMBxxRCpnn+VTAOPeEOsKaNdAv3zB\n\nmy6KEgOlraAi45cR8Ow8R0UFBNMQaU6Bck99t34BGZSQxjMTFw11W9p0GROKZgqG\n\nobj1WiomRQuwy0D6Q90wRSRhvnawKHqIEDoGnQT+SceV5vb6yoLmZSBoFQIDAQAB\n\nAoGASQoIBBdPz7E4fS7VFJqkh7F1ohE/g4iooagkHT7LK1X1j2rdtNF7aHohk9iw\n\nXayo40H7fi2vKyEMrlYZDeGWH1/XHLIyTGUNo91J3HDRbfs0eJhHKxFsdD/a64yV\n\ndYJviM2nsBQkbCC08O3yVCc/0spB7xKSBlpgFaWTnwDj8AECQQDnjen0Or9C7c9N\n\nOQdkefGoRzD0ltwbJoOHmRz3s49TieRmQpX+XkcbR91BPkIzgbQs8tFatK5YQNDp\n\nDTdp/VoVAkEApKCoEv6hNdj3sjY1qGT2e2sNCKbgsJeXfPrMbotypmv2VgK4w0IE\n\nPA+Tysd6G3EojFooDlzAkG2hXsgie2BWAQJBAN/finnSLsdD63CrGaWgbO+Y3REt\n\npmMtqm94rtQiLAnFwSjJagHEHxWWNqn0ysbHuW7X2WfMVuAG0rTwTUpRZD0CQQCQ\n\nhY0nJ6vkdrV0GIzgaMnNLPxDNSSZQms1x4JCJV8f5DVb6oXCvCi1hUNMR/PVNXDQ\n\nTbFOcnSGFggNCgrjXn4BAkAgoDpFUVa5wLvkWQpTnKXv//xMG4fS3xmlDHi3xE8d\n\nMfEPCgKd8giHPaW0p4XtTAmhk1sdpuR2op4ZfDorCmEC\n\n-----END PRIVATE KEY-----\n", + "client_email": "someguy@some.com", + "client_id": "whateverthisis", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/sometest" +} +`)) +} + +type testMap map[string]interface{} + +// with patches value of m at keypath with val e.g. keypath=x.y val=z then m[x][y] = z +func (m testMap) with(keypath, val string) testMap { + parts := strings.Split(keypath, ".") + var curr interface{} = m + for _, p := range parts[:len(parts)-1] { + switch m[p].(type) { + case map[string]interface{}, testMap: + curr = m[p] + } + } + + switch x := curr.(type) { + case map[string]interface{}: //, testMap: + x[parts[len(parts)-1]] = val + case testMap: + x[parts[len(parts)-1]] = val + } + return m +} + +func TestValidate(t *testing.T) { + os.Setenv(envGoogleServiceAccount, testServiceAccount()) + defer os.Unsetenv(envGoogleServiceAccount) + + rawBytes := func(m map[string]interface{}) []byte { + data, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + return data + } + + p := New(providerconfig.NewConfigVarResolver(context.Background(), fake2.NewClientBuilder().Build())) + tests := []struct { + name string + mspec v1alpha1.MachineSpec + expectErr bool + }{ + { + "without network family", + v1alpha1.MachineSpec{ + ProviderSpec: v1alpha1.ProviderSpec{ + Value: &runtime.RawExtension{ + Raw: rawBytes(testProviderSpec()), + }, + }, + }, + false, + }, + { + "empty network family", + v1alpha1.MachineSpec{ + ProviderSpec: v1alpha1.ProviderSpec{ + Value: &runtime.RawExtension{ + Raw: rawBytes(testMap(testProviderSpec()). + with("network.networkFamily", ""), + ), + }, + }, + }, + false, + }, + { + "with network family", + v1alpha1.MachineSpec{ + ProviderSpec: v1alpha1.ProviderSpec{ + Value: &runtime.RawExtension{ + Raw: rawBytes(testMap(testProviderSpec()). + with("network.networkFamily", "IPv4+IPv6"), + ), + }, + }, + }, + false, + }, + { + "with unsupported zone", + v1alpha1.MachineSpec{ + ProviderSpec: v1alpha1.ProviderSpec{ + Value: &runtime.RawExtension{ + Raw: rawBytes(testMap(testProviderSpec()). + with("network.networkFamily", "IPv4+IPv6"). + with("cloudProviderSpec.zone", "europe-west3-a"), + ), + }, + }, + }, + true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := p.Validate(test.mspec) + if (err != nil) != test.expectErr { + t.Fatalf("expectedErr: %t, got: %v", test.expectErr, err) + } + }) + } +} diff --git a/pkg/cloudprovider/provider/gce/service.go b/pkg/cloudprovider/provider/gce/service.go index c27b485b5..f62da9716 100644 --- a/pkg/cloudprovider/provider/gce/service.go +++ b/pkg/cloudprovider/provider/gce/service.go @@ -27,7 +27,10 @@ import ( "golang.org/x/oauth2" "google.golang.org/api/compute/v1" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/klog" ) const ( @@ -72,11 +75,36 @@ func (svc *service) networkInterfaces(cfg *config) ([]*compute.NetworkInterface, Subnetwork: cfg.subnetwork, } + klog.Infof("using network:%s subnetwork: %s", cfg.network, cfg.subnetwork) + if cfg.assignPublicIPAddress { - ifc.AccessConfigs = []*compute.AccessConfig{{ - Name: "External NAT", - Type: "ONE_TO_ONE_NAT", - }} + ifc.AccessConfigs = []*compute.AccessConfig{ + { + Name: "External NAT", + Type: "ONE_TO_ONE_NAT", + }, + } + } + + // Setup IPv6 + // GCP allocates public IPv6 addr so we only try to setup IPv6 + // if assigning public IP addresses is enabled. + if cfg.assignPublicIPAddress { + // GCP doesn't support IPv6 only stack + if cfg.providerConfig.Network.GetNetworkFamily() == util.DualStack { + ifc.StackType = "IPV4_IPV6" + + ifc.Ipv6AccessConfigs = []*compute.AccessConfig{ + { + Name: "external-ipv6", + NetworkTier: "PREMIUM", + Type: "DIRECT_IPV6", + }, + } + } else { + klog.Infof("network family doesn't specify dual stack: %s", cfg.providerConfig.Network.GetNetworkFamily()) + } + } return []*compute.NetworkInterface{ifc}, nil diff --git a/pkg/cloudprovider/provider/gce/types/cloudconfig_test.go b/pkg/cloudprovider/provider/gce/types/cloudconfig_test.go index f7646002e..6b91cefd5 100644 --- a/pkg/cloudprovider/provider/gce/types/cloudconfig_test.go +++ b/pkg/cloudprovider/provider/gce/types/cloudconfig_test.go @@ -66,7 +66,7 @@ func TestCloudConfigAsString(t *testing.T) { t.Fatalf("failed to convert to string: %v", err) } if s != test.contents { - t.Fatalf("output is not as expected") + t.Fatalf("output is not as expected: %s", s) } }) } diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index a2fff0ba6..bd03203d1 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -19,6 +19,7 @@ package types import ( "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + corev1 "k8s.io/api/core/v1" ) diff --git a/pkg/cloudprovider/util/net.go b/pkg/cloudprovider/util/net.go index ec436bedb..bb9fd140e 100644 --- a/pkg/cloudprovider/util/net.go +++ b/pkg/cloudprovider/util/net.go @@ -54,3 +54,13 @@ func GenerateRandMAC() (net.HardwareAddr, error) { return mac, nil } + +// NetworkFamily IPv4 | IPv6 | IPv4+IPv6 +type NetworkFamily string + +const ( + Unspecified NetworkFamily = "" // interpreted as IPv4 + IPv4 NetworkFamily = "IPv4" + IPv6 NetworkFamily = "IPv6" + DualStack NetworkFamily = "IPv4+IPv6" +) diff --git a/pkg/node/poddeletion/pod_deletion.go b/pkg/node/poddeletion/pod_deletion.go index 1b9874aa8..9781b8864 100644 --- a/pkg/node/poddeletion/pod_deletion.go +++ b/pkg/node/poddeletion/pod_deletion.go @@ -22,6 +22,7 @@ import ( "sync" "github.com/kubermatic/machine-controller/pkg/node/nodemanager" + corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 37f75ff24..5fa134a22 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -23,6 +23,7 @@ import ( "fmt" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/jsonutil" corev1 "k8s.io/api/core/v1" @@ -108,9 +109,17 @@ type DNSConfig struct { // NetworkConfig contains a machine's static network configuration type NetworkConfig struct { - CIDR string `json:"cidr"` - Gateway string `json:"gateway"` - DNS DNSConfig `json:"dns"` + CIDR string `json:"cidr"` + Gateway string `json:"gateway"` + DNS DNSConfig `json:"dns"` + NetworkFamily util.NetworkFamily `json:"networkFamily,omitempty"` +} + +func (n *NetworkConfig) GetNetworkFamily() util.NetworkFamily { + if n == nil { + return util.Unspecified + } + return n.NetworkFamily } type Config struct { From 1d62c1832a3f1f7aa3a3f8b374cabf5eb9585946 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Thu, 7 Apr 2022 12:50:17 +0200 Subject: [PATCH 114/489] Support rockylinux in hetzner (#1243) * support rockylinux on Hetzner cloud Signed-off-by: Moath Qasim * update fixtures Signed-off-by: Moath Qasim --- pkg/cloudprovider/provider/hetzner/provider.go | 2 ++ pkg/userdata/rockylinux/provider.go | 1 + .../rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml | 1 + pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml | 1 + .../rockylinux/testdata/kubelet-v1.21-aws-external.yaml | 1 + pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml | 1 + pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml | 1 + .../rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml | 1 + .../rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml | 1 + pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml | 1 + pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml | 1 + pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml | 1 + 12 files changed, 13 insertions(+) diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 382da7c8b..70de29068 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -73,6 +73,8 @@ func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { return "ubuntu-20.04", nil case providerconfigtypes.OperatingSystemCentOS: return "centos-7", nil + case providerconfigtypes.OperatingSystemRockyLinux: + return "rocky-8", nil } return "", providerconfigtypes.ErrOSNotSupported } diff --git a/pkg/userdata/rockylinux/provider.go b/pkg/userdata/rockylinux/provider.go index cc9671f74..d78cfc449 100644 --- a/pkg/userdata/rockylinux/provider.go +++ b/pkg/userdata/rockylinux/provider.go @@ -216,6 +216,7 @@ write_files: socat \ wget \ curl \ + tar \ {{- if eq .CloudProviderName "vsphere" }} open-vm-tools \ {{- end }} diff --git a/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml index f77f30de9..79fe88184 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml @@ -78,6 +78,7 @@ write_files: socat \ wget \ curl \ + tar \ ipvsadm yum install -y yum-utils diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml index 2ceaddece..53065f046 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml @@ -78,6 +78,7 @@ write_files: socat \ wget \ curl \ + tar \ ipvsadm yum install -y yum-utils diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml index e7d6e4f1a..c8ad36585 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml @@ -78,6 +78,7 @@ write_files: socat \ wget \ curl \ + tar \ ipvsadm yum install -y yum-utils diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml index aca5b1cc1..8e146adca 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml @@ -78,6 +78,7 @@ write_files: socat \ wget \ curl \ + tar \ ipvsadm yum install -y yum-utils diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml index 32615629c..f47d681e2 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml @@ -82,6 +82,7 @@ write_files: socat \ wget \ curl \ + tar \ iscsi-initiator-utils \ ipvsadm systemctl enable --now iscsid diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 5d7056570..da9ebe7b2 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -90,6 +90,7 @@ write_files: socat \ wget \ curl \ + tar \ open-vm-tools \ ipvsadm diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml index b7bbcad16..08e5327bd 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -90,6 +90,7 @@ write_files: socat \ wget \ curl \ + tar \ open-vm-tools \ ipvsadm diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml index a13fb8cdd..ac0a25e90 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml @@ -82,6 +82,7 @@ write_files: socat \ wget \ curl \ + tar \ open-vm-tools \ ipvsadm diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml index 34783e6e8..4b6609ff9 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml @@ -78,6 +78,7 @@ write_files: socat \ wget \ curl \ + tar \ ipvsadm yum install -y yum-utils diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml index f650bcb1f..f4712aa93 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml @@ -78,6 +78,7 @@ write_files: socat \ wget \ curl \ + tar \ ipvsadm yum install -y yum-utils From 4840e16f6d81fe865b9352f1614a1e1dd6fd43a1 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Thu, 7 Apr 2022 19:59:39 +0200 Subject: [PATCH 115/489] support rocky linux for DO (#1245) Signed-off-by: Moath Qasim --- pkg/cloudprovider/provider/digitalocean/provider.go | 2 ++ test/e2e/provisioning/all_e2e_test.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index 14e6ce7a3..4c66c16ac 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -87,6 +87,8 @@ func getSlugForOS(os providerconfigtypes.OperatingSystem) (string, error) { return "ubuntu-20-04-x64", nil case providerconfigtypes.OperatingSystemCentOS: return "centos-7-x64", nil + case providerconfigtypes.OperatingSystemRockyLinux: + return "rockylinux-8-x64", nil } return "", providerconfigtypes.ErrOSNotSupported } diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 517835836..18921e0d2 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -375,7 +375,7 @@ func TestDigitalOceanProvisioningE2E(t *testing.T) { t.Fatal("unable to run the test suite, DO_E2E_TESTS_TOKEN environment variable cannot be empty") } - selector := OsSelector("ubuntu", "centos") + selector := OsSelector("ubuntu", "centos", "rockylinux") // act params := []string{fmt.Sprintf("<< DIGITALOCEAN_TOKEN >>=%s", doToken)} From c2306b54a33296e5ce341a525dc2ed0358612a51 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Fri, 8 Apr 2022 00:52:51 +0200 Subject: [PATCH 116/489] Support Rocky linux for KubeVirt (#1244) * support Rocky linux for KubeVirt Signed-off-by: Moath Qasim * increase PVC size Signed-off-by: Moath Qasim --- pkg/cloudprovider/provider/kubevirt/provider.go | 9 +++++---- test/e2e/provisioning/all_e2e_test.go | 4 ++-- .../testdata/machinedeployment-kubevirt.yaml | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 75f5ecfec..5cae3d223 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -68,10 +68,11 @@ const ( ) var supportedOS = map[providerconfigtypes.OperatingSystem]*struct{}{ - providerconfigtypes.OperatingSystemCentOS: nil, - providerconfigtypes.OperatingSystemUbuntu: nil, - providerconfigtypes.OperatingSystemRHEL: nil, - providerconfigtypes.OperatingSystemFlatcar: nil, + providerconfigtypes.OperatingSystemCentOS: nil, + providerconfigtypes.OperatingSystemUbuntu: nil, + providerconfigtypes.OperatingSystemRHEL: nil, + providerconfigtypes.OperatingSystemFlatcar: nil, + providerconfigtypes.OperatingSystemRockyLinux: nil, } type provider struct { diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 18921e0d2..a4c76c2af 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -287,7 +287,7 @@ func TestKubevirtProvisioningE2E(t *testing.T) { t.Fatalf("Unable to run kubevirt tests, KUBEVIRT_E2E_TESTS_KUBECONFIG must be set") } - selector := OsSelector("ubuntu", "centos", "flatcar") + selector := OsSelector("ubuntu", "centos", "flatcar", "rockylinux") params := []string{ fmt.Sprintf("<< KUBECONFIG >>=%s", kubevirtKubeconfig), @@ -712,7 +712,7 @@ func TestHetznerProvisioningE2E(t *testing.T) { t.Fatal("unable to run the test suite, HZ_E2E_TOKEN environment variable cannot be empty") } - selector := OsSelector("ubuntu", "centos") + selector := OsSelector("ubuntu", "centos", "rockylinux") // act params := []string{fmt.Sprintf("<< HETZNER_TOKEN >>=%s", hzToken)} diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index 23bb843f3..f22f8be4c 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -35,7 +35,7 @@ spec: memory: "4096M" primaryDisk: osImage: http://10.107.208.71/<< OS_NAME >>.img - size: "10Gi" + size: "25Gi" storageClassName: local-path dnsPolicy: "None" dnsConfig: From b6567d9a57c5d88984401367c2e952db828ac9e3 Mon Sep 17 00:00:00 2001 From: Helene Durand <90329641+hdurand0710@users.noreply.github.com> Date: Fri, 8 Apr 2022 13:50:55 +0200 Subject: [PATCH 117/489] KubeVirt restart kubelet at reboot (#1246) * KubeVirt restart kubelet at reboot Signed-off-by: Helene Durand * refactor: script to restart kubelet updated restart-kubelet script has been updated to ensure that we are not restarting kubelet on the first boot Signed-off-by: Waleed Malik * Ran update-fixtures.sh Signed-off-by: Helene Durand Co-authored-by: Waleed Malik --- pkg/userdata/centos/provider.go | 39 ++++++++++++++++ pkg/userdata/flatcar/provider.go | 79 ++++++++++++++++++++++++++++++++ pkg/userdata/rhel/provider.go | 39 ++++++++++++++++ pkg/userdata/ubuntu/provider.go | 39 ++++++++++++++++ 4 files changed, 196 insertions(+) diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index 85f8b74d1..be8cef649 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -246,6 +246,9 @@ write_files: {{ end -}} systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + {{- if eq .CloudProviderName "kubevirt" }} + systemctl enable --now --no-block restart-kubelet.service + {{ end }} - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -334,6 +337,42 @@ write_files: append: true {{- end }} +{{- if eq .CloudProviderName "kubevirt" }} +- path: "/opt/bin/restart-kubelet.sh" + permissions: "0744" + content: | + #!/bin/bash + # Needed for Kubevirt provider because if the virt-launcher pod is deleted, + # the VM and DataVolume states are kept and VM is rebooted. We need to restart the kubelet + # with the new config (new IP) and run this at every boot. + set -xeuo pipefail + + # This helps us avoid an unnecessary restart for kubelet on the first boot + if [ -f /etc/kubelet_needs_restart ]; then + # restart kubelet since it's not the first boot + systemctl daemon-reload + systemctl restart kubelet.service + else + touch /etc/kubelet_needs_restart + fi + +- path: "/etc/systemd/system/restart-kubelet.service" + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + Description=Service responsible for restarting kubelet when the machine is rebooted + + [Service] + Type=oneshot + ExecStart=/opt/bin/restart-kubelet.sh + + [Install] + WantedBy=multi-user.target +{{- end }} + runcmd: - systemctl start setup.service ` diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index 85f4c3941..d87c5ff59 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -239,6 +239,24 @@ systemd: [Install] WantedBy=multi-user.target +{{- if eq .CloudProviderName "kubevirt" }} + - name: restart-kubelet.service + enabled: true + contents: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + Description=Service responsible for restarting kubelet when the machine is rebooted + + [Service] + Type=oneshot + ExecStart=/opt/bin/restart-kubelet.sh + + [Install] + WantedBy=multi-user.target +{{- end }} + - name: kubelet.service enabled: true dropins: @@ -349,6 +367,28 @@ storage: inline: '{{ .MachineSpec.Name }}' {{- end }} +{{- if eq .CloudProviderName "kubevirt" }} + - path: /opt/bin/restart-kubelet.sh + filesystem: root + mode: 0744 + contents: + inline: | + #!/bin/bash + # Needed for Kubevirt provider because if the virt-launcher pod is deleted, + # the VM and DataVolume states are kept and VM is rebooted. We need to restart the kubelet + # with the new config (new IP) and run this at every boot. + set -xeuo pipefail + + # This helps us avoid an unnecessary restart for kubelet on the first boot + if [ -f /etc/kubelet_needs_restart ]; then + # restart kubelet since it's not the first boot + systemctl daemon-reload + systemctl restart kubelet.service + else + touch /etc/kubelet_needs_restart + fi +{{- end }} + - path: /etc/ssh/sshd_config filesystem: root mode: 0600 @@ -537,6 +577,25 @@ coreos: [Install] WantedBy=multi-user.target +{{- if eq .CloudProviderName "kubevirt" }} + - name: restart-kubelet.service + enable: true + command: start + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + Description=Service responsible for restarting kubelet when the machine is rebooted + + [Service] + Type=oneshot + ExecStart=/opt/bin/restart-kubelet.sh + + [Install] + WantedBy=multi-user.target +{{- end }} + write_files: {{- if .HTTPProxy }} - path: /etc/environment @@ -662,4 +721,24 @@ write_files: user: root content: | runtime-endpoint: unix:///run/containerd/containerd.sock + +{{- if eq .CloudProviderName "kubevirt" }} +- path: "/opt/bin/restart-kubelet.sh" + permissions: "0744" + content: | + #!/bin/bash + # Needed for Kubevirt provider because if the virt-launcher pod is deleted, + # the VM and DataVolume states are kept and VM is rebooted. We need to restart the kubelet + # with the new config (new IP) and run this at every boot. + set -xeuo pipefail + + # This helps us avoid an unnecessary restart for kubelet on the first boot + if [ -f /etc/kubelet_needs_restart ]; then + # restart kubelet since it's not the first boot + systemctl daemon-reload + systemctl restart kubelet.service + else + touch /etc/kubelet_needs_restart + fi +{{- end }} ` diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 52ea94964..1337bc057 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -243,6 +243,9 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + {{- if eq .CloudProviderName "kubevirt" }} + systemctl enable --now --no-block restart-kubelet.service + {{ end }} - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -359,6 +362,42 @@ write_files: EnvironmentFile=-/etc/environment ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup +{{- if eq .CloudProviderName "kubevirt" }} +- path: "/opt/bin/restart-kubelet.sh" + permissions: "0744" + content: | + #!/bin/bash + # Needed for Kubevirt provider because if the virt-launcher pod is deleted, + # the VM and DataVolume states are kept and VM is rebooted. We need to restart the kubelet + # with the new config (new IP) and run this at every boot. + set -xeuo pipefail + + # This helps us avoid an unnecessary restart for kubelet on the first boot + if [ -f /etc/kubelet_needs_restart ]; then + # restart kubelet since it's not the first boot + systemctl daemon-reload + systemctl restart kubelet.service + else + touch /etc/kubelet_needs_restart + fi + +- path: "/etc/systemd/system/restart-kubelet.service" + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + Description=Service responsible for restarting kubelet when the machine is rebooted + + [Service] + Type=oneshot + ExecStart=/opt/bin/restart-kubelet.sh + + [Install] + WantedBy=multi-user.target +{{- end }} + rh_subscription: {{- if .OSConfig.RHELUseSatelliteServer }} org: "{{.OSConfig.RHELOrganizationName}}" diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index aa9399a78..4127cf7d8 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -238,6 +238,9 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + {{- if eq .CloudProviderName "kubevirt" }} + systemctl enable --now --no-block restart-kubelet.service + {{ end }} - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -329,6 +332,42 @@ write_files: append: true {{- end }} +{{- if eq .CloudProviderName "kubevirt" }} +- path: "/opt/bin/restart-kubelet.sh" + permissions: "0744" + content: | + #!/bin/bash + # Needed for Kubevirt provider because if the virt-launcher pod is deleted, + # the VM and DataVolume states are kept and VM is rebooted. We need to restart the kubelet + # with the new config (new IP) and run this at every boot. + set -xeuo pipefail + + # This helps us avoid an unnecessary restart for kubelet on the first boot + if [ -f /etc/kubelet_needs_restart ]; then + # restart kubelet since it's not the first boot + systemctl daemon-reload + systemctl restart kubelet.service + else + touch /etc/kubelet_needs_restart + fi + +- path: "/etc/systemd/system/restart-kubelet.service" + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + Description=Service responsible for restarting kubelet when the machine is rebooted + + [Service] + Type=oneshot + ExecStart=/opt/bin/restart-kubelet.sh + + [Install] + WantedBy=multi-user.target +{{- end }} + runcmd: - systemctl start setup.service ` From f09820656f8070f096d87fbe645197f097d8211e Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Fri, 8 Apr 2022 21:14:20 +0200 Subject: [PATCH 118/489] support rocky linux for OS (#1248) Signed-off-by: Moath Qasim --- test/e2e/provisioning/all_e2e_test.go | 2 +- test/e2e/provisioning/helper.go | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index a4c76c2af..8d0197fa1 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -321,7 +321,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { fmt.Sprintf("<< NETWORK_NAME >>=%s", osNetwork), } - selector := Not(OsSelector("sles", "rhel", "amzn2", "rockylinux")) + selector := Not(OsSelector("sles", "rhel", "amzn2")) runScenarios(t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index aa356318d..c49bc5f0b 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -50,10 +50,11 @@ var ( } openStackImages = map[string]string{ - string(providerconfigtypes.OperatingSystemUbuntu): "machine-controller-e2e-ubuntu-20-04", - string(providerconfigtypes.OperatingSystemCentOS): "machine-controller-e2e-centos", - string(providerconfigtypes.OperatingSystemRHEL): "machine-controller-e2e-rhel-8-5", - string(providerconfigtypes.OperatingSystemFlatcar): "machine-controller-e2e-flatcar-stable-2983", + string(providerconfigtypes.OperatingSystemUbuntu): "machine-controller-e2e-ubuntu-20-04", + string(providerconfigtypes.OperatingSystemCentOS): "machine-controller-e2e-centos", + string(providerconfigtypes.OperatingSystemRHEL): "machine-controller-e2e-rhel-8-5", + string(providerconfigtypes.OperatingSystemFlatcar): "machine-controller-e2e-flatcar-stable-2983", + string(providerconfigtypes.OperatingSystemRockyLinux): "machine-controller-e2e-rockylinux", } ) From 9cbcad48957826ee03fac375aba962478a7d31e8 Mon Sep 17 00:00:00 2001 From: Sankalp Rangare Date: Tue, 12 Apr 2022 11:19:51 +0200 Subject: [PATCH 119/489] add DataVolumeSourcePVC for KubeVirt (#1250) Signed-off-by: Sankalp Rangare --- .../provider/kubevirt/provider.go | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 5cae3d223..9dbcc6956 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -254,7 +254,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if rawConfig.VirtualMachine.DNSConfig != nil { config.DNSConfig = rawConfig.VirtualMachine.DNSConfig } - config.SecondaryDisks = make([]SecondaryDisks, len(rawConfig.VirtualMachine.Template.SecondaryDisks)) + config.SecondaryDisks = make([]SecondaryDisks, 0, len(rawConfig.VirtualMachine.Template.SecondaryDisks)) for _, sd := range rawConfig.VirtualMachine.Template.SecondaryDisks { sdSizeString, err := p.configVarResolver.GetConfigVarStringValue(sd.Size) @@ -698,6 +698,7 @@ func getVMVolumes(config *Config, dataVolumeName string, userDataSecretName stri } func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1.DataVolumeTemplateSpec { + dataVolumeSource := getDataVolumeSource(config.OsImage) pvcRequest := corev1.ResourceList{corev1.ResourceStorage: config.PVCSize} dataVolumeTemplates := []kubevirtv1.DataVolumeTemplateSpec{ { @@ -714,11 +715,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. Requests: pvcRequest, }, }, - Source: &cdiv1beta1.DataVolumeSource{ - HTTP: &cdiv1beta1.DataVolumeSourceHTTP{ - URL: config.OsImage.URL, - }, - }, + Source: dataVolumeSource, }, }, } @@ -737,17 +734,30 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. Requests: corev1.ResourceList{corev1.ResourceStorage: sd.Size}, }, }, - Source: &cdiv1beta1.DataVolumeSource{ - HTTP: &cdiv1beta1.DataVolumeSourceHTTP{ - URL: config.OsImage.URL, - }, - }, + Source: dataVolumeSource, }, }) } return dataVolumeTemplates } +// getDataVolumeSource returns DataVolumeSource, HTTP or PVC +func getDataVolumeSource(osImage OSImage) *cdiv1beta1.DataVolumeSource { + dataVolumeSource := &cdiv1beta1.DataVolumeSource{} + if osImage.URL != "" { + dataVolumeSource.HTTP = &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage.URL} + } else if osImage.DataVolumeName != "" { + if nameSpaceAndName := strings.Split(osImage.DataVolumeName, "/"); len(nameSpaceAndName) >= 2 { + dataVolumeSource.PVC = &cdiv1beta1.DataVolumeSourcePVC{ + Namespace: nameSpaceAndName[0], + Name: nameSpaceAndName[1], + } + } + + } + return dataVolumeSource +} + func getAffinity(config *Config, matchKey, matchValue string) *corev1.Affinity { affinity := &corev1.Affinity{} From 53281a5acde22967d1ab3f1f670d341d3dcb187b Mon Sep 17 00:00:00 2001 From: Pratik Deoghare Date: Wed, 13 Apr 2022 13:44:12 +0530 Subject: [PATCH 120/489] add dual stack support for AWS (#1242) * add dual stack support for AWS * validate network family, vpc * . --- pkg/cloudprovider/provider/aws/provider.go | 31 +++++++++++++++++++++- pkg/cloudprovider/provider/gce/provider.go | 3 +-- pkg/cloudprovider/util/net.go | 10 +++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index b41e8d186..9db3cdd01 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -41,6 +41,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" awstypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/aws/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" "github.com/kubermatic/machine-controller/pkg/userdata/convert" @@ -611,10 +612,22 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { } } - if _, err := getVpc(ec2Client, config.VpcID); err != nil { + vpc, err := getVpc(ec2Client, config.VpcID) + if err != nil { return fmt.Errorf("invalid vpc %q specified: %v", config.VpcID, err) } + switch f := pc.Network.GetNetworkFamily(); f { + case util.Unspecified, util.IPv4: + // noop + case util.IPv6, util.DualStack: + if len(vpc.Ipv6CidrBlockAssociationSet) == 0 { + return fmt.Errorf("vpc %q does not have IPv6 CIDR block", aws.StringValue(vpc.VpcId)) + } + default: + return fmt.Errorf(util.ErrUnknownNetworkFamily, f) + } + _, err = ec2Client.DescribeAvailabilityZones(&ec2.DescribeAvailabilityZonesInput{ZoneNames: aws.StringSlice([]string{config.AvailabilityZone})}) if err != nil { return fmt.Errorf("invalid zone %q specified: %v", config.AvailabilityZone, err) @@ -810,6 +823,10 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert }, } + if pc.Network.GetNetworkFamily() == util.IPv6 || pc.Network.GetNetworkFamily() == util.DualStack { + instanceRequest.NetworkInterfaces[0].Ipv6AddressCount = aws.Int64(1) + } + runOut, err := ec2Client.RunInstances(instanceRequest) if err != nil { return nil, awsErrorToTerminalError(err, "failed create instance at aws") @@ -1016,6 +1033,18 @@ func (d *awsInstance) Addresses() map[string]v1.NodeAddressType { aws.StringValue(d.instance.PrivateDnsName): v1.NodeInternalDNS, } + for _, netInterface := range d.instance.NetworkInterfaces { + for _, addr := range netInterface.Ipv6Addresses { + ipAddr := aws.StringValue(addr.Ipv6Address) + + // link-local addresses not very useful in machine status + // filter them out + if !util.IsLinkLocal(ipAddr) { + addresses[ipAddr] = v1.NodeExternalIP + } + } + } + delete(addresses, "") return addresses diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 61f80259b..c85f29b41 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -50,7 +50,6 @@ const ( errConnect = "Failed to connect: %v" errInvalidServiceAccount = "Service account is missing" errIPv6UnsupportedZone = "IPv6 is not supported in zone: %s" - errUnknownNetworkFamily = "Unknown network family only IPv4,IPv6,IPv4+IPv6 are valid values, got: %q" errIPv6OnlyUnsupported = "IPv6 only network family not supported yet" errInvalidZone = "Zone is missing" errInvalidMachineType = "Machine type is missing" @@ -128,7 +127,7 @@ func (p *Provider) Validate(spec clusterv1alpha1.MachineSpec) error { return newError(common.InvalidConfigurationMachineError, errIPv6UnsupportedZone, cfg.zone) } default: - return newError(common.InvalidConfigurationMachineError, errUnknownNetworkFamily, cfg.providerConfig.Network.GetNetworkFamily()) + return newError(common.InvalidConfigurationMachineError, util.ErrUnknownNetworkFamily, cfg.providerConfig.Network.GetNetworkFamily()) } if cfg.machineType == "" { diff --git a/pkg/cloudprovider/util/net.go b/pkg/cloudprovider/util/net.go index bb9fd140e..9b04758a9 100644 --- a/pkg/cloudprovider/util/net.go +++ b/pkg/cloudprovider/util/net.go @@ -23,6 +23,10 @@ import ( "net" ) +const ( + ErrUnknownNetworkFamily = "Unknown network family %q only IPv4,IPv6,IPv4+IPv6 are valid values" +) + func CIDRToIPAndNetMask(ipv4 string) (string, string, int, error) { ip, ipNet, err := net.ParseCIDR(ipv4) if err != nil { @@ -64,3 +68,9 @@ const ( IPv6 NetworkFamily = "IPv6" DualStack NetworkFamily = "IPv4+IPv6" ) + +// IsLinkLocal checks if given ip address is link local +func IsLinkLocal(ipAddr string) bool { + addr := net.ParseIP(ipAddr) + return addr.IsLinkLocalMulticast() || addr.IsLinkLocalUnicast() +} From efa60e35964c64fb7b1bf89f38b399c9e2ad7cd6 Mon Sep 17 00:00:00 2001 From: tlamr <34898768+tlamr@users.noreply.github.com> Date: Mon, 18 Apr 2022 13:38:27 +0200 Subject: [PATCH 121/489] make openstack provider work on ovh cloud (#1249) * make openstack provider work on ovh cloud Signed-off-by: Tomas Lamr * allow openstack#validate cyclo complexity 31 Signed-off-by: Tomas Lamr * code review changes Signed-off-by: Tomas Lamr Co-authored-by: Tomas Lamr --- .golangci.yml | 1 + pkg/cloudprovider/provider/openstack/provider.go | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 2531e4c67..9e18acff9 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -36,3 +36,4 @@ issues: - 'cyclomatic complexity 31 of func `verifyMigrateUID` is high' - 'cyclomatic complexity 31 of func `main` is high' - 'cyclomatic complexity 34 of func `\(\*provider\)\.getConfig` is high' + - 'cyclomatic complexity 31 of func `\(\*provider\)\.Validate` is high' diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index f86f119fb..b5dc24fe0 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -115,6 +115,7 @@ type Config struct { const ( machineUIDMetaKey = "machine-uid" securityGroupName = "kubernetes-v1" + ovhAuthURL = "auth.cloud.ovh.net" ) // Protects floating ip assignment @@ -480,8 +481,10 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { } // Required fields - if _, err := getRegion(client, c.Region); err != nil { - return fmt.Errorf("failed to get region %q: %v", c.Region, err) + if !strings.Contains(c.IdentityEndpoint, ovhAuthURL) { + if _, err := getRegion(client, c.Region); err != nil { + return fmt.Errorf("failed to get region %q: %v", c.Region, err) + } } // Get OS Compute Client From e08dc74062278feaa4be12125a8197a059ba285f Mon Sep 17 00:00:00 2001 From: Pratik Deoghare Date: Tue, 19 Apr 2022 15:49:42 +0530 Subject: [PATCH 122/489] add dual-stack support for Azure (#1253) * add dual-stack support for Azure * organize imports * ignore cyclomatic complexity of Create in Azure * yamllint fix * . * comment on standard sku usage * refactor interface names * reduce indentation --- .golangci.yml | 1 + .../provider/azure/create_delete_resources.go | 51 +++-- .../provider/azure/get_client.go | 2 +- pkg/cloudprovider/provider/azure/provider.go | 214 +++++++++++------- pkg/cloudprovider/provider/gce/provider.go | 3 +- pkg/cloudprovider/util/net.go | 1 + 6 files changed, 177 insertions(+), 95 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 9e18acff9..834cf8c47 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -37,3 +37,4 @@ issues: - 'cyclomatic complexity 31 of func `main` is high' - 'cyclomatic complexity 34 of func `\(\*provider\)\.getConfig` is high' - 'cyclomatic complexity 31 of func `\(\*provider\)\.Validate` is high' + - 'cyclomatic complexity 31 of func `\(\*provider\)\.Create` is high' diff --git a/pkg/cloudprovider/provider/azure/create_delete_resources.go b/pkg/cloudprovider/provider/azure/create_delete_resources.go index 01ee555f5..355b2509d 100644 --- a/pkg/cloudprovider/provider/azure/create_delete_resources.go +++ b/pkg/cloudprovider/provider/azure/create_delete_resources.go @@ -25,6 +25,8 @@ import ( "github.com/Azure/go-autorest/autorest/azure/auth" "github.com/Azure/go-autorest/autorest/to" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" + "k8s.io/apimachinery/pkg/types" "k8s.io/klog" ) @@ -47,7 +49,7 @@ func deleteInterfacesByMachineUID(ctx context.Context, c *config, machineUID typ for list.NotDone() { allInterfaces = append(allInterfaces, list.Values()...) - if err = list.Next(); err != nil { + if err = list.NextWithContext(ctx); err != nil { return fmt.Errorf("failed to iterate the result list: %s", err) } } @@ -192,7 +194,7 @@ func getDisksByMachineUID(ctx context.Context, disksClient *compute.DisksClient, return matchingDisks, nil } -func createOrUpdatePublicIPAddress(ctx context.Context, ipName string, machineUID types.UID, c *config) (*network.PublicIPAddress, error) { +func createOrUpdatePublicIPAddress(ctx context.Context, ipName string, ipVersion network.IPVersion, sku network.PublicIPAddressSkuName, ipAllocationMethod network.IPAllocationMethod, machineUID types.UID, c *config) (*network.PublicIPAddress, error) { klog.Infof("Creating public IP %q", ipName) ipClient, err := getIPClient(c) if err != nil { @@ -203,12 +205,16 @@ func createOrUpdatePublicIPAddress(ctx context.Context, ipName string, machineUI Name: to.StringPtr(ipName), Location: to.StringPtr(c.Location), PublicIPAddressPropertiesFormat: &network.PublicIPAddressPropertiesFormat{ - PublicIPAddressVersion: network.IPVersionIPv4, - PublicIPAllocationMethod: network.IPAllocationMethodStatic, + PublicIPAddressVersion: ipVersion, + PublicIPAllocationMethod: ipAllocationMethod, }, Tags: map[string]*string{machineUIDTag: to.StringPtr(string(machineUID))}, Zones: &c.Zones, + Sku: &network.PublicIPAddressSku{ + Name: sku, + }, } + future, err := ipClient.CreateOrUpdate(ctx, c.ResourceGroup, ipName, ipParams) if err != nil { return nil, fmt.Errorf("failed to create public IP address: %v", err) @@ -313,7 +319,7 @@ func getVirtualNetwork(ctx context.Context, c *config) (network.VirtualNetwork, return virtualNetworksClient.Get(ctx, c.VNetResourceGroup, c.VNetName, "") } -func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineUID types.UID, config *config, publicIP *network.PublicIPAddress) (*network.Interface, error) { +func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineUID types.UID, config *config, publicIP, publicIPv6 *network.PublicIPAddress, netFamily util.NetworkFamily) (*network.Interface, error) { ifClient, err := getInterfacesClient(config) if err != nil { return nil, fmt.Errorf("failed to create interfaces client: %v", err) @@ -328,19 +334,34 @@ func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineU Name: to.StringPtr(ifName), Location: &config.Location, InterfacePropertiesFormat: &network.InterfacePropertiesFormat{ - IPConfigurations: &[]network.InterfaceIPConfiguration{ - { - Name: to.StringPtr("ip-config-1"), - InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ - Subnet: &subnet, - PrivateIPAllocationMethod: network.IPAllocationMethodDynamic, - PublicIPAddress: publicIP, - }, - }, - }, + IPConfigurations: &[]network.InterfaceIPConfiguration{}, }, Tags: map[string]*string{machineUIDTag: to.StringPtr(string(machineUID))}, } + + *ifSpec.InterfacePropertiesFormat.IPConfigurations = append(*ifSpec.InterfacePropertiesFormat.IPConfigurations, network.InterfaceIPConfiguration{ + Name: to.StringPtr("ip-config-1"), + InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ + Subnet: &subnet, + PrivateIPAllocationMethod: network.IPAllocationMethodDynamic, + PublicIPAddress: publicIP, + Primary: to.BoolPtr(true), + }, + }) + + if netFamily == util.DualStack { + *ifSpec.InterfacePropertiesFormat.IPConfigurations = append(*ifSpec.InterfacePropertiesFormat.IPConfigurations, network.InterfaceIPConfiguration{ + Name: to.StringPtr("ip-config-2"), + InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ + PrivateIPAllocationMethod: network.IPAllocationMethodDynamic, + Subnet: &subnet, + PublicIPAddress: publicIPv6, + Primary: to.BoolPtr(false), + PrivateIPAddressVersion: network.IPVersionIPv6, + }, + }) + } + if config.SecurityGroupName != "" { authorizer, err := auth.NewClientCredentialsConfig(config.ClientID, config.ClientSecret, config.TenantID).Authorizer() if err != nil { diff --git a/pkg/cloudprovider/provider/azure/get_client.go b/pkg/cloudprovider/provider/azure/get_client.go index 9396178f8..80de3b35f 100644 --- a/pkg/cloudprovider/provider/azure/get_client.go +++ b/pkg/cloudprovider/provider/azure/get_client.go @@ -29,7 +29,7 @@ func getIPClient(c *config) (*network.PublicIPAddressesClient, error) { ipClient := network.NewPublicIPAddressesClient(c.SubscriptionID) ipClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() if err != nil { - return nil, fmt.Errorf("failed to create authorizer: %s", err.Error()) + return nil, fmt.Errorf("failed to create authorizer: %v", err) } return &ipClient, nil diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index fa7112887..ccc1dfd8d 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -55,10 +55,11 @@ const ( machineUIDTag = "Machine-UID" - finalizerPublicIP = "kubermatic.io/cleanup-azure-public-ip" - finalizerNIC = "kubermatic.io/cleanup-azure-nic" - finalizerDisks = "kubermatic.io/cleanup-azure-disks" - finalizerVM = "kubermatic.io/cleanup-azure-vm" + finalizerPublicIP = "kubermatic.io/cleanup-azure-public-ip" + finalizerPublicIPv6 = "kubermatic.io/cleanup-azure-public-ipv6" + finalizerNIC = "kubermatic.io/cleanup-azure-nic" + finalizerDisks = "kubermatic.io/cleanup-azure-disks" + finalizerVM = "kubermatic.io/cleanup-azure-vm" ) const ( @@ -411,40 +412,50 @@ func getNICIPAddresses(ctx context.Context, c *config, ifaceName string) (map[st ipAddresses := map[string]v1.NodeAddressType{} - if netIf.IPConfigurations != nil { - for _, conf := range *netIf.IPConfigurations { - var name string - if conf.Name != nil { - name = *conf.Name - } else { - klog.Warningf("IP configuration of NIC %q was returned with no name, trying to dissect the ID.", ifaceName) - if conf.ID == nil || len(*conf.ID) == 0 { - return nil, fmt.Errorf("IP configuration of NIC %q was returned with no ID", ifaceName) - } - splitConfID := strings.Split(*conf.ID, "/") - name = splitConfID[len(splitConfID)-1] + if netIf.IPConfigurations == nil { + return ipAddresses, nil + } + + for _, conf := range *netIf.IPConfigurations { + var name string + if conf.Name != nil { + name = *conf.Name + } else { + klog.Warningf("IP configuration of NIC %q was returned with no name, trying to dissect the ID.", ifaceName) + if conf.ID == nil || len(*conf.ID) == 0 { + return nil, fmt.Errorf("IP configuration of NIC %q was returned with no ID", ifaceName) } + splitConfID := strings.Split(*conf.ID, "/") + name = splitConfID[len(splitConfID)-1] + } - if c.AssignPublicIP { - publicIPName := ifaceName + "-pubip" - publicIPs, err := getIPAddressStrings(ctx, c, publicIPName) - if err != nil { - return nil, fmt.Errorf("failed to retrieve IP string for IP %q: %v", name, err) - } - for _, ip := range publicIPs { - ipAddresses[ip] = v1.NodeExternalIP - } + if c.AssignPublicIP { + publicIPs, err := getIPAddressStrings(ctx, c, publicIPName(ifaceName)) + if err != nil { + return nil, fmt.Errorf("failed to retrieve IP string for IP %q: %v", name, err) + } + for _, ip := range publicIPs { + ipAddresses[ip] = v1.NodeExternalIP } - internalIPs, err := getInternalIPAddresses(ctx, c, ifaceName, name) + publicIP6s, err := getIPAddressStrings(ctx, c, publicIPv6Name(ifaceName)) if err != nil { - return nil, fmt.Errorf("failed to retrieve internal IP string for IP %q: %v", name, err) + return nil, fmt.Errorf("failed to retrieve IP string for IP %q: %v", name, err) } - for _, ip := range internalIPs { - ipAddresses[ip] = v1.NodeInternalIP + for _, ip := range publicIP6s { + ipAddresses[ip] = v1.NodeExternalIP } } + + internalIPs, err := getInternalIPAddresses(ctx, c, ifaceName, name) + if err != nil { + return nil, fmt.Errorf("failed to retrieve internal IP string for IP %q: %v", name, err) + } + for _, ip := range internalIPs { + ipAddresses[ip] = v1.NodeInternalIP + } + } return ipAddresses, nil @@ -561,9 +572,19 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return nil, fmt.Errorf("failed to generate ssh key: %v", err) } - ifaceName := machine.Name + "-netiface" - publicIPName := ifaceName + "-pubip" - var publicIP *network.PublicIPAddress + netFamily := providerCfg.Network.GetNetworkFamily() + sku := network.PublicIPAddressSkuNameBasic + if netFamily == util.DualStack { + // 1. Cannot specify basic sku PublicIp for an IPv6 network interface ipConfiguration. + // 2. Different basic sku and standard sku public Ip resources in availability set is not allowed. + // 1 & 2 means we have to use standard sku in dual-stack configuration. + + // It is not clear from the documentation, but you get the + // errors if you try mixing skus or try to create IPv6 public IP with + // basic sku. + sku = network.PublicIPAddressSkuNameStandard + } + var publicIP, publicIPv6 *network.PublicIPAddress if config.AssignPublicIP { if err = data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { if !kuberneteshelper.HasFinalizer(updatedMachine, finalizerPublicIP) { @@ -572,10 +593,17 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert }); err != nil { return nil, err } - publicIP, err = createOrUpdatePublicIPAddress(context.TODO(), publicIPName, machine.UID, config) + publicIP, err = createOrUpdatePublicIPAddress(context.TODO(), publicIPName(ifaceName(machine)), network.IPVersionIPv4, sku, network.IPAllocationMethodStatic, machine.UID, config) if err != nil { return nil, fmt.Errorf("failed to create public IP: %v", err) } + + if netFamily == util.DualStack { + publicIPv6, err = createOrUpdatePublicIPAddress(context.TODO(), publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodStatic, machine.UID, config) + if err != nil { + return nil, fmt.Errorf("failed to create public IP: %v", err) + } + } } if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { @@ -585,7 +613,8 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert }); err != nil { return nil, err } - iface, err := createOrUpdateNetworkInterface(context.TODO(), ifaceName, machine.UID, config, publicIP) + + iface, err := createOrUpdateNetworkInterface(context.TODO(), ifaceName(machine), machine.UID, config, publicIP, publicIPv6, netFamily) if err != nil { return nil, fmt.Errorf("failed to generate main network interface: %v", err) } @@ -606,6 +635,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert if err != nil { return nil, fmt.Errorf("failed to get StorageProfile: %v", err) } + vmSpec := compute.VirtualMachine{ Location: &config.Location, Plan: osPlane, @@ -908,8 +938,45 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri return s, "azure", nil } +func validateDiskSKUs(c *config) error { + if c.OSDiskSKU != nil || c.DataDiskSKU != nil { + sku, err := getSKU(context.TODO(), c) + if err != nil { + return fmt.Errorf("failed to get VM SKU: %w", err) + } + + if c.OSDiskSKU != nil { + if _, ok := osDiskSKUs[*c.OSDiskSKU]; !ok { + return fmt.Errorf("invalid OS disk SKU '%s'", *c.OSDiskSKU) + } + + if err := supportsDiskSKU(sku, *c.OSDiskSKU, c.Zones); err != nil { + return err + } + } + + if c.DataDiskSKU != nil { + if _, ok := dataDiskSKUs[*c.DataDiskSKU]; !ok { + return fmt.Errorf("invalid data disk SKU '%s'", *c.DataDiskSKU) + } + + // Ultra SSDs do not support availability sets, see for reference: + // https://docs.microsoft.com/en-us/azure/virtual-machines/disks-enable-ultra-ssd#ga-scope-and-limitations + if *c.DataDiskSKU == compute.StorageAccountTypesUltraSSDLRS && ((c.AssignAvailabilitySet != nil && *c.AssignAvailabilitySet) || c.AvailabilitySet != "") { + return fmt.Errorf("data disk SKU '%s' does not support availability sets", *c.DataDiskSKU) + } + + if err := supportsDiskSKU(sku, *c.DataDiskSKU, c.Zones); err != nil { + return err + } + } + } + + return nil +} + func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { - c, providerCfg, err := p.getConfig(spec.ProviderSpec) + c, providerConfig, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %v", err) } @@ -946,6 +1013,17 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return errors.New("subnetName is missing") } + switch f := providerConfig.Network.GetNetworkFamily(); f { + case util.Unspecified, util.IPv4: + //noop + case util.IPv6: + return fmt.Errorf(util.ErrIPv6OnlyUnsupported) + case util.DualStack: + // validate + default: + return fmt.Errorf(util.ErrUnknownNetworkFamily, f) + } + vmClient, err := getVMClient(c) if err != nil { return fmt.Errorf("failed to (create) vm client: %v", err.Error()) @@ -968,48 +1046,23 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return fmt.Errorf("failed to validate disk SKUs: %w", err) } - _, err = getOSImageReference(c, providerCfg.OperatingSystem) + _, err = getOSImageReference(c, providerConfig.OperatingSystem) return err } -func validateDiskSKUs(c *config) error { - if c.OSDiskSKU != nil || c.DataDiskSKU != nil { - sku, err := getSKU(context.TODO(), c) - if err != nil { - return fmt.Errorf("failed to get VM SKU: %w", err) - } - - if c.OSDiskSKU != nil { - if _, ok := osDiskSKUs[*c.OSDiskSKU]; !ok { - return fmt.Errorf("invalid OS disk SKU '%s'", *c.OSDiskSKU) - } - - if err := supportsDiskSKU(sku, *c.OSDiskSKU, c.Zones); err != nil { - return err - } - } - - if c.DataDiskSKU != nil { - if _, ok := dataDiskSKUs[*c.DataDiskSKU]; !ok { - return fmt.Errorf("invalid data disk SKU '%s'", *c.DataDiskSKU) - } - - // Ultra SSDs do not support availability sets, see for reference: - // https://docs.microsoft.com/en-us/azure/virtual-machines/disks-enable-ultra-ssd#ga-scope-and-limitations - if *c.DataDiskSKU == compute.StorageAccountTypesUltraSSDLRS && ((c.AssignAvailabilitySet != nil && *c.AssignAvailabilitySet) || c.AvailabilitySet != "") { - return fmt.Errorf("data disk SKU '%s' does not support availability sets", *c.DataDiskSKU) - } +func ifaceName(machine *clusterv1alpha1.Machine) string { + return machine.Name + "-netiface" +} - if err := supportsDiskSKU(sku, *c.DataDiskSKU, c.Zones); err != nil { - return err - } - } - } +func publicIPName(ifaceName string) string { + return ifaceName + "-pubip" +} - return nil +func publicIPv6Name(ifaceName string) string { + return ifaceName + "-pubipv6" } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -1026,19 +1079,26 @@ func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) e return fmt.Errorf("failed to create VM client: %v", err) } - ifaceName := machine.Name + "-netiface" - publicIPName := ifaceName + "-pubip" - var publicIP *network.PublicIPAddress + var publicIP, publicIPv6 *network.PublicIPAddress + sku := network.PublicIPAddressSkuNameBasic + + if kuberneteshelper.HasFinalizer(machine, finalizerPublicIPv6) { + sku = network.PublicIPAddressSkuNameStandard + _, err = createOrUpdatePublicIPAddress(ctx, publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodDynamic, newUID, config) + if err != nil { + return fmt.Errorf("failed to update UID on public IP: %v", err) + } + } if kuberneteshelper.HasFinalizer(machine, finalizerPublicIP) { - _, err = createOrUpdatePublicIPAddress(ctx, publicIPName, new, config) + _, err = createOrUpdatePublicIPAddress(ctx, publicIPName(ifaceName(machine)), network.IPVersionIPv4, sku, network.IPAllocationMethodStatic, newUID, config) if err != nil { return fmt.Errorf("failed to update UID on public IP: %v", err) } } if kuberneteshelper.HasFinalizer(machine, finalizerNIC) { - _, err = createOrUpdateNetworkInterface(ctx, ifaceName, new, config, publicIP) + _, err = createOrUpdateNetworkInterface(ctx, ifaceName(machine), newUID, config, publicIP, publicIPv6, util.Unspecified) if err != nil { return fmt.Errorf("failed to update UID on main network interface: %v", err) } @@ -1056,7 +1116,7 @@ func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) e } for _, disk := range disks { - disk.Tags[machineUIDTag] = to.StringPtr(string(new)) + disk.Tags[machineUIDTag] = to.StringPtr(string(newUID)) future, err := disksClient.CreateOrUpdate(ctx, config.ResourceGroup, *disk.Name, disk) if err != nil { return fmt.Errorf("failed to update UID for disk %s: %v", *disk.Name, err) @@ -1071,7 +1131,7 @@ func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) e for k, v := range config.Tags { tags[k] = to.StringPtr(v) } - tags[machineUIDTag] = to.StringPtr(string(new)) + tags[machineUIDTag] = to.StringPtr(string(newUID)) vmSpec := compute.VirtualMachine{Location: &config.Location, Tags: tags} future, err := vmClient.CreateOrUpdate(ctx, config.ResourceGroup, machine.Name, vmSpec) diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index c85f29b41..592cd6b69 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -50,7 +50,6 @@ const ( errConnect = "Failed to connect: %v" errInvalidServiceAccount = "Service account is missing" errIPv6UnsupportedZone = "IPv6 is not supported in zone: %s" - errIPv6OnlyUnsupported = "IPv6 only network family not supported yet" errInvalidZone = "Zone is missing" errInvalidMachineType = "Machine type is missing" errInvalidDiskSize = "Disk size must be a positive number" @@ -121,7 +120,7 @@ func (p *Provider) Validate(spec clusterv1alpha1.MachineSpec) error { case util.Unspecified, util.IPv4: // noop case util.IPv6: - return newError(common.InvalidConfigurationMachineError, errIPv6OnlyUnsupported) + return newError(common.InvalidConfigurationMachineError, util.ErrIPv6OnlyUnsupported) case util.DualStack: if !isIPv6Supported(cfg.zone) { return newError(common.InvalidConfigurationMachineError, errIPv6UnsupportedZone, cfg.zone) diff --git a/pkg/cloudprovider/util/net.go b/pkg/cloudprovider/util/net.go index 9b04758a9..f685af833 100644 --- a/pkg/cloudprovider/util/net.go +++ b/pkg/cloudprovider/util/net.go @@ -24,6 +24,7 @@ import ( ) const ( + ErrIPv6OnlyUnsupported = "IPv6 only network family not supported yet" ErrUnknownNetworkFamily = "Unknown network family %q only IPv4,IPv6,IPv4+IPv6 are valid values" ) From 4bfe7ad34eea34a45641a6c65c92b6a12e09f90b Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Wed, 20 Apr 2022 10:40:47 +0300 Subject: [PATCH 123/489] support rocky linux for vSphere (#1255) Signed-off-by: Moath Qasim --- test/e2e/provisioning/all_e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 8d0197fa1..bc7050f42 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -814,7 +814,7 @@ func getVSphereTestParams(t *testing.T) []string { func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() - selector := Not(OsSelector("sles", "amzn2", "rockylinux")) + selector := Not(OsSelector("sles", "amzn2")) params := getVSphereTestParams(t) runScenarios(t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) From a0be5f2b3a8b7a245e53accf3b2c7a5c6b231fbd Mon Sep 17 00:00:00 2001 From: Pratik Deoghare Date: Wed, 20 Apr 2022 14:57:34 +0530 Subject: [PATCH 124/489] rename NetworkFamily to AddressFamily (#1254) * rename NetworkFamily to AddressFamily * fix tests * fix log message Co-authored-by: Rastislav Szabo * rename AddressFamily to IPFamily Ref: https://kubernetes.io/docs/concepts/services-networking/dual-stack/ Co-authored-by: Rastislav Szabo --- pkg/cloudprovider/provider/aws/provider.go | 4 ++-- .../provider/azure/create_delete_resources.go | 4 ++-- pkg/cloudprovider/provider/azure/provider.go | 10 +++++----- pkg/cloudprovider/provider/gce/provider.go | 4 ++-- pkg/cloudprovider/provider/gce/provider_test.go | 12 ++++++------ pkg/cloudprovider/provider/gce/service.go | 4 ++-- pkg/cloudprovider/util/net.go | 14 +++++++------- pkg/providerconfig/types/types.go | 12 ++++++------ 8 files changed, 32 insertions(+), 32 deletions(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 9db3cdd01..b2528c0ae 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -617,7 +617,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return fmt.Errorf("invalid vpc %q specified: %v", config.VpcID, err) } - switch f := pc.Network.GetNetworkFamily(); f { + switch f := pc.Network.GetIPFamily(); f { case util.Unspecified, util.IPv4: // noop case util.IPv6, util.DualStack: @@ -823,7 +823,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert }, } - if pc.Network.GetNetworkFamily() == util.IPv6 || pc.Network.GetNetworkFamily() == util.DualStack { + if pc.Network.GetIPFamily() == util.IPv6 || pc.Network.GetIPFamily() == util.DualStack { instanceRequest.NetworkInterfaces[0].Ipv6AddressCount = aws.Int64(1) } diff --git a/pkg/cloudprovider/provider/azure/create_delete_resources.go b/pkg/cloudprovider/provider/azure/create_delete_resources.go index 355b2509d..45248e1b1 100644 --- a/pkg/cloudprovider/provider/azure/create_delete_resources.go +++ b/pkg/cloudprovider/provider/azure/create_delete_resources.go @@ -319,7 +319,7 @@ func getVirtualNetwork(ctx context.Context, c *config) (network.VirtualNetwork, return virtualNetworksClient.Get(ctx, c.VNetResourceGroup, c.VNetName, "") } -func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineUID types.UID, config *config, publicIP, publicIPv6 *network.PublicIPAddress, netFamily util.NetworkFamily) (*network.Interface, error) { +func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineUID types.UID, config *config, publicIP, publicIPv6 *network.PublicIPAddress, ipFamily util.IPFamily) (*network.Interface, error) { ifClient, err := getInterfacesClient(config) if err != nil { return nil, fmt.Errorf("failed to create interfaces client: %v", err) @@ -349,7 +349,7 @@ func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineU }, }) - if netFamily == util.DualStack { + if ipFamily == util.DualStack { *ifSpec.InterfacePropertiesFormat.IPConfigurations = append(*ifSpec.InterfacePropertiesFormat.IPConfigurations, network.InterfaceIPConfiguration{ Name: to.StringPtr("ip-config-2"), InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index ccc1dfd8d..949d65ef9 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -572,9 +572,9 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return nil, fmt.Errorf("failed to generate ssh key: %v", err) } - netFamily := providerCfg.Network.GetNetworkFamily() + ipFamily := providerCfg.Network.GetIPFamily() sku := network.PublicIPAddressSkuNameBasic - if netFamily == util.DualStack { + if ipFamily == util.DualStack { // 1. Cannot specify basic sku PublicIp for an IPv6 network interface ipConfiguration. // 2. Different basic sku and standard sku public Ip resources in availability set is not allowed. // 1 & 2 means we have to use standard sku in dual-stack configuration. @@ -598,7 +598,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return nil, fmt.Errorf("failed to create public IP: %v", err) } - if netFamily == util.DualStack { + if ipFamily == util.DualStack { publicIPv6, err = createOrUpdatePublicIPAddress(context.TODO(), publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodStatic, machine.UID, config) if err != nil { return nil, fmt.Errorf("failed to create public IP: %v", err) @@ -614,7 +614,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return nil, err } - iface, err := createOrUpdateNetworkInterface(context.TODO(), ifaceName(machine), machine.UID, config, publicIP, publicIPv6, netFamily) + iface, err := createOrUpdateNetworkInterface(context.TODO(), ifaceName(machine), machine.UID, config, publicIP, publicIPv6, ipFamily) if err != nil { return nil, fmt.Errorf("failed to generate main network interface: %v", err) } @@ -1013,7 +1013,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return errors.New("subnetName is missing") } - switch f := providerConfig.Network.GetNetworkFamily(); f { + switch f := providerConfig.Network.GetIPFamily(); f { case util.Unspecified, util.IPv4: //noop case util.IPv6: diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 592cd6b69..6532bba52 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -116,7 +116,7 @@ func (p *Provider) Validate(spec clusterv1alpha1.MachineSpec) error { return newError(common.InvalidConfigurationMachineError, errInvalidZone) } - switch cfg.providerConfig.Network.GetNetworkFamily() { + switch cfg.providerConfig.Network.GetIPFamily() { case util.Unspecified, util.IPv4: // noop case util.IPv6: @@ -126,7 +126,7 @@ func (p *Provider) Validate(spec clusterv1alpha1.MachineSpec) error { return newError(common.InvalidConfigurationMachineError, errIPv6UnsupportedZone, cfg.zone) } default: - return newError(common.InvalidConfigurationMachineError, util.ErrUnknownNetworkFamily, cfg.providerConfig.Network.GetNetworkFamily()) + return newError(common.InvalidConfigurationMachineError, util.ErrUnknownNetworkFamily, cfg.providerConfig.Network.GetIPFamily()) } if cfg.machineType == "" { diff --git a/pkg/cloudprovider/provider/gce/provider_test.go b/pkg/cloudprovider/provider/gce/provider_test.go index f19965bfe..c05ad298b 100644 --- a/pkg/cloudprovider/provider/gce/provider_test.go +++ b/pkg/cloudprovider/provider/gce/provider_test.go @@ -120,7 +120,7 @@ func TestValidate(t *testing.T) { expectErr bool }{ { - "without network family", + "without IP family", v1alpha1.MachineSpec{ ProviderSpec: v1alpha1.ProviderSpec{ Value: &runtime.RawExtension{ @@ -131,12 +131,12 @@ func TestValidate(t *testing.T) { false, }, { - "empty network family", + "empty IP family", v1alpha1.MachineSpec{ ProviderSpec: v1alpha1.ProviderSpec{ Value: &runtime.RawExtension{ Raw: rawBytes(testMap(testProviderSpec()). - with("network.networkFamily", ""), + with("network.ipFamily", ""), ), }, }, @@ -144,12 +144,12 @@ func TestValidate(t *testing.T) { false, }, { - "with network family", + "with IP family", v1alpha1.MachineSpec{ ProviderSpec: v1alpha1.ProviderSpec{ Value: &runtime.RawExtension{ Raw: rawBytes(testMap(testProviderSpec()). - with("network.networkFamily", "IPv4+IPv6"), + with("network.ipFamily", "IPv4+IPv6"), ), }, }, @@ -162,7 +162,7 @@ func TestValidate(t *testing.T) { ProviderSpec: v1alpha1.ProviderSpec{ Value: &runtime.RawExtension{ Raw: rawBytes(testMap(testProviderSpec()). - with("network.networkFamily", "IPv4+IPv6"). + with("network.ipFamily", "IPv4+IPv6"). with("cloudProviderSpec.zone", "europe-west3-a"), ), }, diff --git a/pkg/cloudprovider/provider/gce/service.go b/pkg/cloudprovider/provider/gce/service.go index f62da9716..ed2ee566e 100644 --- a/pkg/cloudprovider/provider/gce/service.go +++ b/pkg/cloudprovider/provider/gce/service.go @@ -91,7 +91,7 @@ func (svc *service) networkInterfaces(cfg *config) ([]*compute.NetworkInterface, // if assigning public IP addresses is enabled. if cfg.assignPublicIPAddress { // GCP doesn't support IPv6 only stack - if cfg.providerConfig.Network.GetNetworkFamily() == util.DualStack { + if cfg.providerConfig.Network.GetIPFamily() == util.DualStack { ifc.StackType = "IPV4_IPV6" ifc.Ipv6AccessConfigs = []*compute.AccessConfig{ @@ -102,7 +102,7 @@ func (svc *service) networkInterfaces(cfg *config) ([]*compute.NetworkInterface, }, } } else { - klog.Infof("network family doesn't specify dual stack: %s", cfg.providerConfig.Network.GetNetworkFamily()) + klog.Infof("IP family doesn't specify dual stack: %s", cfg.providerConfig.Network.GetIPFamily()) } } diff --git a/pkg/cloudprovider/util/net.go b/pkg/cloudprovider/util/net.go index f685af833..fc640f1b6 100644 --- a/pkg/cloudprovider/util/net.go +++ b/pkg/cloudprovider/util/net.go @@ -25,7 +25,7 @@ import ( const ( ErrIPv6OnlyUnsupported = "IPv6 only network family not supported yet" - ErrUnknownNetworkFamily = "Unknown network family %q only IPv4,IPv6,IPv4+IPv6 are valid values" + ErrUnknownNetworkFamily = "Unknown IP family %q only IPv4,IPv6,IPv4+IPv6 are valid values" ) func CIDRToIPAndNetMask(ipv4 string) (string, string, int, error) { @@ -60,14 +60,14 @@ func GenerateRandMAC() (net.HardwareAddr, error) { return mac, nil } -// NetworkFamily IPv4 | IPv6 | IPv4+IPv6 -type NetworkFamily string +// IPFamily IPv4 | IPv6 | IPv4+IPv6 +type IPFamily string const ( - Unspecified NetworkFamily = "" // interpreted as IPv4 - IPv4 NetworkFamily = "IPv4" - IPv6 NetworkFamily = "IPv6" - DualStack NetworkFamily = "IPv4+IPv6" + Unspecified IPFamily = "" // interpreted as IPv4 + IPv4 IPFamily = "IPv4" + IPv6 IPFamily = "IPv6" + DualStack IPFamily = "IPv4+IPv6" ) // IsLinkLocal checks if given ip address is link local diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 5fa134a22..6d8c5ef32 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -109,17 +109,17 @@ type DNSConfig struct { // NetworkConfig contains a machine's static network configuration type NetworkConfig struct { - CIDR string `json:"cidr"` - Gateway string `json:"gateway"` - DNS DNSConfig `json:"dns"` - NetworkFamily util.NetworkFamily `json:"networkFamily,omitempty"` + CIDR string `json:"cidr"` + Gateway string `json:"gateway"` + DNS DNSConfig `json:"dns"` + IPFamily util.IPFamily `json:"ipFamily,omitempty"` } -func (n *NetworkConfig) GetNetworkFamily() util.NetworkFamily { +func (n *NetworkConfig) GetIPFamily() util.IPFamily { if n == nil { return util.Unspecified } - return n.NetworkFamily + return n.IPFamily } type Config struct { From c4a4228c8160f22f67a7fd3dc90c45c077eff095 Mon Sep 17 00:00:00 2001 From: tlamr <34898768+tlamr@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:29:20 +0200 Subject: [PATCH 125/489] Expose google provisioning model to be able to acquire spot instances (#1252) * expose google provisioning model to be able to acquire spot instances Signed-off-by: Tomas Lamr reformat sources Signed-off-by: Tomas Lamr Make automaticRestart and provisioningModel Optional in pkg/cloudprovider/provider/gce/types/types.go Co-authored-by: Waleed Malik lint Signed-off-by: Tomas Lamr * incorporate PR changes Signed-off-by: Tomas Lamr * thanks ahmedwaleedmalik for being patient with my commits :) Signed-off-by: Tomas Lamr * refactor: improve handling for GCE config Signed-off-by: Waleed Malik Co-authored-by: Tomas Lamr Co-authored-by: Waleed Malik --- pkg/cloudprovider/provider/gce/config.go | 22 ++++++++++++++ pkg/cloudprovider/provider/gce/provider.go | 9 ++++++ .../provider/gce/provider_test.go | 1 + pkg/cloudprovider/provider/gce/types/types.go | 30 ++++++++++--------- 4 files changed, 48 insertions(+), 14 deletions(-) diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index 6befe983d..00e544406 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -101,6 +101,8 @@ type config struct { network string subnetwork string preemptible bool + automaticRestart *bool + provisioningModel *string labels map[string]string tags []string jwtConfig *jwt.Config @@ -167,6 +169,26 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide return nil, fmt.Errorf("cannot retrieve preemptible: %v", err) } + if cpSpec.AutomaticRestart != nil { + automaticRestart, _, err := resolver.GetConfigVarBoolValue(*cpSpec.AutomaticRestart) + if err != nil { + return nil, fmt.Errorf("cannot retrieve automaticRestart: %v", err) + } + cfg.automaticRestart = &automaticRestart + + if *cfg.automaticRestart && cfg.preemptible { + return nil, fmt.Errorf("automatic restart option can only be enabled for standard instances. Preemptible instances cannot be automatically restarted") + } + } + + if cpSpec.ProvisioningModel != nil { + provisioningModel, err := resolver.GetConfigVarStringValue(*cpSpec.ProvisioningModel) + if err != nil { + return nil, fmt.Errorf("cannot retrieve provisioningModel: %v", err) + } + cfg.provisioningModel = &provisioningModel + } + // make it true by default cfg.assignPublicIPAddress = true diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 6532bba52..6c5db721f 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -288,6 +288,15 @@ func (p *Provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert Items: cfg.tags, }, } + + if cfg.automaticRestart != nil { + inst.Scheduling.AutomaticRestart = cfg.automaticRestart + } + + if cfg.provisioningModel != nil { + inst.Scheduling.ProvisioningModel = *cfg.provisioningModel + } + op, err := svc.Instances.Insert(cfg.projectID, cfg.zone, inst).Do() if err != nil { return nil, newError(common.InvalidConfigurationMachineError, errInsertInstance, err) diff --git a/pkg/cloudprovider/provider/gce/provider_test.go b/pkg/cloudprovider/provider/gce/provider_test.go index c05ad298b..036918699 100644 --- a/pkg/cloudprovider/provider/gce/provider_test.go +++ b/pkg/cloudprovider/provider/gce/provider_test.go @@ -44,6 +44,7 @@ func testProviderSpec() map[string]interface{} { "multizone": false, "network": "global/networks/default", "preemptible": false, + "provisioningModel": "STANDARD", "regional": false, "serviceAccount": "", "subnetwork": "", diff --git a/pkg/cloudprovider/provider/gce/types/types.go b/pkg/cloudprovider/provider/gce/types/types.go index 8eeed7207..7b05eef38 100644 --- a/pkg/cloudprovider/provider/gce/types/types.go +++ b/pkg/cloudprovider/provider/gce/types/types.go @@ -30,20 +30,22 @@ import ( // CloudProviderSpec contains the specification of the cloud provider taken // from the provider configuration. type CloudProviderSpec struct { - ServiceAccount providerconfigtypes.ConfigVarString `json:"serviceAccount,omitempty"` - Zone providerconfigtypes.ConfigVarString `json:"zone"` - MachineType providerconfigtypes.ConfigVarString `json:"machineType"` - DiskSize int64 `json:"diskSize"` - DiskType providerconfigtypes.ConfigVarString `json:"diskType"` - Network providerconfigtypes.ConfigVarString `json:"network"` - Subnetwork providerconfigtypes.ConfigVarString `json:"subnetwork"` - Preemptible providerconfigtypes.ConfigVarBool `json:"preemptible"` - Labels map[string]string `json:"labels,omitempty"` - Tags []string `json:"tags,omitempty"` - AssignPublicIPAddress *providerconfigtypes.ConfigVarBool `json:"assignPublicIPAddress,omitempty"` - MultiZone providerconfigtypes.ConfigVarBool `json:"multizone"` - Regional providerconfigtypes.ConfigVarBool `json:"regional"` - CustomImage providerconfigtypes.ConfigVarString `json:"customImage,omitempty"` + ServiceAccount providerconfigtypes.ConfigVarString `json:"serviceAccount,omitempty"` + Zone providerconfigtypes.ConfigVarString `json:"zone"` + MachineType providerconfigtypes.ConfigVarString `json:"machineType"` + DiskSize int64 `json:"diskSize"` + DiskType providerconfigtypes.ConfigVarString `json:"diskType"` + Network providerconfigtypes.ConfigVarString `json:"network"` + Subnetwork providerconfigtypes.ConfigVarString `json:"subnetwork"` + Preemptible providerconfigtypes.ConfigVarBool `json:"preemptible"` + AutomaticRestart *providerconfigtypes.ConfigVarBool `json:"automaticRestart,omitempty"` + ProvisioningModel *providerconfigtypes.ConfigVarString `json:"provisioningModel,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Tags []string `json:"tags,omitempty"` + AssignPublicIPAddress *providerconfigtypes.ConfigVarBool `json:"assignPublicIPAddress,omitempty"` + MultiZone providerconfigtypes.ConfigVarBool `json:"multizone"` + Regional providerconfigtypes.ConfigVarBool `json:"regional"` + CustomImage providerconfigtypes.ConfigVarString `json:"customImage,omitempty"` } // UpdateProviderSpec updates the given provider spec with changed From 99a8a3fe1d34a65b7a4cb1133557d60765fae383 Mon Sep 17 00:00:00 2001 From: Pratik Deoghare Date: Tue, 26 Apr 2022 13:17:45 +0530 Subject: [PATCH 126/489] define static ip config (#1263) * unwire pod cidrs command line flag * define static ip config * remove refs to pod cidr flag * ignore cyclomatic complexity of aws getconfig --- cmd/machine-controller/main.go | 11 +++-------- pkg/apis/plugin/plugin.go | 1 - pkg/cloudprovider/provider/aws/provider.go | 7 ++----- pkg/controller/machine/machine_controller.go | 6 +----- pkg/providerconfig/types/types.go | 9 +++++++++ pkg/userdata/amzn2/provider.go | 2 +- pkg/userdata/centos/provider.go | 2 +- pkg/userdata/rhel/provider.go | 2 +- pkg/userdata/rhel/provider_test.go | 1 - pkg/userdata/rockylinux/provider.go | 2 +- pkg/userdata/sles/provider.go | 2 +- pkg/userdata/ubuntu/provider.go | 2 +- 12 files changed, 21 insertions(+), 26 deletions(-) diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 93f5caa76..14f5d3234 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -79,7 +79,7 @@ var ( nodeRegistryMirrors string nodePauseImage string nodeContainerRuntime string - podCIDRs string + podCIDR string nodePortRange string nodeRegistryCredentialsSecret string nodeContainerdRegistryMirrors = containerruntime.RegistryMirrorsFlags{} @@ -127,9 +127,6 @@ type controllerRunOptions struct { useOSM bool - // Assigns the POD networks that will be allocated. - podCIDRs []string - // A port range to reserve for services with NodePort visibility nodePortRange string } @@ -140,7 +137,7 @@ func main() { klog.InitFlags(nil) // This is also being registered in kubevirt.io/kubevirt/pkg/kubecli/kubecli.go so // we have to guard it - //TODO: Evaluate alternatives to importing the CLI. Generate our own client? Use a dynamic client? + // TODO: Evaluate alternatives to importing the CLI. Generate our own client? Use a dynamic client? if flag.Lookup("kubeconfig") == nil { flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") } @@ -166,7 +163,7 @@ func main() { flag.Var(&nodeContainerdRegistryMirrors, "node-containerd-registry-mirrors", "Configure registry mirrors endpoints. Can be used multiple times to specify multiple mirrors") flag.StringVar(&caBundleFile, "ca-bundle", "", "path to a file containing all PEM-encoded CA certificates (will be used instead of the host's certificates if set)") flag.BoolVar(&nodeCSRApprover, "node-csr-approver", true, "Enable NodeCSRApprover controller to automatically approve node serving certificate requests") - flag.StringVar(&podCIDRs, "pod-cidr", "172.25.0.0/16", "Comma separated network ranges from which POD networks are allocated. Example: 172.25.0.0/16,fd00::/104") + flag.StringVar(&podCIDR, "pod-cidr", "172.25.0.0/16", "WARNING: flag is unused, kept only for backwards compatibility") flag.StringVar(&nodePortRange, "node-port-range", "30000-32767", "A port range to reserve for services with NodePort visibility") flag.StringVar(&nodeRegistryCredentialsSecret, "node-registry-credentials-secret", "", "A Secret object reference, that containt auth info for image registry in namespace/secret-name form, example: kube-system/registry-credentials. See doc at https://github.com/kubermaric/machine-controller/blob/master/docs/registry-authentication.md") flag.BoolVar(&useOSM, "use-osm", false, "use osm controller for node bootstrap") @@ -266,7 +263,6 @@ func main() { ContainerRuntime: containerRuntimeConfig, }, useOSM: useOSM, - podCIDRs: strings.Split(podCIDRs, ","), nodePortRange: nodePortRange, } @@ -402,7 +398,6 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { bs.opt.skipEvictionAfter, bs.opt.node, bs.opt.useOSM, - bs.opt.podCIDRs, bs.opt.nodePortRange, ); err != nil { return fmt.Errorf("failed to add Machine controller to manager: %v", err) diff --git a/pkg/apis/plugin/plugin.go b/pkg/apis/plugin/plugin.go index 0cc81ac6d..2ed8ea5fb 100644 --- a/pkg/apis/plugin/plugin.go +++ b/pkg/apis/plugin/plugin.go @@ -54,7 +54,6 @@ type UserDataRequest struct { KubeletFeatureGates map[string]bool KubeletConfigs map[string]string ContainerRuntime containerruntime.Config - PodCIDRs []string NodePortRange string } diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index b2528c0ae..b0be676ff 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -362,11 +362,8 @@ func getDefaultRootDevicePath(os providerconfigtypes.OperatingSystem) (string, e return "", fmt.Errorf("no default root path found for %s operating system", os) } +//gocyclo:ignore func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *awstypes.RawConfig, error) { - if provSpec.Value == nil { - return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, nil, err @@ -848,7 +845,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertyp return false, err } - //(*Config, *providerconfigtypes.Config, *awstypes.RawConfig, error) + // (*Config, *providerconfigtypes.Config, *awstypes.RawConfig, error) config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 9094c75a2..95870092b 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -118,7 +118,6 @@ type Reconciler struct { satelliteSubscriptionManager rhsm.SatelliteSubscriptionManager useOSM bool - podCIDRs []string nodePortRange string } @@ -176,7 +175,6 @@ func Add( skipEvictionAfter time.Duration, nodeSettings NodeSettings, useOSM bool, - podCIDRs []string, nodePortRange string, ) error { reconciler := &Reconciler{ @@ -195,7 +193,6 @@ func Add( satelliteSubscriptionManager: rhsm.NewSatelliteSubscriptionManager(), useOSM: useOSM, - podCIDRs: podCIDRs, nodePortRange: nodePortRange, } m, err := userdatamanager.New() @@ -426,7 +423,7 @@ func (r *Reconciler) reconcile(ctx context.Context, machine *clusterv1alpha1.Mac node, err := r.getNodeByNodeRef(ctx, machine.Status.NodeRef) if err != nil { - //In case we cannot find a node for the NodeRef we must remove the NodeRef & recreate an instance on the next sync + // In case we cannot find a node for the NodeRef we must remove the NodeRef & recreate an instance on the next sync if kerrors.IsNotFound(err) { klog.V(3).Infof("found invalid NodeRef on machine %s. Deleting reference...", machine.Name) return nil, r.updateMachine(machine, func(m *clusterv1alpha1.Machine) { @@ -803,7 +800,6 @@ func (r *Reconciler) ensureInstanceExistsForMachine( NoProxy: r.nodeSettings.NoProxy, HTTPProxy: r.nodeSettings.HTTPProxy, ContainerRuntime: crRuntime, - PodCIDRs: r.podCIDRs, NodePortRange: r.nodePortRange, } diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 6d8c5ef32..80b302a9a 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -115,6 +115,15 @@ type NetworkConfig struct { IPFamily util.IPFamily `json:"ipFamily,omitempty"` } +func (n *NetworkConfig) IsStaticIPConfig() bool { + if n == nil { + return false + } + return n.CIDR != "" || + n.Gateway != "" || + len(n.DNS.Servers) != 0 +} + func (n *NetworkConfig) GetIPFamily() util.IPFamily { if n == nil { return util.Unspecified diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index 27a22d0f8..70df9daac 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -57,7 +57,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { req.CloudConfig = *pconfig.OverwriteCloudConfig } - if pconfig.Network != nil { + if pconfig.Network.IsStaticIPConfig() { return "", errors.New("static IP config is not supported with Amazon Linux 2") } diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index be8cef649..40490338e 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -57,7 +57,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { req.CloudConfig = *pconfig.OverwriteCloudConfig } - if pconfig.Network != nil { + if pconfig.Network.IsStaticIPConfig() { return "", errors.New("static IP config is not supported with CentOS") } diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 1337bc057..12861a2e0 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -57,7 +57,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { req.CloudConfig = *pconfig.OverwriteCloudConfig } - if pconfig.Network != nil { + if pconfig.Network.IsStaticIPConfig() { return "", errors.New("static IP config is not supported with RHEL") } diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index fc48cec11..6270ad01a 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -283,7 +283,6 @@ func TestUserDataGeneration(t *testing.T) { PauseImage: test.pauseImage, KubeletFeatureGates: kubeletFeatureGates, ContainerRuntime: containerRuntimeConfig, - PodCIDRs: []string{"172.25.0.0/16", "fd00::/104"}, } s, err := provider.UserData(req) if err != nil { diff --git a/pkg/userdata/rockylinux/provider.go b/pkg/userdata/rockylinux/provider.go index d78cfc449..96de00df3 100644 --- a/pkg/userdata/rockylinux/provider.go +++ b/pkg/userdata/rockylinux/provider.go @@ -57,7 +57,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { req.CloudConfig = *pconfig.OverwriteCloudConfig } - if pconfig.Network != nil { + if pconfig.Network.IsStaticIPConfig() { return "", errors.New("static IP config is not supported with RockyLinux") } diff --git a/pkg/userdata/sles/provider.go b/pkg/userdata/sles/provider.go index b2af70268..c58fb4da8 100644 --- a/pkg/userdata/sles/provider.go +++ b/pkg/userdata/sles/provider.go @@ -58,7 +58,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { req.CloudConfig = *pconfig.OverwriteCloudConfig } - if pconfig.Network != nil { + if pconfig.Network.IsStaticIPConfig() { return "", errors.New("static IP config is not supported with SLES") } diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index 4127cf7d8..f5c4cd2ce 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -57,7 +57,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { req.CloudConfig = *pconfig.OverwriteCloudConfig } - if pconfig.Network != nil { + if pconfig.Network.IsStaticIPConfig() { return "", errors.New("static IP config is not supported with Ubuntu") } From 0e9d89d8aea1c3e79818d021963f4f5aa8767108 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Wed, 27 Apr 2022 18:43:40 +0200 Subject: [PATCH 127/489] Publish image to quay.io/kubermatic/machine-controller (#1267) Signed-off-by: Marvin Beckers --- Makefile | 2 +- examples/machine-controller.yaml | 4 ++-- test/tools/integration/master_install_script.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 547f98103..ba82e967d 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ export GIT_TAG ?= $(shell git tag --points-at HEAD) export GOFLAGS?=-mod=readonly -trimpath -REGISTRY ?= docker.io +REGISTRY ?= quay.io REGISTRY_NAMESPACE ?= kubermatic LDFLAGS ?= -ldflags '-s -w' diff --git a/examples/machine-controller.yaml b/examples/machine-controller.yaml index d3ee84c21..e6011dde8 100644 --- a/examples/machine-controller.yaml +++ b/examples/machine-controller.yaml @@ -232,7 +232,7 @@ spec: spec: serviceAccountName: machine-controller containers: - - image: kubermatic/machine-controller:latest + - image: quay.io/kubermatic/machine-controller:latest imagePullPolicy: IfNotPresent name: machine-controller command: @@ -278,7 +278,7 @@ spec: spec: serviceAccountName: machine-controller containers: - - image: kubermatic/machine-controller:latest + - image: quay.io/kubermatic/machine-controller:latest imagePullPolicy: IfNotPresent name: webhook command: diff --git a/test/tools/integration/master_install_script.sh b/test/tools/integration/master_install_script.sh index 19ddfe152..16205b047 100644 --- a/test/tools/integration/master_install_script.sh +++ b/test/tools/integration/master_install_script.sh @@ -139,7 +139,7 @@ if ! ls machine-controller-deployed; then mkdir "images" buildah push localhost/kubermatic/machine-controller oci-archive:./images/machine-controller.tar:localhost/kubermatic/machine-controller:latest ctr --debug --namespace=k8s.io images import --all-platforms --no-unpack images/machine-controller.tar - sed -i "s_- image: kubermatic/machine-controller:latest_- image: localhost/kubermatic/machine-controller:latest_g" examples/machine-controller.yaml + sed -i "s_- image: quay.io/kubermatic/machine-controller:latest_- image: localhost/kubermatic/machine-controller:latest_g" examples/machine-controller.yaml # The 10 minute window given by default for the node to appear is too short # when we upgrade the instance during the upgrade test if [[ ${LC_JOB_NAME:-} = "pull-machine-controller-e2e-ubuntu-upgrade" ]]; then From 2f38b909063d7bb5d2f6b3e916861b420608a99c Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Thu, 28 Apr 2022 16:10:41 +0200 Subject: [PATCH 128/489] Update dependencies to k8s 1.23.6 and controller-runtime 0.11.2 (#1270) * Bump to Kubernetes 1.23.6 and controller-runtime 0.11.2 Signed-off-by: Marvin Beckers * Update fixtures Signed-off-by: Marvin Beckers --- go.mod | 41 +++---- go.sum | 110 +++++++++++++----- .../containerd-kubelet-v1.20-aws.yaml | 7 +- .../amzn2/testdata/kubelet-v1.20-aws.yaml | 7 +- .../testdata/kubelet-v1.21-aws-external.yaml | 7 +- .../amzn2/testdata/kubelet-v1.21-aws.yaml | 7 +- .../kubelet-v1.21-vsphere-mirrors.yaml | 7 +- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 7 +- .../amzn2/testdata/kubelet-v1.21-vsphere.yaml | 7 +- .../amzn2/testdata/kubelet-v1.22-aws.yaml | 7 +- .../amzn2/testdata/kubelet-v1.23-aws.yaml | 7 +- .../kubelet-containerd-v1.20-aws.yaml | 7 +- .../centos/testdata/kubelet-v1.20-aws.yaml | 7 +- .../testdata/kubelet-v1.21-aws-external.yaml | 7 +- .../centos/testdata/kubelet-v1.21-aws.yaml | 7 +- .../testdata/kubelet-v1.21-nutanix.yaml | 7 +- .../kubelet-v1.21-vsphere-mirrors.yaml | 7 +- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 7 +- .../testdata/kubelet-v1.21-vsphere.yaml | 7 +- .../centos/testdata/kubelet-v1.22-aws.yaml | 7 +- .../centos/testdata/kubelet-v1.23-aws.yaml | 7 +- .../flatcar/testdata/cloud-init_v1.20.14.yaml | 7 +- .../flatcar/testdata/cloud-init_v1.21.8.yaml | 7 +- .../flatcar/testdata/cloud-init_v1.22.5.yaml | 7 +- .../flatcar/testdata/cloud-init_v1.23.0.yaml | 7 +- pkg/userdata/flatcar/testdata/containerd.yaml | 7 +- .../flatcar/testdata/ignition_v1.20.14.json | 2 +- .../flatcar/testdata/ignition_v1.21.8.json | 2 +- .../flatcar/testdata/ignition_v1.22.5.json | 2 +- .../flatcar/testdata/ignition_v1.23.0.json | 2 +- .../kubelet-containerd-v1.20-aws.yaml | 7 +- .../rhel/testdata/kubelet-v1.20-aws.yaml | 7 +- .../rhel/testdata/kubelet-v1.21-aws.yaml | 7 +- .../rhel/testdata/kubelet-v1.22-aws.yaml | 7 +- .../rhel/testdata/kubelet-v1.22-nutanix.yaml | 7 +- .../testdata/kubelet-v1.23-aws-external.yaml | 7 +- .../rhel/testdata/kubelet-v1.23-aws.yaml | 7 +- .../kubelet-v1.23-vsphere-mirrors.yaml | 7 +- .../testdata/kubelet-v1.23-vsphere-proxy.yaml | 7 +- .../rhel/testdata/kubelet-v1.23-vsphere.yaml | 7 +- .../rhel/testdata/pod-cidr-azure-rhel.yaml | 7 +- .../kubelet-containerd-v1.20-aws.yaml | 7 +- .../testdata/kubelet-v1.20-aws.yaml | 7 +- .../testdata/kubelet-v1.21-aws-external.yaml | 7 +- .../testdata/kubelet-v1.21-aws.yaml | 7 +- .../testdata/kubelet-v1.21-nutanix.yaml | 7 +- .../kubelet-v1.21-vsphere-mirrors.yaml | 7 +- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 7 +- .../testdata/kubelet-v1.21-vsphere.yaml | 7 +- .../testdata/kubelet-v1.22-aws.yaml | 7 +- .../testdata/kubelet-v1.23-aws.yaml | 7 +- .../sles/testdata/dist-upgrade-on-boot.yaml | 7 +- .../kubelet-version-without-v-prefix.yaml | 7 +- .../sles/testdata/multiple-dns-servers.yaml | 7 +- .../sles/testdata/multiple-ssh-keys.yaml | 7 +- .../openstack-overwrite-cloud-config.yaml | 7 +- pkg/userdata/sles/testdata/openstack.yaml | 7 +- .../sles/testdata/version-1.20.14.yaml | 7 +- .../sles/testdata/version-1.21.8.yaml | 7 +- .../sles/testdata/version-1.22.5.yaml | 7 +- .../sles/testdata/version-1.23.0.yaml | 7 +- .../sles/testdata/vsphere-mirrors.yaml | 7 +- pkg/userdata/sles/testdata/vsphere-proxy.yaml | 7 +- pkg/userdata/sles/testdata/vsphere.yaml | 7 +- pkg/userdata/ubuntu/testdata/containerd.yaml | 7 +- .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 7 +- .../kubelet-version-without-v-prefix.yaml | 7 +- .../ubuntu/testdata/multiple-dns-servers.yaml | 7 +- .../ubuntu/testdata/multiple-ssh-keys.yaml | 7 +- pkg/userdata/ubuntu/testdata/nutanix.yaml | 7 +- .../openstack-overwrite-cloud-config.yaml | 7 +- pkg/userdata/ubuntu/testdata/openstack.yaml | 7 +- .../ubuntu/testdata/version-1.20.14.yaml | 7 +- .../ubuntu/testdata/version-1.21.8.yaml | 7 +- .../ubuntu/testdata/version-1.22.5.yaml | 7 +- .../ubuntu/testdata/version-1.23.0.yaml | 7 +- .../ubuntu/testdata/vsphere-mirrors.yaml | 7 +- .../ubuntu/testdata/vsphere-proxy.yaml | 7 +- pkg/userdata/ubuntu/testdata/vsphere.yaml | 7 +- 79 files changed, 546 insertions(+), 124 deletions(-) diff --git a/go.mod b/go.mod index 3eb2471e3..2d740a79f 100644 --- a/go.mod +++ b/go.mod @@ -43,17 +43,17 @@ require ( gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b k8c.io/operating-system-manager v0.4.0 - k8s.io/api v0.22.2 - k8s.io/apiextensions-apiserver v0.22.2 - k8s.io/apimachinery v0.22.2 + k8s.io/api v0.23.6 + k8s.io/apiextensions-apiserver v0.23.6 + k8s.io/apimachinery v0.23.6 k8s.io/client-go v12.0.0+incompatible k8s.io/klog v1.0.0 - k8s.io/kubelet v0.22.2 - k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a + k8s.io/kubelet v0.23.6 + k8s.io/utils v0.0.0-20211116205334-6203023598ed kubevirt.io/api v0.48.1 kubevirt.io/containerized-data-importer-api v1.41.0 - sigs.k8s.io/controller-runtime v0.10.2 - sigs.k8s.io/yaml v1.2.0 + sigs.k8s.io/controller-runtime v0.11.2 + sigs.k8s.io/yaml v1.3.0 ) require ( @@ -81,10 +81,10 @@ require ( github.com/dimchansky/utfbom v1.1.0 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/emicklei/go-restful v2.11.2+incompatible // indirect - github.com/evanphx/json-patch v4.11.0+incompatible // indirect + github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect - github.com/fsnotify/fsnotify v1.4.9 // indirect - github.com/go-logr/logr v0.4.0 // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/go-logr/logr v1.2.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.5 // indirect github.com/go-openapi/spec v0.19.15 // indirect @@ -103,7 +103,7 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/json-iterator/go v1.1.11 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/kr/pretty v0.2.1 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect @@ -111,11 +111,11 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/reflectwalk v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca // indirect github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.26.0 // indirect + github.com/prometheus/common v0.28.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/smartystreets/assertions v1.2.0 // indirect @@ -135,20 +135,21 @@ require ( google.golang.org/protobuf v1.27.1 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.57.0 // indirect + gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/component-base v0.22.2 // indirect - k8s.io/klog/v2 v2.9.0 // indirect - k8s.io/kube-openapi v0.0.0-20210527164424-3c818078ee3d // indirect + k8s.io/component-base v0.23.6 // indirect + k8s.io/klog/v2 v2.30.0 // indirect + k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect kubevirt.io/controller-lifecycle-operator-sdk v0.2.1 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect + sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect ) replace ( github.com/packethost/packngo => github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888 - k8s.io/client-go => k8s.io/client-go v0.22.2 - k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.22.2 + k8s.io/client-go => k8s.io/client-go v0.23.6 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.6 k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd ) diff --git a/go.sum b/go.sum index c1ae4c8b5..3b1fc62c5 100644 --- a/go.sum +++ b/go.sum @@ -197,6 +197,7 @@ github.com/anexia-it/go-anxcloud v0.3.26 h1:uStosj8srS6OA1OsPsMJBFqd4Znzl6fEhUv8 github.com/anexia-it/go-anxcloud v0.3.26/go.mod h1:fiEBxEtBXx78/OWBJvL7+2o4TESrnEcrDYjLeonGkDw= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apoydence/onpar v0.0.0-20200406201722-06f95a1c68e8/go.mod h1:maauOJD0kdDqIz4xmkunipFVbBoTM6pFSy0kkWBcIUY= @@ -234,6 +235,7 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d/go.mod h1:IyUJYN1gvWjtLF5ZuygmxbnsAyP3aJS6cHzIuZY50B0= @@ -383,8 +385,9 @@ github.com/evanphx/json-patch v4.1.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.11.0+incompatible h1:glyUF9yIYtMHzn8xaKw5rMhdWcwsYV8dZHIq5567/xs= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -401,8 +404,9 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsouza/fake-gcs-server v0.0.0-20180612165233-e85be23bdaa8/go.mod h1:1/HufuJ+eaDf4KTnYdS6HJMGvMRU8d4cYTuu/1QaBbI= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -431,14 +435,16 @@ github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.2.1-0.20200730175230-ee2de8da5be6/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-logr/zapr v0.1.1/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= github.com/go-logr/zapr v0.3.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= -github.com/go-logr/zapr v0.4.0 h1:uc1uML3hRYL9/ZZPdgHS/n8Nzo+eaYL/Efxkkamf7OM= github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= +github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= +github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.17.2/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -600,6 +606,7 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -649,6 +656,8 @@ github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Z github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/cel-go v0.9.0/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= +github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -853,8 +862,9 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jsonnet-bundler/jsonnet-bundler v0.1.0/go.mod h1:YKsSFc9VFhhLITkJS3X2PrRqWG9u2Jq99udTdDjQLfM= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -913,6 +923,7 @@ github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -972,6 +983,7 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= @@ -982,8 +994,9 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= @@ -1032,8 +1045,9 @@ github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9k github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -1048,8 +1062,9 @@ github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= -github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/open-policy-agent/cert-controller v0.0.0-20200921224206-24b87bbc4b6e/go.mod h1:/y33mmiq3Cc0N+6cickevrLI/iBbWcUwcEVjSKHA0z0= github.com/open-policy-agent/frameworks/constraint v0.0.0-20200929072634-d96896eff389/go.mod h1:Dr3QxvH+NTQcPPZWSt1ueNOsxW4VwgUltaLL7Ttnrac= @@ -1099,6 +1114,7 @@ github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUr github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= @@ -1157,8 +1173,9 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.25.0/go.mod h1:H6QK/N6XVT42whUeIdI3dp36w49c+/iMDk7UAI2qm7Q= -github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.28.0 h1:vGVfV9KrDTvWt5boZO0I19g2E3CsWfpPPKZM9dt3mEw= +github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1240,6 +1257,7 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.3.2/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -1252,6 +1270,7 @@ github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHN github.com/spf13/cobra v1.0.1-0.20200713175500-884edc58ad08/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -1264,6 +1283,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stormcat24/protodep v0.0.0-20200505140716-b02c9ba62816/go.mod h1:mBd5PI4uI6NkqJpCyiWiYzWyTFs4QRDss/JTMC2b4kc= @@ -1327,6 +1347,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.etcd.io/bbolt v1.3.1-etcd.7/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -1383,8 +1404,10 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= @@ -1401,8 +1424,9 @@ go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.19.0 h1:mZQZefskPPCMIBCSEH0v2/iUqqLrYtaeqwD6FUGUnFE= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go4.org v0.0.0-20200104003542-c7e774b10ea0/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= @@ -1440,6 +1464,7 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8= golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1471,7 +1496,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -1550,7 +1574,10 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de h1:pZB1TWnKi+o4bENlbzAgLrEbY4RMYmUIRobMcSmfeYc= @@ -1568,6 +1595,7 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= @@ -1691,9 +1719,12 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1704,8 +1735,8 @@ golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 h1:eJv7u3ksNXoLbGSKuv2s/SIO4 golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1821,8 +1852,8 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1862,6 +1893,7 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= @@ -1932,6 +1964,7 @@ google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201026171402-d4b8fe4fd877/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201030142918-24207fddd1c3/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201102152239-715cce707fb0/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201106154455-f9bfe239b0ba/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -2055,8 +2088,9 @@ gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.46.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.52.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= @@ -2136,8 +2170,10 @@ k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= k8s.io/api v0.19.4/go.mod h1:SbtJ2aHCItirzdJ36YslycFNzWADYH3tgOhvBEFtZAk= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= -k8s.io/api v0.22.2 h1:M8ZzAD0V6725Fjg53fKeTJxGsJvRbk4TEm/fexHMtfw= k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= +k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8= +k8s.io/api v0.23.6 h1:yOK34wbYECH4RsJbQ9sfkFK3O7f/DUHRlzFehkqZyVw= +k8s.io/api v0.23.6/go.mod h1:1kFaYxGCFHYp3qd6a85DAj/yW8aVD6XLZMqJclkoi9g= k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY= k8s.io/apiextensions-apiserver v0.0.0-20190918201827-3de75813f604/go.mod h1:7H8sjDlWQu89yWB3FhZfsLyRCRLuoXoCoY5qtwW1q6I= k8s.io/apiextensions-apiserver v0.16.4/go.mod h1:HYQwjujEkXmQNhap2C9YDdIVOSskGZ3et0Mvjcyjbto= @@ -2151,8 +2187,10 @@ k8s.io/apiextensions-apiserver v0.19.2/go.mod h1:EYNjpqIAvNZe+svXVx9j4uBaVhTB4C9 k8s.io/apiextensions-apiserver v0.19.4/go.mod h1:B9rpH/nu4JBCtuUp3zTTk8DEjZUupZTBEec7/2zNRYw= k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= k8s.io/apiextensions-apiserver v0.20.2/go.mod h1:F6TXp389Xntt+LUq3vw6HFOLttPa0V8821ogLGwb6Zs= -k8s.io/apiextensions-apiserver v0.22.2 h1:zK7qI8Ery7j2CaN23UCFaC1hj7dMiI87n01+nKuewd4= k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA= +k8s.io/apiextensions-apiserver v0.23.5/go.mod h1:ntcPWNXS8ZPKN+zTXuzYMeg731CP0heCTl6gYBxLcuQ= +k8s.io/apiextensions-apiserver v0.23.6 h1:v58cQ6Z0/GK1IXYr+oW0fnYl52o9LTY0WgoWvI8uv5Q= +k8s.io/apiextensions-apiserver v0.23.6/go.mod h1:YVh17Mphv183THQJA5spNFp9XfoidFyL3WoDgZxQIZU= k8s.io/apimachinery v0.0.0-20181015213631-60666be32c5d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/apimachinery v0.0.0-20181110190943-2a7c93004028/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= k8s.io/apimachinery v0.0.0-20190703205208-4cfb76a8bf76/go.mod h1:M2fZgZL9DbLfeJaPBCDqSqNsdsmLN+V29knYJnIXlMA= @@ -2174,8 +2212,10 @@ k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlm k8s.io/apimachinery v0.19.4/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.22.2 h1:ejz6y/zNma8clPVfNDLnPbleBo6MpoFy/HBiBqCouVk= k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= +k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= +k8s.io/apimachinery v0.23.6 h1:RH1UweWJkWNTlFx0D8uxOpaU1tjIOvVVWV/bu5b3/NQ= +k8s.io/apimachinery v0.23.6/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg= k8s.io/apiserver v0.0.0-20190918200908-1e17798da8c1/go.mod h1:4FuDU+iKPjdsdQSN3GsEKZLB/feQsj1y9dhhBDVV2Ns= k8s.io/apiserver v0.16.4/go.mod h1:kbLJOak655g6W7C+muqu1F76u9wnEycfKMqbVaXIdAc= @@ -2191,11 +2231,13 @@ k8s.io/apiserver v0.19.4/go.mod h1:X8WRHCR1UGZDd7HpV0QDc1h/6VbbpAeAGyxSh8yzZXw= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.2/go.mod h1:2nKd93WyMhZx4Hp3RfgH2K5PhwyTrprrkWYnI7id7jA= k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= +k8s.io/apiserver v0.23.5/go.mod h1:7wvMtGJ42VRxzgVI7jkbKvMbuCbVbgsWFT7RyXiRNTw= +k8s.io/apiserver v0.23.6/go.mod h1:5PU32F82tfErXPmf7FXhd/UcuLfh97tGepjKUgJ2atg= k8s.io/autoscaler v0.0.0-20190218140445-7f77136aeea4/go.mod h1:QEXezc9uKPT91dwqhSJq3GNI3B1HxFRQHiku9kmrsSA= k8s.io/cli-runtime v0.19.0/go.mod h1:tun9l0eUklT8IHIM0jors17KmUjcrAxn0myoBYwuNuo= k8s.io/cli-runtime v0.19.4/go.mod h1:m8G32dVbKOeaX1foGhleLEvNd6REvU7YnZyWn5//9rw= -k8s.io/client-go v0.22.2 h1:DaSQgs02aCC1QcwUdkKZWOeaVsQjYvWv8ZazcZ6JcHc= -k8s.io/client-go v0.22.2/go.mod h1:sAlhrkVDf50ZHx6z4K0S40wISNTarf1r800F+RlCF6U= +k8s.io/client-go v0.23.6 h1:7h4SctDVQAQbkHQnR4Kzi7EyUyvla5G1pFWf4+Od7hQ= +k8s.io/client-go v0.23.6/go.mod h1:Umt5icFOMLV/+qbtZ3PR0D+JA6lvvb3syzodv4irpK4= k8s.io/cloud-provider v0.17.0/go.mod h1:Ze4c3w2C0bRsjkBUoHpFi+qWe3ob1wI2/7cUn+YQIDE= k8s.io/code-generator v0.0.0-20181114232248-ae218e241252/go.mod h1:IPqxl/YHk05nodzupwjke6ctMjyNRdV2zZ5/j3/F204= k8s.io/code-generator v0.0.0-20190612205613-18da4a14b22b/go.mod h1:G8bQwmHm2eafm5bgtX67XDZQ8CWKSGu9DekI+yN4Y5I= @@ -2215,6 +2257,8 @@ k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbW k8s.io/code-generator v0.20.2/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.4/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= +k8s.io/code-generator v0.23.5/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= +k8s.io/code-generator v0.23.6/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA= k8s.io/component-base v0.0.0-20190918200425-ed2f0867c778/go.mod h1:DFWQCXgXVLiWtzFaS17KxHdlUeUymP7FLxZSkmL9/jU= k8s.io/component-base v0.16.4/go.mod h1:GYQ+4hlkEwdlpAp59Ztc4gYuFhdoZqiAJD1unYDJ3FM= @@ -2229,8 +2273,10 @@ k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agE k8s.io/component-base v0.19.4/go.mod h1:ZzuSLlsWhajIDEkKF73j64Gz/5o0AgON08FgRbEPI70= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= -k8s.io/component-base v0.22.2 h1:vNIvE0AIrLhjX8drH0BgCNJcR4QZxMXcJzBsDplDx9M= k8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug= +k8s.io/component-base v0.23.5/go.mod h1:c5Nq44KZyt1aLl0IpHX82fhsn84Sb0jjzwjpcA42bY0= +k8s.io/component-base v0.23.6 h1:8dhVZ4VrRcNdV2EGjl8tj8YOHwX6ysgCGMJ2Oyy0NW8= +k8s.io/component-base v0.23.6/go.mod h1:FGMPeMrjYu0UZBSAFcfloVDplj9IvU+uRMTOdE23Fj0= k8s.io/csi-translation-lib v0.17.0/go.mod h1:HEF7MEz7pOLJCnxabi45IPkhSsE/KmxPQksuCrHKWls= k8s.io/gengo v0.0.0-20181106084056-51747d6e00da/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -2243,6 +2289,7 @@ k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8 k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v0.0.0-20190306015804-8e90cee79f82/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= @@ -2254,8 +2301,9 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.3.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw= +k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-aggregator v0.19.0/go.mod h1:1Ln45PQggFAG8xOqWPIYMxUq8WNtpPnYsbUJ39DpF/A= k8s.io/kube-aggregator v0.19.4/go.mod h1:cTkvun110194d797AuThyydBBlgm+cKIFUeS2uzGJfU= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= @@ -2263,8 +2311,9 @@ k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAG k8s.io/kubectl v0.19.0/go.mod h1:gPCjjsmE6unJzgaUNXIFGZGafiUp5jh0If3F/x7/rRg= k8s.io/kubectl v0.19.4/go.mod h1:XPmlu4DJEYgD83pvZFeKF8+MSvGnYGqunbFSrJsqHv0= k8s.io/kubelet v0.19.4/go.mod h1:zJnPeb7nJCRvtAwxJhe9fFCtMLXL3cXbQiczPmpDrLU= -k8s.io/kubelet v0.22.2 h1:7ol5AXXxcW97dUE8W/QiPjkXu1ZuGshG5VmgDmviZsc= k8s.io/kubelet v0.22.2/go.mod h1:ORIRua2/wTcx5UnEvxWosu650/8fatmzbMRC7m6WjAM= +k8s.io/kubelet v0.23.6 h1:tuscMqYCt9cxWursmTU9OJ2tPLv66Ji+AGbuV1Z/lug= +k8s.io/kubelet v0.23.6/go.mod h1:ROttmKIUkB9in4NyX/SfnAoXGfW/Dju3VCGFP34F5ac= k8s.io/legacy-cloud-providers v0.17.0/go.mod h1:DdzaepJ3RtRy+e5YhNtrCYwlgyK87j/5+Yfp0L9Syp8= k8s.io/metrics v0.19.0/go.mod h1:WykpW8B60OeAJx1imdwUgyOID2kDljr/Q+1zrPJ98Wo= k8s.io/metrics v0.19.4/go.mod h1:a0gvAzrxQPw2ouBqnXI7X9qlggpPkKAFgWU/Py+KZiU= @@ -2282,8 +2331,9 @@ k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a h1:8dYfu/Fc9Gz2rNJKB9IQRGgQOh2clmRzNIPPY1xLY5g= k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE= +k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= knative.dev/caching v0.0.0-20190719140829-2032732871ff/go.mod h1:dHXFU6CGlLlbzaWc32g80cR92iuBSpsslDNBWI8C7eg= knative.dev/eventing-contrib v0.6.1-0.20190723221543-5ce18048c08b/go.mod h1:SnXZgSGgMSMLNFTwTnpaOH7hXDzTFtw0J8OmHflNx3g= knative.dev/pkg v0.0.0-20191111150521-6d806b998379/go.mod h1:pgODObA1dTyhNoFxPZTTjNWfx6F0aKsKzn+vaT9XO/Q= @@ -2309,6 +2359,7 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= sigs.k8s.io/controller-runtime v0.3.0/go.mod h1:Cw6PkEg0Sa7dAYovGT4R0tRkGhHXpYijwNxYhAnAZZk= sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns= sigs.k8s.io/controller-runtime v0.6.1/go.mod h1:XRYBPdbf5XJu9kpS84VJiZ7h/u1hF3gEORz0efEja7A= @@ -2316,12 +2367,15 @@ sigs.k8s.io/controller-runtime v0.6.2/go.mod h1:vhcq/rlnENJ09SIRp3EveTaZ0yqH526h sigs.k8s.io/controller-runtime v0.6.3/go.mod h1:WlZNXcM0++oyaQt4B7C2lEE5JYRs8vJUzRP4N4JpdAY= sigs.k8s.io/controller-runtime v0.7.0/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= -sigs.k8s.io/controller-runtime v0.10.2 h1:jW8qiY+yMnnPx6O9hu63tgcwaKzd1yLYui+mpvClOOc= sigs.k8s.io/controller-runtime v0.10.2/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= +sigs.k8s.io/controller-runtime v0.11.2 h1:H5GTxQl0Mc9UjRJhORusqfJCIjBO8UtUxGggCwL1rLA= +sigs.k8s.io/controller-runtime v0.11.2/go.mod h1:P6QCzrEjLaZGqHsfd+os7JQ+WFZhvB8MRFsn4dWF7O4= sigs.k8s.io/controller-tools v0.2.4/go.mod h1:m/ztfQNocGYBgTTCmFdnK94uVvgxeZeE3LtJvd/jIzA= sigs.k8s.io/controller-tools v0.2.9-0.20200414181213-645d44dca7c0/go.mod h1:YKE/iHvcKITCljdnlqHYe+kAt7ZldvtAwUzQff0k1T0= sigs.k8s.io/controller-tools v0.4.1/go.mod h1:G9rHdZMVlBDocIxGkK3jHLWqcTMNvveypYJwrvYKjWU= sigs.k8s.io/controller-tools v0.5.0/go.mod h1:JTsstrMpxs+9BUj6eGuAaEb6SDSPTeVtUyp0jmnAM/I= +sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s= +sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= @@ -2332,13 +2386,15 @@ sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnM sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= software.sslmate.com/src/go-pkcs12 v0.0.0-20180114231543-2291e8f0f237/go.mod h1:/xvNRWUqm0+/ZMiF4EX00vrSCMsE4/NHb+Pt3freEeQ= software.sslmate.com/src/go-pkcs12 v0.0.0-20200830195227-52f69702a001/go.mod h1:/xvNRWUqm0+/ZMiF4EX00vrSCMsE4/NHb+Pt3freEeQ= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml index eb3c59a01..4d153c419 100644 --- a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml @@ -328,7 +328,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml index 12f9ae2dc..a02201885 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml @@ -325,7 +325,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml index bcee3bc4d..485632d9b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml @@ -325,7 +325,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml index fd0294c00..02d52a5a5 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml @@ -325,7 +325,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml index de7f75839..b8290b7cd 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -342,7 +342,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml index e7bec561a..ccc154543 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -342,7 +342,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml index cbac1694e..390fc9784 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml @@ -333,7 +333,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml index 1040b8d0d..581b817e6 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml @@ -325,7 +325,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml index aa7340673..cf51102d6 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml @@ -323,7 +323,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml index 482b6fb28..3b61e8445 100644 --- a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml @@ -334,7 +334,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml index bda03b455..0fdc64ab5 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml @@ -335,7 +335,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml index 07c0f2b8d..4887f0bd5 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml @@ -335,7 +335,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml index 94bed7d94..9e25af45f 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml @@ -335,7 +335,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml index 534abb608..711776a8f 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml @@ -343,7 +343,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 867c272f9..d7662b439 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -352,7 +352,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml index 61d4bd4a9..a67e5e54c 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -352,7 +352,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml index 6f7891bc8..18171adfe 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml @@ -343,7 +343,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index e7c800083..4a161f747 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -335,7 +335,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index b98a62fd8..c7d8b6200 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -333,7 +333,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml index a1485c7cc..9bb1f3186 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml @@ -195,7 +195,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml index f773b158f..4a23fb10b 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml @@ -195,7 +195,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml index 52ca397e4..30c242335 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml @@ -195,7 +195,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml index 78e44ad4f..4a2251c00 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml @@ -193,7 +193,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index b24e425d1..2a5d8328e 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -178,7 +178,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json b/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json index 6c7dd9a99..dcd22147b 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.14%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.14%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json b/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json index 737e8c4f6..3fa405037 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.8%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.8%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json b/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json index d1a95c62a..80f09bdbc 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json index 9867446fa..6e5a4c226 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%20%7B%7D%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml index 41d1fc4a5..05ccaed17 100644 --- a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml @@ -329,7 +329,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml index 7236a9128..3b8dd9824 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml @@ -330,7 +330,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml index a94fb37f5..a17711a82 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml @@ -330,7 +330,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index e7e4c7751..a527bf430 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -330,7 +330,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml index af7856f57..695b02ba3 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml @@ -339,7 +339,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index 30e6d0b07..4dd5ca128 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -328,7 +328,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index a9773509f..db44f8876 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -328,7 +328,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index e25b602bf..e5368bc62 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -346,7 +346,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index d620aed6b..9f899a025 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -346,7 +346,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index a108bc213..b067a10b5 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -337,7 +337,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index f759a4f54..ab6b831a3 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -338,7 +338,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml index 79fe88184..575c8915d 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml @@ -329,7 +329,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml index 53065f046..7cb5b1ca4 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml @@ -330,7 +330,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml index c8ad36585..f5bc91a24 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml @@ -330,7 +330,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml index 8e146adca..abb4be043 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml @@ -330,7 +330,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml index f47d681e2..9b53ab40d 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml @@ -338,7 +338,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml index da9ebe7b2..30d48ac51 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -347,7 +347,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml index 08e5327bd..b96a049a2 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -347,7 +347,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml index ac0a25e90..3736aee85 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml @@ -338,7 +338,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml index 4b6609ff9..e336d03bc 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml @@ -330,7 +330,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml index f4712aa93..65a534e8b 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml @@ -328,7 +328,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml index faa9f488f..99ceebed7 100644 --- a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml @@ -343,7 +343,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml index deadd8625..54e826d01 100644 --- a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml @@ -341,7 +341,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml index f2c31b733..c5dcbd600 100644 --- a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml @@ -343,7 +343,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml index 02ea10f5f..e22248039 100644 --- a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml @@ -343,7 +343,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml index c3eb299b1..8858a7ce1 100644 --- a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml @@ -345,7 +345,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/sles/testdata/openstack.yaml b/pkg/userdata/sles/testdata/openstack.yaml index 3cfb660d2..f67ac1ba0 100644 --- a/pkg/userdata/sles/testdata/openstack.yaml +++ b/pkg/userdata/sles/testdata/openstack.yaml @@ -345,7 +345,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/sles/testdata/version-1.20.14.yaml b/pkg/userdata/sles/testdata/version-1.20.14.yaml index 68903a0f9..70ed85fd0 100644 --- a/pkg/userdata/sles/testdata/version-1.20.14.yaml +++ b/pkg/userdata/sles/testdata/version-1.20.14.yaml @@ -341,7 +341,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/sles/testdata/version-1.21.8.yaml b/pkg/userdata/sles/testdata/version-1.21.8.yaml index b4dbc3252..53c2958d0 100644 --- a/pkg/userdata/sles/testdata/version-1.21.8.yaml +++ b/pkg/userdata/sles/testdata/version-1.21.8.yaml @@ -341,7 +341,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/sles/testdata/version-1.22.5.yaml b/pkg/userdata/sles/testdata/version-1.22.5.yaml index deadd8625..54e826d01 100644 --- a/pkg/userdata/sles/testdata/version-1.22.5.yaml +++ b/pkg/userdata/sles/testdata/version-1.22.5.yaml @@ -341,7 +341,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/sles/testdata/version-1.23.0.yaml b/pkg/userdata/sles/testdata/version-1.23.0.yaml index 966a28453..a24f95948 100644 --- a/pkg/userdata/sles/testdata/version-1.23.0.yaml +++ b/pkg/userdata/sles/testdata/version-1.23.0.yaml @@ -339,7 +339,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml index 260a60994..ed88682b4 100644 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml @@ -356,7 +356,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml index 88c66dd8b..cafb9cfd9 100644 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/sles/testdata/vsphere-proxy.yaml @@ -356,7 +356,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml index e4b0ff840..5e377e19b 100644 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ b/pkg/userdata/sles/testdata/vsphere.yaml @@ -346,7 +346,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index b7abc275e..d451999dc 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -421,7 +421,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index b9d497f8d..22f215094 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -394,7 +394,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index cfab1a8dd..470ae4581 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -392,7 +392,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index f819a1ff7..cdef7de4e 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -394,7 +394,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 655a9101a..067868e0e 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -394,7 +394,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index cd0bc21fa..41ca26fff 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -399,7 +399,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index abea1da1e..9bf32e2e6 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -396,7 +396,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 4fb4d8f52..f4ca3a745 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -396,7 +396,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml b/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml index fd1ee3dc2..4b5e69de5 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml @@ -392,7 +392,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml b/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml index 2019f39a8..3f914dbf5 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml @@ -392,7 +392,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml index cfab1a8dd..470ae4581 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml @@ -392,7 +392,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml index 2b0a1c8e5..a304d0815 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml @@ -390,7 +390,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index 13d39f4b5..749469a8c 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -407,7 +407,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 5422d4635..4a767f33a 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -407,7 +407,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index ccfaf8ae8..3f4a95a91 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -397,7 +397,12 @@ write_files: cpu: 200m ephemeral-storage: 1Gi memory: 200Mi - logging: {} + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s nodeStatusUpdateFrequency: 0s From b2b3c3e64ab74a7384b469e131826b9fc6522690 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Fri, 29 Apr 2022 11:47:45 +0200 Subject: [PATCH 129/489] Update to golang 1.18.1 and corresponding kubermatic/build image (#1268) * Bump prowjobs to golang 1.18.1 and corresponding kubermatic/build image Signed-off-by: Marvin Beckers * Bump Dockerfile and Makefile to Go 1.18.1 as well Signed-off-by: Marvin Beckers * Bump to Go 1.18 Signed-off-by: Marvin Beckers * go mod tidy Signed-off-by: Marvin Beckers --- .prow.yaml | 74 ++--- Dockerfile | 2 +- Makefile | 2 +- go.mod | 2 +- go.sum | 834 ----------------------------------------------------- 5 files changed, 40 insertions(+), 874 deletions(-) diff --git a/.prow.yaml b/.prow.yaml index 89c10d5d8..f9ea92b65 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -8,7 +8,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - make args: @@ -26,7 +26,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - make args: @@ -136,7 +136,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - make args: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -189,7 +189,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -213,7 +213,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -236,7 +236,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -259,7 +259,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -282,7 +282,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -305,7 +305,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -327,7 +327,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -349,7 +349,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -372,7 +372,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -395,7 +395,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -419,7 +419,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -442,7 +442,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -463,7 +463,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -486,7 +486,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -509,7 +509,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -532,7 +532,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -555,7 +555,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -579,7 +579,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -601,7 +601,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -623,7 +623,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -644,7 +644,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -667,7 +667,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -689,7 +689,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -712,7 +712,7 @@ presubmits: preset-rhel: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -734,7 +734,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -756,7 +756,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -778,7 +778,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -801,7 +801,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -824,7 +824,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -846,7 +846,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -868,7 +868,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.17.1 + - image: golang:1.18.1 command: - "./hack/ci-e2e-test.sh" args: @@ -892,7 +892,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.17-node-14-kind-0.11-0 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.12-1 command: - /bin/bash - -c @@ -920,7 +920,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.17-node-14-kind-0.11-0 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.12-1 command: - "./hack/ci-upload-gocache.sh" resources: diff --git a/Dockerfile b/Dockerfile index 633dce168..6af23a884 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.17.1 +ARG GO_VERSION=1.18.1 FROM golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index ba82e967d..a27616783 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.17.1 +GO_VERSION ?= 1.18.1 GOOS ?= $(shell go env GOOS) diff --git a/go.mod b/go.mod index 2d740a79f..297672d94 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/kubermatic/machine-controller -go 1.17 +go 1.18 require ( cloud.google.com/go/logging v1.1.2 diff --git a/go.sum b/go.sum index 3b1fc62c5..7a47f6e2f 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,10 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.30.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.47.0/go.mod h1:5p3Ky/7f3N10VBkhuR5LFtddroTiMyjZV/Kj5qOQFxU= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= @@ -19,7 +15,6 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.71.0/go.mod h1:qZfY4Y7AEIQwG/fQYD3xrxLNkQZ0Xzf3HGeqCkA6LVM= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.73.0/go.mod h1:BkDh9dFvGjCitVw03TNjKbBxXNKULXXIq6orU6HrJ4Q= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= @@ -47,7 +42,6 @@ cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6m cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/logging v1.0.0/go.mod h1:V1cc3ogwobYzQq5f2R7DS/GvRIrI4FKj01Gs5glwAls= cloud.google.com/go/logging v1.1.2 h1:KNALX0NZn8UJhqKnqoHxhMqyoZfBZoh5wF7CQJZ5XrU= cloud.google.com/go/logging v1.1.2/go.mod h1:KrljuAHIw631j9+QXsnq9vDwsrwmdxfGpivMR68M7DY= cloud.google.com/go/monitoring v1.4.0 h1:05+IuNMbh40hbxcqQ4SnynbwZbLG1Wc9dysIJxnfv7U= @@ -61,42 +55,17 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -code.cloudfoundry.org/go-pubsub v0.0.0-20180503211407-becd51dc37cb/go.mod h1:2mohpzdn59JWHT85lXjjglNpGLF51tk6hHqfxpc0utk= -contrib.go.opencensus.io/exporter/ocagent v0.4.12/go.mod h1:450APlNTSR6FrvC3CTRqYosuDstRB9un7SOx2k/9ckA= -contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= -contrib.go.opencensus.io/exporter/stackdriver v0.12.8/go.mod h1:XyyafDnFOsqoxHJgTFycKZMrRUrPThLh2iYTJF6uoO0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= -github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= -github.com/Azure/azure-sdk-for-go v28.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v35.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v38.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v46.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v49.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v62.0.0+incompatible h1:8N2k27SYtc12qj5nTsuFMFJPZn5CGmgMWqTy4y9I7Jw= github.com/Azure/azure-sdk-for-go v62.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-storage-blob-go v0.0.0-20190123011202-457680cc0804/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.1.0/go.mod h1:AKyIcETwSUFxIcs/Wnq/C+kwCtlEYGUVd7FPNb2slmg= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= -github.com/Azure/go-autorest/autorest v0.9.5/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= -github.com/Azure/go-autorest/autorest v0.11.6/go.mod h1:V6p3pKZx1KKkJubbxnDWrzNhEIfOy/pTGasLqzHIPHs= github.com/Azure/go-autorest/autorest v0.11.13/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest/adal v0.1.0/go.mod h1:MeS4XhScH55IST095THyTxElntu7WqB7pNbZo8Q5G3E= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= -github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.9.4/go.mod h1:/3SMAM86bP6wC9Ev35peQDUeqFZBMH07vvUOmg4z/fE= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.8/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= @@ -105,53 +74,27 @@ github.com/Azure/go-autorest/autorest/azure/auth v0.5.5 h1:7HT2JTm2BOsBMPrT1/iWZ github.com/Azure/go-autorest/autorest/azure/auth v0.5.5/go.mod h1:ptW4D47I+eIUe/lulFLYTVfG4rAARZoXIe1vmTQ+ol8= github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY= github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= -github.com/Azure/go-autorest/autorest/to v0.1.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc= -github.com/Azure/go-autorest/autorest/to v0.2.0/go.mod h1:GunWKJp1AEqgMaGLV+iocmRAJWqST1wQYhyyjXJ3SJc= -github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= -github.com/Azure/go-autorest/autorest/validation v0.1.0/go.mod h1:Ha3z/SqBeaalWQvokg3NZAlQTalVMtOIAs1aGK7G6u8= -github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI= -github.com/Azure/go-autorest/autorest/validation v0.3.0/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.1.0/go.mod h1:ROEEAFwXycQw7Sn3DXNtEedEvdeRAgDr0izn4z5Ij88= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.0/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/GoogleCloudPlatform/cloud-builders/gcs-fetcher v0.0.0-20191203181535-308b93ad1f39/go.mod h1:yfGmCjKuUzk9WzubMlW2zwjhCraIc/J+M40cufdemRM= -github.com/GoogleCloudPlatform/k8s-cloud-provider v0.0.0-20190822182118-27a4ced34534/go.mod h1:iroGtC8B3tQiqtds1l+mgk/BBOrxbqjH+eUfFQYRc14= -github.com/GoogleCloudPlatform/testgrid v0.0.1-alpha.4/go.mod h1:f96W2HYy3tiBNV5zbbRc+NczwYHgG1PHXMQfoEWv680= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= -github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Masterminds/semver/v3 v3.1.0/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Masterminds/sprig/v3 v3.1.0/go.mod h1:ONGMf7UfYGAbMXCZmQLy8x3lCDIPrEZE/rU8pmrbihA= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= @@ -160,22 +103,14 @@ github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXn github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OneOfOne/xxhash v1.2.7/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= -github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/Venafi/vcert/v4 v4.11.0/go.mod h1:OE+UZ0cj8qqVUuk0u7R4GIk4ZB6JMSf/WySqnBPNwws= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/a8m/expect v1.0.0/go.mod h1:4IwSCMumY49ScypDnjNbYEjgVeqy1/U2cEs3Lat96eA= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= -github.com/ajeddeloh/go-json v0.0.0-20170920214419-6a2fe990e083/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 h1:4SPQljF/GJ8Q+QlCWMWxRBepub4DresnOm4eI2ebFGc= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd h1:NlKlOv3aVJ5ODMC0JWPvddw05KENkL3cZttIuu8kJRo= @@ -190,9 +125,6 @@ github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRY github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.61.751 h1:PX0jCn9kBBgaybsFltpmQ8F7O74hQXY/3yNyCjInDag= github.com/aliyun/alibaba-cloud-sdk-go v1.61.751/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/andygrunwald/go-gerrit v0.0.0-20190120104749-174420ebee6c/go.mod h1:0iuRQp6WJ44ts+iihy5E/WlPqfg5RNeQxOmzRkxCdtk= -github.com/anexia-it/go-anxcloud v0.3.8/go.mod h1:cevqezsbOJ4GBlAWaztfLKl9w4VzxJBt4ipgHORi3gw= github.com/anexia-it/go-anxcloud v0.3.26 h1:uStosj8srS6OA1OsPsMJBFqd4Znzl6fEhUv8b3+G8FU= github.com/anexia-it/go-anxcloud v0.3.26/go.mod h1:fiEBxEtBXx78/OWBJvL7+2o4TESrnEcrDYjLeonGkDw= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= @@ -200,7 +132,6 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apoydence/onpar v0.0.0-20200406201722-06f95a1c68e8/go.mod h1:maauOJD0kdDqIz4xmkunipFVbBoTM6pFSy0kkWBcIUY= github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30/go.mod h1:4AJxUpXUhv4N+ziTvIcWWXgeorXpxPZOfk9HdEVr96M= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -208,25 +139,12 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= -github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= -github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= -github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= -github.com/aws/aws-k8s-tester v0.0.0-20190114231546-b411acf57dfe/go.mod h1:1ADF5tAtU1/mVtfMcHAYSm2fPw71DA7fFk0yed64/0I= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.16.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.16.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.23.20/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.27.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.27.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= -github.com/aws/aws-sdk-go v1.34.30/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.36.2 h1:UAeFPct+jHqWM+tgiqDrC9/sfbWj6wkcvpsJ+zdcsvA= github.com/aws/aws-sdk-go v1.36.2/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/bazelbuild/buildtools v0.0.0-20190917191645-69366ca98f89/go.mod h1:5JP0TXzWDHXv8qvxRC4InIazwdyDseBDbzESUMKk1yU= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -236,31 +154,21 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= -github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/brancz/gojsontoyaml v0.0.0-20190425155809-e8bd32d46b3d/go.mod h1:IyUJYN1gvWjtLF5ZuygmxbnsAyP3aJS6cHzIuZY50B0= github.com/briandowns/spinner v1.8.0/go.mod h1://Zf9tMcxfRUA36V23M6YGEAv+kECGfvpnLTnb8n4XQ= -github.com/bwmarrin/snowflake v0.0.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= -github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/clarketm/json v1.13.4/go.mod h1:ynr2LRfb0fQU34l07csRNBTcivjySLLiY1YzQqKVfdo= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudevents/sdk-go v0.0.0-20190509003705-56931988abe3/go.mod h1:j1nZWMLGg3om8SswStBoY6/SHvcLM19MuZqwDtMtmzs= -github.com/cloudflare/cloudflare-go v0.13.2/go.mod h1:27kfc1apuifUmJhp069y0+hwlKDg4bd8LWlu7oKeZvM= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -274,25 +182,15 @@ github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/container-linux-config-transpiler v0.9.0 h1:UBGpT8qWqzi48hNLrzMAgAUNJsR0LW8Gk5/dR/caI8U= github.com/coreos/container-linux-config-transpiler v0.9.0/go.mod h1:SlcxXZQ2c42knj8pezMiQsM1f+ADxFMjGetuMKR/YSQ= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.17+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-oidc v0.0.0-20180117170138-065b426bd416/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.0.0-20180108230905-e214231b295a/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -300,59 +198,42 @@ github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/ignition v0.35.0 h1:UFodoYq1mOPrbEjtxIsZbThcDyQwAI1owczRDqWmKkQ= github.com/coreos/ignition v0.35.0/go.mod h1:WJQapxzEn9DE0ryxsGvm8QnBajm/XsS/PkrDqSpz+bA= -github.com/coreos/locksmith v0.6.2/go.mod h1:mSLRr7SVSEAIugjic7+TXif/+ZQQq0zCks1vptuj2fs= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/prometheus-operator v0.35.0/go.mod h1:XHYZUStZWcwd1yk/1DjZv/fywqKIyAJ6pSwvIr+v9BQ= -github.com/cpu/goacmedns v0.0.3/go.mod h1:4MipLkI+qScwqtVxcNO6okBhbgRrr7/tKXUSgSL0teQ= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cristim/ec2-instances-info v0.0.0-20201110114654-2dfcc09f67d4/go.mod h1:0yCjO4zBzlwWSGh/zGfW2Zq1NX605qCYVBHD1fPXKNs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE= -github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1:dv4zxwHi5C/8AeI+4gX4dCWOIvNi7I6JCSX0HvlKPgE= -github.com/denisenkom/go-mssqldb v0.0.0-20190111225525-2fea367d496d/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/digitalocean/godo v1.44.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= github.com/digitalocean/godo v1.54.0 h1:KP0Nv87pgViR8k/7De3VrmflCL5pJqXbNnkcw0bwG10= github.com/digitalocean/godo v1.54.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= -github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8= github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= -github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= @@ -362,7 +243,6 @@ github.com/embik/nutanix-client-go v0.0.0-20220106131900-50b8f27e5f60 h1:0FVKOkp github.com/embik/nutanix-client-go v0.0.0-20220106131900-50b8f27e5f60/go.mod h1:gkKNSxfEt3QtYG3S/wKiN8OmrJ4fpU7JbTlbnrMDOL8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.10.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.11.2+incompatible h1:Z4Z0K2AuOw+QtgwkkJnwpT165MBr12qS8rnBwjP/Pzs= github.com/emicklei/go-restful v2.11.2+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= @@ -376,165 +256,72 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0= -github.com/etcd-io/gofail v0.0.0-20190801230047-ad7f989257ca/go.mod h1:49H/RkXP8pKaZy4h0d+NW16rSLhyVBt4o6VLJbmOqDE= -github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.0.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.1.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= -github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= -github.com/fsouza/fake-gcs-server v0.0.0-20180612165233-e85be23bdaa8/go.mod h1:1/HufuJ+eaDf4KTnYdS6HJMGvMRU8d4cYTuu/1QaBbI= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= -github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= -github.com/go-bindata/go-bindata v3.1.2+incompatible/go.mod h1:xK8Dsgwmeed+BBsSy2XTopBn/8uK2HWuGSnA11C3Joo= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.62.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.2.1-0.20200730175230-ee2de8da5be6/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= -github.com/go-logr/zapr v0.1.1/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= -github.com/go-logr/zapr v0.3.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= -github.com/go-logr/zapr v0.4.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= -github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= -github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.17.2/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= -github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= -github.com/go-openapi/analysis v0.19.4/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= -github.com/go-openapi/analysis v0.19.5/go.mod h1:hkEAkxagaIvIP7VTn8ygJNkd4kAYON2rCu0v0ObL0AU= -github.com/go-openapi/analysis v0.19.10/go.mod h1:qmhS3VNFxBlquFJ0RGoDtylO9y4pgTAUNE9AEEMdlJQ= -github.com/go-openapi/analysis v0.19.16/go.mod h1:GLInF007N83Ad3m8a/CbQ5TPzdnGT7workfHwuVjNVk= -github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.17.2/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= -github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= -github.com/go-openapi/errors v0.19.3/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= -github.com/go-openapi/errors v0.19.6/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.19.7/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/inflect v0.19.0/go.mod h1:lHpZVlpIQqLyKwJ4N+YSc9hchQy/i12fJykb83CRBH4= -github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= -github.com/go-openapi/jsonpointer v0.19.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= -github.com/go-openapi/jsonreference v0.19.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.4/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.17.2/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= -github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= -github.com/go-openapi/loads v0.19.3/go.mod h1:YVfqhUCdahYwR3f3iiwQLhicVRvLlU/WO5WPaZvcvSI= -github.com/go-openapi/loads v0.19.4/go.mod h1:zZVHonKd8DXyxyw4yfnVjPzBjIQcLt0CCsn0N0ZrQsk= -github.com/go-openapi/loads v0.19.5/go.mod h1:dswLCAdonkRufe/gSUC3gN8nTSaB9uaS2es0x5/IbjY= -github.com/go-openapi/loads v0.19.6/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= -github.com/go-openapi/loads v0.19.7/go.mod h1:brCsvE6j8mnbmGBh103PT/QLHfbyDxA4hsKvYBNEGVc= -github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= -github.com/go-openapi/runtime v0.17.2/go.mod h1:QO936ZXeisByFmZEO1IS1Dqhtf4QV1sYYFtIq6Ld86Q= -github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= -github.com/go-openapi/runtime v0.19.4/go.mod h1:X277bwSUBxVlCYR3r7xgZZGKVvBd/29gLDlFGtJ8NL4= -github.com/go-openapi/runtime v0.19.15/go.mod h1:dhGWCTKRXlAfGnQG0ONViOZpjfg0m2gUt9nTQPQZuoo= -github.com/go-openapi/runtime v0.19.16/go.mod h1:5P9104EJgYcizotuXhEuUrzVc+j1RiSjahULvYmlv98= -github.com/go-openapi/runtime v0.19.20/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= -github.com/go-openapi/runtime v0.19.24/go.mod h1:Lm9YGCeecBnUUkFTxPC4s1+lwrkJ0pthx8YvyjCfkgk= -github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.17.2/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= -github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.19.4/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= -github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= github.com/go-openapi/spec v0.19.15 h1:uxh8miNJEfMm8l8ekpY7i39LcORm1xSRtoipEGl1JPk= github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= -github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= -github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= -github.com/go-openapi/strfmt v0.19.2/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= -github.com/go-openapi/strfmt v0.19.4/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= -github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= -github.com/go-openapi/strfmt v0.19.11/go.mod h1:UukAYgTaQfqJuAFlNxxMWNvMYiwiXtLsF2VwmoFtbtc= -github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.17.2/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= -github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.4/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.7/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= -github.com/go-openapi/swag v0.19.9/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfTe5McPyhY= github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/validate v0.17.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= -github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= -github.com/go-openapi/validate v0.19.3/go.mod h1:90Vh6jjkTn+OT1Eefm0ZixWNFjhtOH7vS9k0lo6zwJo= -github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= -github.com/go-openapi/validate v0.19.10/go.mod h1:RKEZTUWDkxKQxN2jDT7ZnZi2bhZlbNMAuKvKB+IaGx8= -github.com/go-openapi/validate v0.19.12/go.mod h1:Rzou8hA/CBw8donlS6WNEUQupNvUZ0waH08tGe6kAQ4= -github.com/go-openapi/validate v0.19.15/go.mod h1:tbn/fdOwYHgrhPBzidZfJC2MIVvs9GA7monOmWBbeCI= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= @@ -542,64 +329,26 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 h1:JVrqSeQfdhYRFk24TvhTZWU0q8lfCojxZQFi3Ou7+uY= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= -github.com/go-sql-driver/mysql v0.0.0-20160411075031-7ebe0a500653/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-swagger/go-swagger v0.25.0/go.mod h1:9639ioXrPX9E6BbnbaDklGXjNz7upAXoNBwL4Ok11Vk= -github.com/go-swagger/scan-repo-boundary v0.0.0-20180623220736-973b3573c013/go.mod h1:b65mBPzqzZWxOZGxSWrqs4GInLIn+u99Q9q7p+GKni0= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= -github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= -github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= -github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.5/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9kaifseQ= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/envy v1.7.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w= -github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.5/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= -github.com/gobuffalo/flect v0.2.0/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= github.com/gobuffalo/flect v0.2.2/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= -github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= -github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= -github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kETFqjFoFlOc= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godror/godror v0.13.3/go.mod h1:2ouUT4kdhUBk7TAkHWD4SN0CdI0pgEQbo8FVHhbSKWg= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -608,14 +357,12 @@ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2V github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -625,8 +372,6 @@ github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v0.0.0-20181025225059-d3de96c4c28e/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= -github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -646,13 +391,7 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golangplus/bytes v0.0.0-20160111154220-45c989fe5450/go.mod h1:Bk6SMAONeMXrxql8uvOKuAZSu8aM5RUGv+1C6IJaEho= -github.com/golangplus/fmt v0.0.0-20150411045040-2a5d6d7d2995/go.mod h1:lJgMEyOkYFkPcDKwRXegd+iM6E7matEszMG5HhwytU8= -github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= -github.com/gomodule/redigo v1.7.0/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= -github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= @@ -672,31 +411,24 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-containerregistry v0.0.0-20200115214256-379933c9c22b/go.mod h1:Wtl/v6YdQxv397EREtzwgd9+Ud7Q5D8XMbi3Zazgkrs= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-licenses v0.0.0-20191112164736-212ea350c932/go.mod h1:16wa6pRqNDUIhOtwF0GcROVqMeXHZJ7H6eGDFUh5Pfk= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/licenseclassifier v0.0.0-20190926221455-842c0d70d702/go.mod h1:qsqn2hxC+vURpyBRygGUuinTO42MFRLcsmQ/P8v94+M= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201117184057-ae444373da19/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -706,13 +438,9 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v0.0.0-20170306145142-6a5e28554805/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= -github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= -github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -720,142 +448,85 @@ github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0 github.com/googleapis/gax-go/v2 v2.2.0 h1:s7jOdKSaksJVOxE0Y/S32otcfiP+UQ0cL8/GTKaONwE= github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.2.2/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= -github.com/gophercloud/gophercloud v0.14.0/go.mod h1:VX0Ibx85B60B5XOrZr6kaNwrmPUzcmMpwxvQ1WQIIWM= github.com/gophercloud/gophercloud v0.24.0 h1:jDsIMGJ1KZpAjYfQgGI2coNQj5Q83oPzuiGJRFWgMzw= github.com/gophercloud/gophercloud v0.24.0/go.mod h1:Q8fZtyi5zZxPS/j9aj3sSxtvj41AdQMDwyo1myduD5c= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/csrf v1.6.2/go.mod h1:7tSf8kmjNYr7IWDCYhd3U8Ck34iQ/Yw5CJu7bAkHEGI= -github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= -github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= -github.com/gorilla/mux v0.0.0-20181024020800-521ea7b17d02/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/sessions v1.1.3/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170330212424-2500245aa611/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.15.2/go.mod h1:vO11I9oWA+KsxmfFQPhLnnIb1VDE24M+pdxZFiuZcA8= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/h2non/gock v1.0.9/go.mod h1:CZMcB0Lg5IWnr9bF79pPMg9WeV6WumxQiUJ1UvdO1iE= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= -github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= -github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= -github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= -github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= -github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 h1:GT4RsKmHh1uZyhmTkWJTDALRjSHYQp6FRKrotf0zhAs= github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= -github.com/hetznercloud/hcloud-go v1.23.1/go.mod h1:xng8lbDUg+xM1dgc0yGHX5EeqbwIq7UYlMWMTx3SQVg= -github.com/hetznercloud/hcloud-go v1.25.0/go.mod h1:2C5uMtBiMoFr3m7lBFPf7wXTdh33CevmZpQIIDPGYJI= github.com/hetznercloud/hcloud-go v1.33.1 h1:W1HdO2bRLTKU4WsyqAasDSpt54fYO4WNckWYfH5AuCQ= github.com/hetznercloud/hcloud-go v1.33.1/go.mod h1:XX/TQub3ge0yWR2yHWmnDVIrB+MQbda1pHxkUmDlUME= -github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= -github.com/iancoleman/strcase v0.1.2/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.0.0-20171009183408-7fe0c75c13ab/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/improbable-eng/thanos v0.3.2/go.mod h1:GZewVGILKuJVPNRn7L4Zw+7X96qzFOwj63b22xYGXBE= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb v0.0.0-20161215172503-049f9b42e9a5/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= -github.com/jenkins-x/go-scm v1.5.65/go.mod h1:MgGRkJScE/rJ30J/bXYqduN5sDPZqZFITJopsnZmTOw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jetstack/cert-manager v1.1.0/go.mod h1:GULIHTGjSc2LjlgBCLhQ8u5WmQ95hk9FAiQbhjMthMk= -github.com/jinzhu/gorm v0.0.0-20170316141641-572d0a0ab1eb/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= -github.com/jinzhu/inflection v0.0.0-20190603042836-f5c5f50e6090/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/jonboulle/clockwork v0.0.0-20141017032234-72f9bd7c4e0c/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -865,29 +536,17 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/ github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jsonnet-bundler/jsonnet-bundler v0.1.0/go.mod h1:YKsSFc9VFhhLITkJS3X2PrRqWG9u2Jq99udTdDjQLfM= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/k8snetworkplumbingwg/network-attachment-definition-client v0.0.0-20191119172530-79f836b90111/go.mod h1:MP2HbArq3QT+oVp8pmtHNZnSnkhdkHtDnc7h6nJXmBU= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/kelseyhightower/envconfig v1.3.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/kinvolk/container-linux-config-transpiler v0.9.1/go.mod h1:pjTzCvFfbXjWuMVNFjA9FdbsdmruK6+vki0hK0lDmnU= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/knative/build v0.1.2/go.mod h1:/sU74ZQkwlYA5FwYDJhYTy61i/Kn+5eWfln2jDbw3Qo= -github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -897,92 +556,59 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubermatic/machine-controller v1.23.1/go.mod h1:mXWbT7SjqpgFhzCFT3yMEHKdIlT+KkGy4KQCkNRM9Fc= -github.com/kubermatic/machine-controller v1.26.0/go.mod h1:dcJ+GdDSCxCwM0poxwOK8hVO7epiOORDmNMmb2veyw4= -github.com/kubermatic/machine-controller v1.36.1/go.mod h1:6BFZEvEMZi8OT8aHOsS7DXYsF6ZSpmsNxsci7OLTTn8= -github.com/kubermatic/machine-controller v1.40.1/go.mod h1:5LVcN4tCybGg+55hIHcVzCjNsBJy2PlnXG0xIzKmXGY= -github.com/kubermatic/machine-controller v1.42.2/go.mod h1:vr6i5XWfd5FIq2yodXcgdlKvOhMnM5uzn2XEZ2wcoFM= -github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.1-0.20191011153232-f91d3411e481/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linode/linodego v0.24.0 h1:o6hNS0T7jeikOfUHoJhUhA/e2QTCsw9MGccVmRHRLE4= github.com/linode/linodego v0.24.0/go.mod h1:GSBKPpjoQfxEfryoCRcgkuUOCuVtGHWhzI8OMdycNTE= -github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/markbates/inflect v1.0.4/go.mod h1:1fR9+pO2KHEO9ZRtto13gDwwZaAKstQzferVeWqbgNs= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.6/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-oci8 v0.0.7/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= -github.com/mattn/go-runewidth v0.0.0-20181025052659-b20a3daf6a39/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.5/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v0.0.0-20160514122348-38ee283dabf1/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= -github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/hashstructure v0.0.0-20170609045927-2bca23e0e452/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/ioprogress v0.0.0-20180201004757-6a23b12fa88e/go.mod h1:waEya8ee1Ro/lgxpVhkJI4BVASzkm3UZqkx/cFJiYHM= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= @@ -997,132 +623,79 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/munnerz/crd-schema-fuzz v1.0.0/go.mod h1:4z/rcm37JxUkSsExFcLL6ZIT1SgDRdLiu7qq1evdVS0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= -github.com/nats-io/gnatsd v1.4.1/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ= -github.com/nats-io/go-nats v1.7.0/go.mod h1:+t7RHT5ApZebkrQdnn6AhQJmhJJiKAvJUio1PiiCtj0= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.0/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= -github.com/nelsam/hel v0.0.0-20200611165952-2d829bae0c66/go.mod h1:Rl/hm4V2s75ScsPmI9cNz87HLNg5MoFAMJwA90fzbkw= -github.com/nelsam/hel/v2 v2.3.2/go.mod h1:1ZTGfU2PFTOd5mx22i5O0Lc2GY933lQ2wb/ggy+rL3w= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= -github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ= -github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= -github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/open-policy-agent/cert-controller v0.0.0-20200921224206-24b87bbc4b6e/go.mod h1:/y33mmiq3Cc0N+6cickevrLI/iBbWcUwcEVjSKHA0z0= -github.com/open-policy-agent/frameworks/constraint v0.0.0-20200929072634-d96896eff389/go.mod h1:Dr3QxvH+NTQcPPZWSt1ueNOsxW4VwgUltaLL7Ttnrac= -github.com/open-policy-agent/frameworks/constraint v0.0.0-20201118071520-0d37681951a4/go.mod h1:vvhkBONv7Uah2fvS/bQ/N1u0rSLvxZOs2ErR6m+4QtQ= -github.com/open-policy-agent/gatekeeper v0.0.0-20201111000257-4450f08fa95e/go.mod h1:IseSnWz7MX7IhEpZ4CLhA3NrMazc+T6a5rtSq9pOEc4= -github.com/open-policy-agent/opa v0.19.1/go.mod h1:rrwxoT/b011T0cyj+gg2VvxqTtn6N3gp/jzmr3fjW44= -github.com/open-policy-agent/opa v0.21.0/go.mod h1:cZaTfhxsj7QdIiUI0U9aBtOLLTqVNe+XE60+9kZKLHw= -github.com/open-policy-agent/opa v0.24.0/go.mod h1:qEyD/i8j+RQettHGp4f86yjrjvv+ZYia+JHCMv2G7wA= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/openshift/api v0.0.0-20191219222812-2987a591a72c/go.mod h1:dh9o4Fs58gpFXGSYfnVxGR9PnV53I8TW84pQaJDdGiY= -github.com/openshift/client-go v0.0.0-20191125132246-f6563a70e19a/go.mod h1:6rzn+JTr7+WYS2E1TExP4gByoABxMznR6y2SnUIkmxk= -github.com/openshift/custom-resource-status v0.0.0-20190822192428-e62f2f3b79f3/go.mod h1:GDjWl0tX6FNIj82vIxeudWeSx2Ff6nDZ8uJn0ohUFvo= github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca h1:F1MEnOMwSrTA0YAkO0he9ip9w0JhYzI/iCB2mXmaSPg= github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca/go.mod h1:GDjWl0tX6FNIj82vIxeudWeSx2Ff6nDZ8uJn0ohUFvo= -github.com/openshift/prom-label-proxy v0.1.1-0.20191016113035-b8153a7f39f1/go.mod h1:p5MuxzsYP1JPsNGwtjtcgRHHlGziCJJfztff91nNixw= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= -github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.0/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY= -github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= -github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888 h1:ARzSqjQJcSR9IXGMEkXN1IQEZChwd2MqXWm6YNGvO5o= github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888/go.mod h1:RQHg5xR1F614BwJyepfMqrKN+32IH0i7yX+ey43rEeQ= github.com/packethost/pkg v0.0.0-20200903155310-0433e0605550/go.mod h1:GSv7cTtIjns4yc0pyajaM1RE/KE4djJONoblFIRDrxA= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pavel-v-chernykh/keystore-go v2.1.0+incompatible/go.mod h1:xlUlxe/2ItGlQyMTstqeDv9r3U4obH7xYd26TbDQutY= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.3.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= -github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/pelletier/go-toml v1.8.0/go.mod h1:D6yutnOGMveHEPV7VQOuvI/gXY61bv+9bAOTRnLElKs= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= -github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/errors v0.0.0-20181023235946-059132a15dd0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -1132,26 +705,15 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/poy/onpar v0.0.0-20200406201722-06f95a1c68e8/go.mod h1:nSbFQvMj97ZyhFRSJYtut+msi4sOY6zJDGCdSc+/rZU= -github.com/poy/onpar v1.0.1/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/prometheus/client_golang v0.0.0-20181025174421-f30f42803563/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0/7TSWwj+ITvv0TnM= -github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1159,67 +721,45 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= -github.com/prometheus/common v0.25.0/go.mod h1:H6QK/N6XVT42whUeIdI3dp36w49c+/iMDk7UAI2qm7Q= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.28.0 h1:vGVfV9KrDTvWt5boZO0I19g2E3CsWfpPPKZM9dt3mEw= github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/prometheus v2.3.2+incompatible/go.mod h1:oAIUtOny2rjMX0OWN5vPR5/q/twIROJvdqnQKDdil/s= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/prometheus/tsdb v0.8.0/go.mod h1:fSI0j+IUQrDd7+ZtR9WKIGtoYAYAJUKcKhYLG25tN4g= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rcrowley/go-metrics v0.0.0-20190706150252-9beb055b7962/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= -github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg= -github.com/rubiojr/go-vhd v0.0.0-20160810183302-0bfd3b39853c/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7 h1:Do8ksLD4Nr3pA0x0hnLOLftZgkiTDvwPDShRTUxtXpE= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8= -github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -1227,17 +767,8 @@ github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/githubv4 v0.0.0-20180925043049-51d7b505e2e9/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= -github.com/shurcooL/githubv4 v0.0.0-20190718010115-4ba037080260/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= -github.com/shurcooL/githubv4 v0.0.0-20191102174205-af46314aec7b/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo= -github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= -github.com/shurcooL/graphql v0.0.0-20180924043259-e4a3a37e6d42/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= -github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -1249,24 +780,18 @@ github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYl github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.3.2/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.0-20180319062004-c439c4fa0937/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.0-20181021141114-fe5e611709b0/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.0.1-0.20200713175500-884edc58ad08/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= @@ -1274,14 +799,11 @@ github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t6 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v0.0.0-20181024212040-082b515c9490/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= @@ -1301,9 +823,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/tektoncd/pipeline v0.10.1/go.mod h1:D2X0exT46zYx95BU7ByM8+erpjoN7thmUBvlKThOszU= -github.com/tektoncd/plumbing v0.0.0-20191216083742-847dcf196de9/go.mod h1:QZHgU07PRBTRF6N57w4+ApRu8OgfYLFNqCDlfEZaD9Y= -github.com/tektoncd/plumbing/pipelinerun-logs v0.0.0-20191206114338-712d544c2c21/go.mod h1:S62EUWtqmejjJgUMOGB1CCCHRp6C706laH06BoALkzU= github.com/testcontainers/testcontainers-go v0.9.0/go.mod h1:b22BFXhRbg4PJmeMVWh6ftqjyZHgiIl3w274e9r3C2E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda h1:uAHwUH+06gowZMVLqQXm7jN1y3Sl+CDJHThNiKyLHus= @@ -1311,37 +830,19 @@ github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda/go.mod h1:s4k7CORR github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/toqueteos/webbrowser v1.2.0/go.mod h1:XWoZq4cyp9WeUeak7w7LXRUQf1F1ATJMir8RTqb4ayM= -github.com/ugorji/go v1.1.1/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/urfave/cli v1.18.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= -github.com/vdemeester/k8s-pkg-credentialprovider v0.0.0-20200107171650-7c61ffa44238/go.mod h1:JwQJCMWpUDqjZrB5jpw0f5VbN7U95zxFy1ZDpoEarGo= -github.com/vdemeester/k8s-pkg-credentialprovider v1.13.12-1/go.mod h1:Fko0rTxEtDW2kju5Ky7yFJNS3IcNvW8IPsp4/e9oev0= -github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= -github.com/vincent-petithory/dataurl v0.0.0-20160330182126-9a301d65acbb/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50 h1:uxE3GYdXIOfhMv3unJKETJEhw78gvzuQqRX/rVirc2A= github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= -github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= github.com/vmware/govmomi v0.23.1 h1:vU09hxnNR/I7e+4zCJvW+5vHu5dO64Aoe2Lw7Yi/KRg= github.com/vmware/govmomi v0.23.1/go.mod h1:Y+Wq4lst78L85Ge/F8+ORXIWiKYqaro1vhAulACy9Lc= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1/go.mod h1:QcJo0QPSfTONNIgpN5RA8prR7fF8nkF6cTWTcNerRO8= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yashtewari/glob-intersection v0.0.0-20180916065949-5c77d914dd0b/go.mod h1:HptNXiXVDcJjXe9SqMd0v2FsL9f8dz4GnXgltU6q/co= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1349,14 +850,11 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= -go.etcd.io/bbolt v1.3.1-etcd.7/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd v0.0.0-20181031231232-83304cfc808c/go.mod h1:weASp41xM3dk0YHg1s/W8ecdGP5G4teSTMBPpYAaUgA= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200819165624-17cef6e3e9d5/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= @@ -1365,20 +863,12 @@ go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lL go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= -go.etcd.io/etcd/v3 v3.3.0-rc.0.0.20200728214110-6c81b20ec8de/go.mod h1:UENlOa05tkNvLx9VnNziSerG4Ro74upGK6Apd4v6M/Y= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= -go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= -go.mongodb.org/mongo-driver v1.3.5/go.mod h1:Ual6Gkco7ZGQw8wE1t4tLnvBsf6yVSM60qW6TgOeJ5c= -go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= -go.opencensus.io v0.17.0/go.mod h1:mp1VrMQxhlqqDpKvH4UcQUa4YwlzNmymAjPrDdfxNpI= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -1397,7 +887,6 @@ go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1405,65 +894,41 @@ go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= -go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go4.org v0.0.0-20200104003542-c7e774b10ea0/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= -golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181015023909-0c41d7ab0a0e/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191202143827-86a70503ff7e/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201124201722-c8d3bf9c5392/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8= golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -1472,9 +937,7 @@ golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191002040644-a1355ae1e2c3/go.mod h1:NOZ3BPKG0ec/BKJQgnvsSFpcKLM5xXVWnvZS97DWHgE= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -1483,8 +946,6 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20181023182221-1baf3a9d7d67/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1499,7 +960,6 @@ golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20190806162312-597adff16ade/go.mod h1:AlhUtkH4DA4asiFC5RgK7ZKmauvtkAVcy9L0epCzlWo= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -1509,11 +969,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1522,25 +980,17 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190912160710-24e19bdeb0f2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190930134127-c5a3c61f89f3/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191119073136-fc4aabc6c914/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1553,13 +1003,9 @@ golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201016165138-7b1cca2348c0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201026091529-146b70c837a4/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -1570,9 +1016,7 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1583,8 +1027,6 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220325170049-de3da57026de h1:pZB1TWnKi+o4bENlbzAgLrEbY4RMYmUIRobMcSmfeYc= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1608,7 +1050,6 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1617,7 +1058,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1628,46 +1068,28 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190219203350-90b0e4468f99/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190310054646-10058d7d4faa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190912141932-bc967efca4b8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190927073244-c990c680b611/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1686,25 +1108,20 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1712,7 +1129,6 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1720,11 +1136,9 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211029165221-6e7872819dc8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1733,17 +1147,13 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 h1:eJv7u3ksNXoLbGSKuv2s/SIO4tJVxc/A+MTpzxDgz/Q= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -1756,7 +1166,6 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1766,52 +1175,37 @@ golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= -golang.org/x/tools v0.0.0-20190807223507-b346f7fd45de/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190927191325-030b2cf1153e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191010171213-8abd42400456/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112005509-a3f652f18032/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200115165105-de0b1760071a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1822,7 +1216,6 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200313205530-4303120df7d8/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -1832,19 +1225,15 @@ golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200616195046-dc31b401abb5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201017001424-6003fad69a88/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20201030143252-cf7a54d06671/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201105220310-78b158585360/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= @@ -1859,22 +1248,17 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= -google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.0.0-20181021000519-a2651947f503/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.6.1-0.20190607001116-5213b8090861/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.10.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= @@ -1911,32 +1295,23 @@ google.golang.org/api v0.74.0 h1:ExR2D+5TYIrMphWgs5JCgwRhEDlPDXXrLwHHMgPHTXE= google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.2/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto v0.0.0-20170731182057-09f6ed296fc6/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20180608181217-32ee49c4dd80/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181016170114-94acd270e44e/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190708153700-3bdd9d9f5532/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191009194640-548a555dbc03/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= @@ -1969,7 +1344,6 @@ google.golang.org/genproto v0.0.0-20201106154455-f9bfe239b0ba/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201203001206-6486ece9c497/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -2005,12 +1379,8 @@ google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2 google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb h1:0m9wktIpOxGw+SSKmydXWB3Z3GTfcPP6+q75HCQa6HI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/grpc v1.13.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.19.1/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -2019,7 +1389,6 @@ google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -2063,59 +1432,36 @@ google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+Rur google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.0/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.46.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.52.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= -gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= -gopkg.in/natefinch/lumberjack.v2 v2.0.0-20150622162204-20b71e5b60d7/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5/go.mod h1:hiOFpYm0ZJbusNj2ywpbrXowU3G8U6GIQzqn2mw1UIE= -gopkg.in/square/go-jose.v2 v2.0.0-20180411045311-89060dee6a84/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.1/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= -gopkg.in/yaml.v2 v2.1.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -2125,10 +1471,7 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20190709130402-674ba3eaed22/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20190905181640-827449938966/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -2141,206 +1484,60 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.2/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.1.4/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= -k8c.io/kubermatic/v2 v2.16.2/go.mod h1:NdW+2mq4ynRtfZs9yPnvcnFWQpzmM7ngntW6GeuQicU= -k8c.io/operating-system-manager v0.1.0/go.mod h1:ULyZQO1irKjsQTNjIdrHld7SZ+joHjmPnOEs5Db8G8M= -k8c.io/operating-system-manager v0.3.0/go.mod h1:ME5GOCNUrHG+57igEKP1JCJKVHynaLfodT8bRiYH3MY= -k8c.io/operating-system-manager v0.3.9/go.mod h1:aFyB/RH9DBAk0Kj5JVtCixhm9ugTeC8akgRGMW28lPg= k8c.io/operating-system-manager v0.4.0 h1:6F9kxELwHmhqLDLAAlodihBOnSfWM+8FPtbWcOshPGU= k8c.io/operating-system-manager v0.4.0/go.mod h1:pJImhsLb5GJdZunZ47r5Db0ydBwhWxhgL6mUKbU4Vps= -k8s.io/api v0.0.0-20181018013834-843ad2d9b9ae/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= -k8s.io/api v0.0.0-20181115043458-b799cb063522/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/api v0.0.0-20190725062911-6607c48751ae/go.mod h1:1O0xzX/RAtnm7l+5VEUxZ1ysO2ghatfq/OZED4zM9kA= -k8s.io/api v0.0.0-20190918155943-95b840bb6a1f/go.mod h1:uWuOHnjmNrtQomJrvEBg0c0HRNyQ+8KTEERVsK0PW48= -k8s.io/api v0.0.0-20190918195907-bd6ac527cfd2/go.mod h1:AOxZTnaXR/xiarlQL0JUfwQPxjmKDvVYoRp58cA7lUo= -k8s.io/api v0.16.4/go.mod h1:AtzMnsR45tccQss5q8RnF+W8L81DH6XwXwo/joEx9u0= -k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= -k8s.io/api v0.17.2/go.mod h1:BS9fjjLc4CMuqfSO8vgbHPKMt5+SF0ET6u/RVDihTo4= -k8s.io/api v0.18.0/go.mod h1:q2HRQkfDzHMBZL9l/y9rH63PkQl4vae0xRT+8prbrK8= -k8s.io/api v0.18.2/go.mod h1:SJCWI7OLzhZSvbY7U8zwNl9UA4o1fizoug34OV/2r78= -k8s.io/api v0.18.4/go.mod h1:lOIQAKYgai1+vz9J7YcDZwC26Z0zQewYOGWdyIPUUQ4= -k8s.io/api v0.18.5/go.mod h1:tN+e/2nbdGKOAH55NMV8oGrMG+3uRlA9GaRfvnCCSNk= -k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= -k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw= -k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= -k8s.io/api v0.19.4/go.mod h1:SbtJ2aHCItirzdJ36YslycFNzWADYH3tgOhvBEFtZAk= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= -k8s.io/api v0.22.2/go.mod h1:y3ydYpLJAaDI+BbSe2xmGcqxiWHmWjkEeIbiwHvnPR8= -k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8= k8s.io/api v0.23.6 h1:yOK34wbYECH4RsJbQ9sfkFK3O7f/DUHRlzFehkqZyVw= k8s.io/api v0.23.6/go.mod h1:1kFaYxGCFHYp3qd6a85DAj/yW8aVD6XLZMqJclkoi9g= -k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY= -k8s.io/apiextensions-apiserver v0.0.0-20190918201827-3de75813f604/go.mod h1:7H8sjDlWQu89yWB3FhZfsLyRCRLuoXoCoY5qtwW1q6I= -k8s.io/apiextensions-apiserver v0.16.4/go.mod h1:HYQwjujEkXmQNhap2C9YDdIVOSskGZ3et0Mvjcyjbto= -k8s.io/apiextensions-apiserver v0.17.2/go.mod h1:4KdMpjkEjjDI2pPfBA15OscyNldHWdBCfsWMDWAmSTs= -k8s.io/apiextensions-apiserver v0.18.0/go.mod h1:18Cwn1Xws4xnWQNC00FLq1E350b9lUF+aOdIWDOZxgo= -k8s.io/apiextensions-apiserver v0.18.2/go.mod h1:q3faSnRGmYimiocj6cHQ1I3WpLqmDgJFlKL37fC4ZvY= -k8s.io/apiextensions-apiserver v0.18.4/go.mod h1:NYeyeYq4SIpFlPxSAB6jHPIdvu3hL0pc36wuRChybio= -k8s.io/apiextensions-apiserver v0.18.6/go.mod h1:lv89S7fUysXjLZO7ke783xOwVTm6lKizADfvUM/SS/M= -k8s.io/apiextensions-apiserver v0.19.0/go.mod h1:znfQxNpjqz/ZehvbfMg5N6fvBJW5Lqu5HVLTJQdP4Fs= -k8s.io/apiextensions-apiserver v0.19.2/go.mod h1:EYNjpqIAvNZe+svXVx9j4uBaVhTB4C94HkY3w058qcg= -k8s.io/apiextensions-apiserver v0.19.4/go.mod h1:B9rpH/nu4JBCtuUp3zTTk8DEjZUupZTBEec7/2zNRYw= k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= k8s.io/apiextensions-apiserver v0.20.2/go.mod h1:F6TXp389Xntt+LUq3vw6HFOLttPa0V8821ogLGwb6Zs= -k8s.io/apiextensions-apiserver v0.22.2/go.mod h1:2E0Ve/isxNl7tWLSUDgi6+cmwHi5fQRdwGVCxbC+KFA= -k8s.io/apiextensions-apiserver v0.23.5/go.mod h1:ntcPWNXS8ZPKN+zTXuzYMeg731CP0heCTl6gYBxLcuQ= k8s.io/apiextensions-apiserver v0.23.6 h1:v58cQ6Z0/GK1IXYr+oW0fnYl52o9LTY0WgoWvI8uv5Q= k8s.io/apiextensions-apiserver v0.23.6/go.mod h1:YVh17Mphv183THQJA5spNFp9XfoidFyL3WoDgZxQIZU= -k8s.io/apimachinery v0.0.0-20181015213631-60666be32c5d/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= -k8s.io/apimachinery v0.0.0-20181110190943-2a7c93004028/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= -k8s.io/apimachinery v0.0.0-20190703205208-4cfb76a8bf76/go.mod h1:M2fZgZL9DbLfeJaPBCDqSqNsdsmLN+V29knYJnIXlMA= k8s.io/apimachinery v0.0.0-20190719140911-bfcf53abc9f8/go.mod h1:sBJWIJZfxLhp7mRsRyuAE/NfKTr3kXGR1iaqg8O0gJo= -k8s.io/apimachinery v0.0.0-20190816221834-a9f1d8a9c101/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= -k8s.io/apimachinery v0.0.0-20190817020851-f2f3a405f61d/go.mod h1:3jediapYqJ2w1BFw7lAZPCx7scubsTfosqHkhXCWJKw= -k8s.io/apimachinery v0.0.0-20190913080033-27d36303b655/go.mod h1:nL6pwRT8NgfF8TT68DBI8uEePRt89cSvoXUVqbkWHq4= -k8s.io/apimachinery v0.16.4/go.mod h1:llRdnznGEAqC3DcNm6yEj472xaFVfLM7hnYofMb12tQ= -k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.17.1/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.17.2/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= -k8s.io/apimachinery v0.18.0/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= -k8s.io/apimachinery v0.18.2/go.mod h1:9SnR/e11v5IbyPCGbvJViimtJ0SwHG4nfZFjU77ftcA= -k8s.io/apimachinery v0.18.4/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= -k8s.io/apimachinery v0.18.5/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= -k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= -k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= -k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= -k8s.io/apimachinery v0.19.4/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.22.2/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= -k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.23.6 h1:RH1UweWJkWNTlFx0D8uxOpaU1tjIOvVVWV/bu5b3/NQ= k8s.io/apimachinery v0.23.6/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg= -k8s.io/apiserver v0.0.0-20190918200908-1e17798da8c1/go.mod h1:4FuDU+iKPjdsdQSN3GsEKZLB/feQsj1y9dhhBDVV2Ns= -k8s.io/apiserver v0.16.4/go.mod h1:kbLJOak655g6W7C+muqu1F76u9wnEycfKMqbVaXIdAc= -k8s.io/apiserver v0.17.0/go.mod h1:ABM+9x/prjINN6iiffRVNCBR2Wk7uY4z+EtEGZD48cg= -k8s.io/apiserver v0.17.2/go.mod h1:lBmw/TtQdtxvrTk0e2cgtOxHizXI+d0mmGQURIHQZlo= -k8s.io/apiserver v0.18.0/go.mod h1:3S2O6FeBBd6XTo0njUrLxiqk8GNy6wWOftjhJcXYnjw= -k8s.io/apiserver v0.18.2/go.mod h1:Xbh066NqrZO8cbsoenCwyDJ1OSi8Ag8I2lezeHxzwzw= -k8s.io/apiserver v0.18.4/go.mod h1:q+zoFct5ABNnYkGIaGQ3bcbUNdmPyOCoEBcg51LChY8= -k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg= -k8s.io/apiserver v0.19.0/go.mod h1:XvzqavYj73931x7FLtyagh8WibHpePJ1QwWrSJs2CLk= -k8s.io/apiserver v0.19.2/go.mod h1:FreAq0bJ2vtZFj9Ago/X0oNGC51GfubKK/ViOKfVAOA= -k8s.io/apiserver v0.19.4/go.mod h1:X8WRHCR1UGZDd7HpV0QDc1h/6VbbpAeAGyxSh8yzZXw= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.2/go.mod h1:2nKd93WyMhZx4Hp3RfgH2K5PhwyTrprrkWYnI7id7jA= -k8s.io/apiserver v0.22.2/go.mod h1:vrpMmbyjWrgdyOvZTSpsusQq5iigKNWv9o9KlDAbBHI= -k8s.io/apiserver v0.23.5/go.mod h1:7wvMtGJ42VRxzgVI7jkbKvMbuCbVbgsWFT7RyXiRNTw= k8s.io/apiserver v0.23.6/go.mod h1:5PU32F82tfErXPmf7FXhd/UcuLfh97tGepjKUgJ2atg= -k8s.io/autoscaler v0.0.0-20190218140445-7f77136aeea4/go.mod h1:QEXezc9uKPT91dwqhSJq3GNI3B1HxFRQHiku9kmrsSA= -k8s.io/cli-runtime v0.19.0/go.mod h1:tun9l0eUklT8IHIM0jors17KmUjcrAxn0myoBYwuNuo= -k8s.io/cli-runtime v0.19.4/go.mod h1:m8G32dVbKOeaX1foGhleLEvNd6REvU7YnZyWn5//9rw= k8s.io/client-go v0.23.6 h1:7h4SctDVQAQbkHQnR4Kzi7EyUyvla5G1pFWf4+Od7hQ= k8s.io/client-go v0.23.6/go.mod h1:Umt5icFOMLV/+qbtZ3PR0D+JA6lvvb3syzodv4irpK4= -k8s.io/cloud-provider v0.17.0/go.mod h1:Ze4c3w2C0bRsjkBUoHpFi+qWe3ob1wI2/7cUn+YQIDE= -k8s.io/code-generator v0.0.0-20181114232248-ae218e241252/go.mod h1:IPqxl/YHk05nodzupwjke6ctMjyNRdV2zZ5/j3/F204= -k8s.io/code-generator v0.0.0-20190612205613-18da4a14b22b/go.mod h1:G8bQwmHm2eafm5bgtX67XDZQ8CWKSGu9DekI+yN4Y5I= k8s.io/code-generator v0.0.0-20190717022600-77f3a1fe56bb/go.mod h1:cDx5jQmWH25Ff74daM7NVYty9JWw9dvIS9zT9eIubCY= -k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE= -k8s.io/code-generator v0.16.4/go.mod h1:mJUgkl06XV4kstAnLHAIzJPVCOzVR+ZcfPIv4fUsFCY= -k8s.io/code-generator v0.17.1/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= -k8s.io/code-generator v0.17.2/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= -k8s.io/code-generator v0.18.0/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= -k8s.io/code-generator v0.18.2/go.mod h1:+UHX5rSbxmR8kzS+FAv7um6dtYrZokQvjHpDSYRVkTc= -k8s.io/code-generator v0.18.4/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= -k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= -k8s.io/code-generator v0.19.0/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= -k8s.io/code-generator v0.19.2/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= -k8s.io/code-generator v0.19.4/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.2/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= -k8s.io/code-generator v0.20.4/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= -k8s.io/code-generator v0.22.2/go.mod h1:eV77Y09IopzeXOJzndrDyCI88UBok2h6WxAlBwpxa+o= -k8s.io/code-generator v0.23.5/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/code-generator v0.23.6/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA= -k8s.io/component-base v0.0.0-20190918200425-ed2f0867c778/go.mod h1:DFWQCXgXVLiWtzFaS17KxHdlUeUymP7FLxZSkmL9/jU= -k8s.io/component-base v0.16.4/go.mod h1:GYQ+4hlkEwdlpAp59Ztc4gYuFhdoZqiAJD1unYDJ3FM= -k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= -k8s.io/component-base v0.17.2/go.mod h1:zMPW3g5aH7cHJpKYQ/ZsGMcgbsA/VyhEugF3QT1awLs= -k8s.io/component-base v0.18.0/go.mod h1:u3BCg0z1uskkzrnAKFzulmYaEpZF7XC9Pf/uFyb1v2c= -k8s.io/component-base v0.18.2/go.mod h1:kqLlMuhJNHQ9lz8Z7V5bxUUtjFZnrypArGl58gmDfUM= -k8s.io/component-base v0.18.4/go.mod h1:7jr/Ef5PGmKwQhyAz/pjByxJbC58mhKAhiaDu0vXfPk= -k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14= -k8s.io/component-base v0.19.0/go.mod h1:dKsY8BxkA+9dZIAh2aWJLL/UdASFDNtGYTCItL4LM7Y= -k8s.io/component-base v0.19.2/go.mod h1:g5LrsiTiabMLZ40AR6Hl45f088DevyGY+cCE2agEIVo= -k8s.io/component-base v0.19.4/go.mod h1:ZzuSLlsWhajIDEkKF73j64Gz/5o0AgON08FgRbEPI70= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= -k8s.io/component-base v0.22.2/go.mod h1:5Br2QhI9OTe79p+TzPe9JKNQYvEKbq9rTJDWllunGug= -k8s.io/component-base v0.23.5/go.mod h1:c5Nq44KZyt1aLl0IpHX82fhsn84Sb0jjzwjpcA42bY0= k8s.io/component-base v0.23.6 h1:8dhVZ4VrRcNdV2EGjl8tj8YOHwX6ysgCGMJ2Oyy0NW8= k8s.io/component-base v0.23.6/go.mod h1:FGMPeMrjYu0UZBSAFcfloVDplj9IvU+uRMTOdE23Fj0= -k8s.io/csi-translation-lib v0.17.0/go.mod h1:HEF7MEz7pOLJCnxabi45IPkhSsE/KmxPQksuCrHKWls= -k8s.io/gengo v0.0.0-20181106084056-51747d6e00da/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20190116091435-f8a0810f38af/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20190907103519-ebc107f98eab/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20191108084044-e500ee069b5c/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/gengo v0.0.0-20201214224949-b6c5ce23f027/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog v0.0.0-20190306015804-8e90cee79f82/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.3/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.3.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-aggregator v0.19.0/go.mod h1:1Ln45PQggFAG8xOqWPIYMxUq8WNtpPnYsbUJ39DpF/A= -k8s.io/kube-aggregator v0.19.4/go.mod h1:cTkvun110194d797AuThyydBBlgm+cKIFUeS2uzGJfU= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kubectl v0.19.0/go.mod h1:gPCjjsmE6unJzgaUNXIFGZGafiUp5jh0If3F/x7/rRg= -k8s.io/kubectl v0.19.4/go.mod h1:XPmlu4DJEYgD83pvZFeKF8+MSvGnYGqunbFSrJsqHv0= -k8s.io/kubelet v0.19.4/go.mod h1:zJnPeb7nJCRvtAwxJhe9fFCtMLXL3cXbQiczPmpDrLU= -k8s.io/kubelet v0.22.2/go.mod h1:ORIRua2/wTcx5UnEvxWosu650/8fatmzbMRC7m6WjAM= k8s.io/kubelet v0.23.6 h1:tuscMqYCt9cxWursmTU9OJ2tPLv66Ji+AGbuV1Z/lug= k8s.io/kubelet v0.23.6/go.mod h1:ROttmKIUkB9in4NyX/SfnAoXGfW/Dju3VCGFP34F5ac= -k8s.io/legacy-cloud-providers v0.17.0/go.mod h1:DdzaepJ3RtRy+e5YhNtrCYwlgyK87j/5+Yfp0L9Syp8= -k8s.io/metrics v0.19.0/go.mod h1:WykpW8B60OeAJx1imdwUgyOID2kDljr/Q+1zrPJ98Wo= -k8s.io/metrics v0.19.4/go.mod h1:a0gvAzrxQPw2ouBqnXI7X9qlggpPkKAFgWU/Py+KZiU= -k8s.io/test-infra v0.0.0-20181019233642-2e10a0bbe9b3/go.mod h1:2NzXB13Ji0nqpyublHeiPC4FZwU0TknfvyaaNfl/BTA= -k8s.io/test-infra v0.0.0-20200220102703-18fae0a00a2c/go.mod h1:B9KsgNJiVixsZud99/ugFoQys8h9Tyv/A/eG5LMyrEE= -k8s.io/utils v0.0.0-20181019225348-5e321f9a457c/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= -k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0= -k8s.io/utils v0.0.0-20190506122338-8fab8cb257d5/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= -k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20200912215256-4140de9c8800/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210527160623-6fdb442a123b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -knative.dev/caching v0.0.0-20190719140829-2032732871ff/go.mod h1:dHXFU6CGlLlbzaWc32g80cR92iuBSpsslDNBWI8C7eg= -knative.dev/eventing-contrib v0.6.1-0.20190723221543-5ce18048c08b/go.mod h1:SnXZgSGgMSMLNFTwTnpaOH7hXDzTFtw0J8OmHflNx3g= -knative.dev/pkg v0.0.0-20191111150521-6d806b998379/go.mod h1:pgODObA1dTyhNoFxPZTTjNWfx6F0aKsKzn+vaT9XO/Q= kubevirt.io/api v0.48.1 h1:C5i9h8ea7Xy3fJMoKEuzjRP74GnVMF7u2mQV8FGf2XE= kubevirt.io/api v0.48.1/go.mod h1:RoYMmFt76vWvFtw/FSiL0YUHZ2Ao6UfXlgpZAQnRswo= -kubevirt.io/client-go v0.30.0/go.mod h1:JY7hQq+SUT0aLvleXrW/+28fDfZ6BPe4E6f8FyC8jkY= -kubevirt.io/containerized-data-importer v1.10.6/go.mod h1:qF594BtRRkruyrqLwt3zbLCWdPIQNs1qWh4LR1cOzy0= kubevirt.io/containerized-data-importer-api v1.41.0 h1:VdEwYP36N+4asMnTBSadVH4SF7OVPvvraEQMtOd7Vlk= kubevirt.io/containerized-data-importer-api v1.41.0/go.mod h1:0xadDFtaMd8iy+/oD2+dYoPxACZ/YizKqay5QIrQ6cw= kubevirt.io/controller-lifecycle-operator-sdk v0.2.1 h1:I1b14fnhwrVvQLmgksMo9vgje42hmH4QN5kqyYDqbMA= @@ -2350,53 +1547,22 @@ modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= -mvdan.cc/xurls/v2 v2.0.0/go.mod h1:2/webFPYOXN9jp/lzuj0zuAVlF+9g4KPFJANH1oJhRU= -pack.ag/amqp v0.11.0/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= -sigs.k8s.io/controller-runtime v0.3.0/go.mod h1:Cw6PkEg0Sa7dAYovGT4R0tRkGhHXpYijwNxYhAnAZZk= -sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns= -sigs.k8s.io/controller-runtime v0.6.1/go.mod h1:XRYBPdbf5XJu9kpS84VJiZ7h/u1hF3gEORz0efEja7A= -sigs.k8s.io/controller-runtime v0.6.2/go.mod h1:vhcq/rlnENJ09SIRp3EveTaZ0yqH526hjf9iJdbUJ/E= -sigs.k8s.io/controller-runtime v0.6.3/go.mod h1:WlZNXcM0++oyaQt4B7C2lEE5JYRs8vJUzRP4N4JpdAY= -sigs.k8s.io/controller-runtime v0.7.0/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU= sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= -sigs.k8s.io/controller-runtime v0.10.2/go.mod h1:CQp8eyUQZ/Q7PJvnIrB6/hgfTC1kBkGylwsLgOQi1WY= sigs.k8s.io/controller-runtime v0.11.2 h1:H5GTxQl0Mc9UjRJhORusqfJCIjBO8UtUxGggCwL1rLA= sigs.k8s.io/controller-runtime v0.11.2/go.mod h1:P6QCzrEjLaZGqHsfd+os7JQ+WFZhvB8MRFsn4dWF7O4= -sigs.k8s.io/controller-tools v0.2.4/go.mod h1:m/ztfQNocGYBgTTCmFdnK94uVvgxeZeE3LtJvd/jIzA= -sigs.k8s.io/controller-tools v0.2.9-0.20200414181213-645d44dca7c0/go.mod h1:YKE/iHvcKITCljdnlqHYe+kAt7ZldvtAwUzQff0k1T0= -sigs.k8s.io/controller-tools v0.4.1/go.mod h1:G9rHdZMVlBDocIxGkK3jHLWqcTMNvveypYJwrvYKjWU= sigs.k8s.io/controller-tools v0.5.0/go.mod h1:JTsstrMpxs+9BUj6eGuAaEb6SDSPTeVtUyp0jmnAM/I= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= -sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= -sigs.k8s.io/structured-merge-diff v0.0.0-20190302045857-e85c7b244fd2/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= -sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= -sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= -sigs.k8s.io/structured-merge-diff v1.0.1 h1:LOs1LZWMsz1xs77Phr/pkB4LFaavH7IVq/3+WTN9XTA= -sigs.k8s.io/structured-merge-diff v1.0.1/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= -sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/testing_frameworks v0.1.1/go.mod h1:VVBKrHmJ6Ekkfz284YKhQePcdycOzNH9qL6ht1zEr/U= -sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= -software.sslmate.com/src/go-pkcs12 v0.0.0-20180114231543-2291e8f0f237/go.mod h1:/xvNRWUqm0+/ZMiF4EX00vrSCMsE4/NHb+Pt3freEeQ= -software.sslmate.com/src/go-pkcs12 v0.0.0-20200830195227-52f69702a001/go.mod h1:/xvNRWUqm0+/ZMiF4EX00vrSCMsE4/NHb+Pt3freEeQ= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= -vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= -vbom.ml/util v0.0.0-20180919145318-efcd4e0f9787/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj6uzU2+8OWDFv/HxUSs7kI= From 29ac0cd388a11ce341959f4585bbfff84d504aba Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Thu, 5 May 2022 15:38:46 +0200 Subject: [PATCH 130/489] Remove support for Kubernetes 1.20 and bump patch releases (#1274) * Remove Kubernetes 1.20 and bump patch releases Signed-off-by: Marvin Beckers * Update fixtures Signed-off-by: Marvin Beckers --- README.md | 1 - pkg/userdata/amzn2/provider_test.go | 19 - pkg/userdata/centos/provider_test.go | 37 +- .../testdata/kubelet-v1.21-aws-external.yaml | 2 +- .../centos/testdata/kubelet-v1.21-aws.yaml | 2 +- .../kubelet-v1.21-vsphere-mirrors.yaml | 2 +- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 2 +- .../testdata/kubelet-v1.21-vsphere.yaml | 2 +- .../centos/testdata/kubelet-v1.22-aws.yaml | 2 +- .../centos/testdata/kubelet-v1.23-aws.yaml | 2 +- .../testdata/kubelet-v1.23-nutanix.yaml | 450 +++++++++++++++++ pkg/userdata/flatcar/provider_test.go | 88 +--- .../flatcar/testdata/cloud-init_v1.21.10.yaml | 477 ++++++++++++++++++ .../flatcar/testdata/cloud-init_v1.22.7.yaml | 477 ++++++++++++++++++ .../flatcar/testdata/cloud-init_v1.23.5.yaml | 475 +++++++++++++++++ pkg/userdata/flatcar/testdata/containerd.yaml | 2 +- .../flatcar/testdata/ignition_v1.21.10.json | 1 + .../flatcar/testdata/ignition_v1.22.7.json | 1 + .../flatcar/testdata/ignition_v1.23.5.json | 1 + pkg/userdata/helper/common_test.go | 7 +- pkg/userdata/helper/kubelet_test.go | 8 +- .../download_binaries_v1.21.10.golden | 17 + .../testdata/download_binaries_v1.22.7.golden | 17 + .../testdata/download_binaries_v1.23.5.golden | 17 + ...let_systemd_unit_cloud-provider-set.golden | 2 - ...t_systemd_unit_multiple-dns-servers.golden | 2 - ...kublet_systemd_unit_pause-image-set.golden | 2 - .../kublet_systemd_unit_taints-set.golden | 2 - ...temd_unit_version-v1.21.10-external.golden | 38 ++ ...ublet_systemd_unit_version-v1.21.10.golden | 37 ++ ...stemd_unit_version-v1.22.7-external.golden | 38 ++ ...kublet_systemd_unit_version-v1.22.7.golden | 37 ++ ...stemd_unit_version-v1.23.5-external.golden | 36 ++ ...kublet_systemd_unit_version-v1.23.5.golden | 35 ++ pkg/userdata/rhel/provider_test.go | 37 +- .../rhel/testdata/kubelet-v1.21-aws.yaml | 2 +- .../rhel/testdata/kubelet-v1.22-aws.yaml | 2 +- .../rhel/testdata/kubelet-v1.22-nutanix.yaml | 2 +- .../testdata/kubelet-v1.23-aws-external.yaml | 2 +- .../rhel/testdata/kubelet-v1.23-aws.yaml | 2 +- .../kubelet-v1.23-vsphere-mirrors.yaml | 2 +- .../testdata/kubelet-v1.23-vsphere-proxy.yaml | 2 +- .../rhel/testdata/kubelet-v1.23-vsphere.yaml | 2 +- .../rhel/testdata/pod-cidr-azure-rhel.yaml | 2 +- pkg/userdata/rockylinux/provider_test.go | 37 +- .../testdata/kubelet-v1.21-aws-external.yaml | 2 +- .../testdata/kubelet-v1.21-aws.yaml | 2 +- .../kubelet-v1.21-vsphere-mirrors.yaml | 2 +- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 2 +- .../testdata/kubelet-v1.21-vsphere.yaml | 2 +- .../testdata/kubelet-v1.22-aws.yaml | 2 +- .../testdata/kubelet-v1.23-aws.yaml | 2 +- .../testdata/kubelet-v1.23-nutanix.yaml | 445 ++++++++++++++++ pkg/userdata/sles/provider_test.go | 7 +- .../sles/testdata/version-1.21.10.yaml | 424 ++++++++++++++++ .../sles/testdata/version-1.22.7.yaml | 424 ++++++++++++++++ .../sles/testdata/version-1.23.5.yaml | 422 ++++++++++++++++ pkg/userdata/ubuntu/provider_test.go | 23 +- pkg/userdata/ubuntu/testdata/containerd.yaml | 2 +- .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 2 +- .../kubelet-version-without-v-prefix.yaml | 2 +- .../ubuntu/testdata/multiple-dns-servers.yaml | 2 +- .../ubuntu/testdata/multiple-ssh-keys.yaml | 2 +- pkg/userdata/ubuntu/testdata/nutanix.yaml | 2 +- .../openstack-overwrite-cloud-config.yaml | 2 +- pkg/userdata/ubuntu/testdata/openstack.yaml | 2 +- .../ubuntu/testdata/version-1.21.10.yaml | 446 ++++++++++++++++ .../ubuntu/testdata/version-1.22.7.yaml | 446 ++++++++++++++++ .../ubuntu/testdata/version-1.23.5.yaml | 444 ++++++++++++++++ .../ubuntu/testdata/vsphere-mirrors.yaml | 2 +- .../ubuntu/testdata/vsphere-proxy.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere.yaml | 2 +- test/e2e/provisioning/helper.go | 7 +- 73 files changed, 5304 insertions(+), 250 deletions(-) create mode 100644 pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml create mode 100644 pkg/userdata/flatcar/testdata/cloud-init_v1.21.10.yaml create mode 100644 pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml create mode 100644 pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml create mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.21.10.json create mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.22.7.json create mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.23.5.json create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.21.10.golden create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.22.7.golden create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.23.5.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10-external.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.7-external.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.7.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5-external.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5.golden create mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml create mode 100644 pkg/userdata/sles/testdata/version-1.21.10.yaml create mode 100644 pkg/userdata/sles/testdata/version-1.22.7.yaml create mode 100644 pkg/userdata/sles/testdata/version-1.23.5.yaml create mode 100644 pkg/userdata/ubuntu/testdata/version-1.21.10.yaml create mode 100644 pkg/userdata/ubuntu/testdata/version-1.22.7.yaml create mode 100644 pkg/userdata/ubuntu/testdata/version-1.23.5.yaml diff --git a/README.md b/README.md index 7e07d201c..775c7ff99 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,6 @@ Currently supported K8S versions are: - 1.23 - 1.22 - 1.21 -- 1.20 ## What does not work - Master creation (Not planned at the moment) diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index 03b5b172d..bb42d0651 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -99,25 +99,6 @@ func TestUserDataGeneration(t *testing.T) { t.Parallel() tests := []userDataTestCase{ - { - name: "kubelet-v1.20-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.20.14", - }, - }, - }, - { - name: "containerd-kubelet-v1.20-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.20.14", - }, - }, - containerruntime: "containerd", - }, { name: "kubelet-v1.21-aws", spec: clusterv1alpha1.MachineSpec{ diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index 5d37023c4..532b9f561 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -99,31 +99,12 @@ func TestUserDataGeneration(t *testing.T) { t.Parallel() tests := []userDataTestCase{ - { - name: "kubelet-v1.20-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.20.14", - }, - }, - }, - { - name: "kubelet-containerd-v1.20-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.20.14", - }, - }, - containerruntime: "containerd", - }, { name: "kubelet-v1.21-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.8", + Kubelet: "1.21.10", }, }, }, @@ -132,7 +113,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.8", + Kubelet: "1.21.10", }, }, externalCloudProvider: true, @@ -142,7 +123,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.8", + Kubelet: "1.21.10", }, }, cloudProviderName: stringPtr("vsphere"), @@ -152,7 +133,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.8", + Kubelet: "1.21.10", }, }, cloudProviderName: stringPtr("vsphere"), @@ -166,7 +147,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.8", + Kubelet: "1.21.10", }, }, cloudProviderName: stringPtr("vsphere"), @@ -180,7 +161,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.5", + Kubelet: "1.22.7", }, }, }, @@ -189,16 +170,16 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.0", + Kubelet: "1.23.5", }, }, }, { - name: "kubelet-v1.21-nutanix", + name: "kubelet-v1.23-nutanix", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.23.5", }, }, cloudProviderName: stringPtr("nutanix"), diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml index 4887f0bd5..b320e4525 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml index 9e25af45f..d72e3d0a9 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml index d7662b439..df9d20754 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -162,7 +162,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml index a67e5e54c..641e97670 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -162,7 +162,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml index 18171adfe..6e6e7d3bd 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml @@ -154,7 +154,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index 4a161f747..6b886b62d 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index c7d8b6200..744b6273c 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml new file mode 100644 index 000000000..e353cb0e0 --- /dev/null +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml @@ -0,0 +1,450 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + hostnamectl set-hostname node1 + + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + iscsi-initiator-utils \ + ipvsadm + systemctl enable --now iscsid + + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=nutanix \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index 3836bc6cf..bd98b82de 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -122,7 +122,7 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "ignition_v1.20.14", + name: "ignition_v1.21.10", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -138,7 +138,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.20.14", + Kubelet: "v1.21.10", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -153,7 +153,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.21.8", + name: "ignition_v1.22.7", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -169,7 +169,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.21.8", + Kubelet: "v1.22.7", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -184,7 +184,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.22.5", + name: "ignition_v1.23.5", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -200,7 +200,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.22.5", + Kubelet: "v1.23.5", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -215,69 +215,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.23.0", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", - SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, - CAPublicKey: "ssh-rsa AAABBB", - Network: &providerconfigtypes.NetworkConfig{ - CIDR: "192.168.81.4/24", - Gateway: "192.168.81.1", - DNS: providerconfigtypes.DNSConfig{ - Servers: []string{"8.8.8.8"}, - }, - }, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.23.0", - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - osConfig: &Config{ - DisableAutoUpdate: true, - ProvisioningUtility: Ignition, - }, - }, - { - name: "cloud-init_v1.20.14", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "anexia", - SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, - CAPublicKey: "ssh-rsa AAABBB", - Network: &providerconfigtypes.NetworkConfig{ - CIDR: "192.168.81.4/24", - Gateway: "192.168.81.1", - DNS: providerconfigtypes.DNSConfig{ - Servers: []string{"8.8.8.8"}, - }, - }, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.20.14", - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "anexia", - config: "{anexia-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - osConfig: &Config{ - DisableAutoUpdate: true, - ProvisioningUtility: CloudInit, - }, - }, - { - name: "cloud-init_v1.21.8", + name: "cloud-init_v1.21.10", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -293,7 +231,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.21.8", + Kubelet: "v1.21.10", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -308,7 +246,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.22.5", + name: "cloud-init_v1.22.7", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -324,7 +262,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.22.5", + Kubelet: "v1.22.7", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -339,7 +277,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.23.0", + name: "cloud-init_v1.23.5", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -355,7 +293,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.23.0", + Kubelet: "v1.23.5", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -381,7 +319,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.21.8", + Kubelet: "v1.21.10", }, }, ccProvider: &fakeCloudConfigProvider{}, diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.10.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.10.yaml new file mode 100644 index 000000000..23526cb04 --- /dev/null +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.10.yaml @@ -0,0 +1,477 @@ +#cloud-config + +users: +- name: core + ssh_authorized_keys: + - ssh-rsa AAABBB + - ssh-rsa CCCDDD + + +coreos: + units: + - name: static-nic.network + content: | + [Match] + # Because of difficulty predicting specific NIC names on different cloud providers, + # we only support static addressing on VSphere. There should be a single NIC attached + # that we will match by name prefix 'en' which denotes ethernet devices. + Name=en* + + [Network] + DHCP=no + Address=192.168.81.4/24 + Gateway=192.168.81.1 + DNS=8.8.8.8 + + - name: update-engine.service + command: stop + mask: true + - name: locksmithd.service + command: stop + mask: true + - name: download-script.service + enable: true + command: start + content: | + [Unit] + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/download.sh + [Install] + WantedBy=multi-user.target + + - name: kubelet-healthcheck.service + enable: true + command: start + drop-ins: + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + + - name: nodeip.service + enable: true + command: start + content: | + [Unit] + Description=Setup Kubelet Node IP Env + Requires=network-online.target + After=network-online.target + + [Service] + ExecStart=/opt/bin/setup_net_env.sh + RemainAfterExit=yes + Type=oneshot + [Install] + WantedBy=multi-user.target + + - name: kubelet.service + enable: true + command: start + drop-ins: + - name: 10-nodeip.conf + content: | + [Service] + EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=anexia \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + + - name: apply-sysctl-settings.service + enable: true + command: start + content: | + [Unit] + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + ExecStart=/opt/bin/apply_sysctl_settings.sh + [Install] + WantedBy=multi-user.target + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + permissions: "0644" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/etc/kubernetes/kubelet.conf" + permissions: "0644" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /opt/load-kernel-modules.sh + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: /etc/sysctl.d/k8s.conf + permissions: "0644" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: /etc/kubernetes/bootstrap-kubelet.conf + permissions: "0400" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: /etc/kubernetes/cloud-config + permissions: "0400" + content: | + {anexia-config:true} + +- path: /etc/kubernetes/pki/ca.crt + permissions: "0644" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + +- path: /etc/hostname + permissions: "0600" + content: 'node1' + +- path: /etc/ssh/sshd_config + permissions: "0600" + user: root + content: | + # Use most defaults for sshd configuration. + Subsystem sftp internal-sftp + ClientAliveInterval 180 + UseDNS no + UsePAM yes + PrintLastLog no # handled by PAM + PrintMotd no # handled by PAM + PasswordAuthentication no + ChallengeResponseAuthentication no + +- path: /opt/bin/download.sh + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v0.8.7}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + systemctl daemon-reload + systemctl enable --now docker + + systemctl disable download-script.service + +- path: /opt/bin/apply_sysctl_settings.sh + permissions: "0755" + user: root + content: | + #!/bin/bash + set -xeuo pipefail + sysctl --system + systemctl disable apply-sysctl-settings.service + +- path: "/etc/ssh/trusted-user-ca-keys.pem" + content: | + ssh-rsa AAABBB + +- path: "/etc/ssh/sshd_config" + content: | + TrustedUserCAKeys /etc/ssh/trusted-user-ca-keys.pem + CASignatureAlgorithms ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,ssh-ed25519,rsa-sha2-512,rsa-sha2-256,ssh-rsa + append: true + +- path: /etc/docker/daemon.json + permissions: "0644" + user: root + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /etc/crictl.yaml + permissions: "0644" + user: root + content: | + runtime-endpoint: unix:///run/containerd/containerd.sock diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml new file mode 100644 index 000000000..c94066c34 --- /dev/null +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml @@ -0,0 +1,477 @@ +#cloud-config + +users: +- name: core + ssh_authorized_keys: + - ssh-rsa AAABBB + - ssh-rsa CCCDDD + + +coreos: + units: + - name: static-nic.network + content: | + [Match] + # Because of difficulty predicting specific NIC names on different cloud providers, + # we only support static addressing on VSphere. There should be a single NIC attached + # that we will match by name prefix 'en' which denotes ethernet devices. + Name=en* + + [Network] + DHCP=no + Address=192.168.81.4/24 + Gateway=192.168.81.1 + DNS=8.8.8.8 + + - name: update-engine.service + command: stop + mask: true + - name: locksmithd.service + command: stop + mask: true + - name: download-script.service + enable: true + command: start + content: | + [Unit] + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/download.sh + [Install] + WantedBy=multi-user.target + + - name: kubelet-healthcheck.service + enable: true + command: start + drop-ins: + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + + - name: nodeip.service + enable: true + command: start + content: | + [Unit] + Description=Setup Kubelet Node IP Env + Requires=network-online.target + After=network-online.target + + [Service] + ExecStart=/opt/bin/setup_net_env.sh + RemainAfterExit=yes + Type=oneshot + [Install] + WantedBy=multi-user.target + + - name: kubelet.service + enable: true + command: start + drop-ins: + - name: 10-nodeip.conf + content: | + [Service] + EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=anexia \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + + - name: apply-sysctl-settings.service + enable: true + command: start + content: | + [Unit] + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + ExecStart=/opt/bin/apply_sysctl_settings.sh + [Install] + WantedBy=multi-user.target + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + permissions: "0644" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/etc/kubernetes/kubelet.conf" + permissions: "0644" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /opt/load-kernel-modules.sh + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: /etc/sysctl.d/k8s.conf + permissions: "0644" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: /etc/kubernetes/bootstrap-kubelet.conf + permissions: "0400" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: /etc/kubernetes/cloud-config + permissions: "0400" + content: | + {anexia-config:true} + +- path: /etc/kubernetes/pki/ca.crt + permissions: "0644" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + +- path: /etc/hostname + permissions: "0600" + content: 'node1' + +- path: /etc/ssh/sshd_config + permissions: "0600" + user: root + content: | + # Use most defaults for sshd configuration. + Subsystem sftp internal-sftp + ClientAliveInterval 180 + UseDNS no + UsePAM yes + PrintLastLog no # handled by PAM + PrintMotd no # handled by PAM + PasswordAuthentication no + ChallengeResponseAuthentication no + +- path: /opt/bin/download.sh + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v0.8.7}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + systemctl daemon-reload + systemctl enable --now docker + + systemctl disable download-script.service + +- path: /opt/bin/apply_sysctl_settings.sh + permissions: "0755" + user: root + content: | + #!/bin/bash + set -xeuo pipefail + sysctl --system + systemctl disable apply-sysctl-settings.service + +- path: "/etc/ssh/trusted-user-ca-keys.pem" + content: | + ssh-rsa AAABBB + +- path: "/etc/ssh/sshd_config" + content: | + TrustedUserCAKeys /etc/ssh/trusted-user-ca-keys.pem + CASignatureAlgorithms ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,ssh-ed25519,rsa-sha2-512,rsa-sha2-256,ssh-rsa + append: true + +- path: /etc/docker/daemon.json + permissions: "0644" + user: root + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /etc/crictl.yaml + permissions: "0644" + user: root + content: | + runtime-endpoint: unix:///run/containerd/containerd.sock diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml new file mode 100644 index 000000000..708e6f6be --- /dev/null +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml @@ -0,0 +1,475 @@ +#cloud-config + +users: +- name: core + ssh_authorized_keys: + - ssh-rsa AAABBB + - ssh-rsa CCCDDD + + +coreos: + units: + - name: static-nic.network + content: | + [Match] + # Because of difficulty predicting specific NIC names on different cloud providers, + # we only support static addressing on VSphere. There should be a single NIC attached + # that we will match by name prefix 'en' which denotes ethernet devices. + Name=en* + + [Network] + DHCP=no + Address=192.168.81.4/24 + Gateway=192.168.81.1 + DNS=8.8.8.8 + + - name: update-engine.service + command: stop + mask: true + - name: locksmithd.service + command: stop + mask: true + - name: download-script.service + enable: true + command: start + content: | + [Unit] + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/download.sh + [Install] + WantedBy=multi-user.target + + - name: kubelet-healthcheck.service + enable: true + command: start + drop-ins: + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + + - name: nodeip.service + enable: true + command: start + content: | + [Unit] + Description=Setup Kubelet Node IP Env + Requires=network-online.target + After=network-online.target + + [Service] + ExecStart=/opt/bin/setup_net_env.sh + RemainAfterExit=yes + Type=oneshot + [Install] + WantedBy=multi-user.target + + - name: kubelet.service + enable: true + command: start + drop-ins: + - name: 10-nodeip.conf + content: | + [Service] + EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=anexia \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + + - name: apply-sysctl-settings.service + enable: true + command: start + content: | + [Unit] + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + ExecStart=/opt/bin/apply_sysctl_settings.sh + [Install] + WantedBy=multi-user.target + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + permissions: "0644" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/etc/kubernetes/kubelet.conf" + permissions: "0644" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /opt/load-kernel-modules.sh + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: /etc/sysctl.d/k8s.conf + permissions: "0644" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: /etc/kubernetes/bootstrap-kubelet.conf + permissions: "0400" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: /etc/kubernetes/cloud-config + permissions: "0400" + content: | + {anexia-config:true} + +- path: /etc/kubernetes/pki/ca.crt + permissions: "0644" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + +- path: /etc/hostname + permissions: "0600" + content: 'node1' + +- path: /etc/ssh/sshd_config + permissions: "0600" + user: root + content: | + # Use most defaults for sshd configuration. + Subsystem sftp internal-sftp + ClientAliveInterval 180 + UseDNS no + UsePAM yes + PrintLastLog no # handled by PAM + PrintMotd no # handled by PAM + PasswordAuthentication no + ChallengeResponseAuthentication no + +- path: /opt/bin/download.sh + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v0.8.7}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + systemctl daemon-reload + systemctl enable --now docker + + systemctl disable download-script.service + +- path: /opt/bin/apply_sysctl_settings.sh + permissions: "0755" + user: root + content: | + #!/bin/bash + set -xeuo pipefail + sysctl --system + systemctl disable apply-sysctl-settings.service + +- path: "/etc/ssh/trusted-user-ca-keys.pem" + content: | + ssh-rsa AAABBB + +- path: "/etc/ssh/sshd_config" + content: | + TrustedUserCAKeys /etc/ssh/trusted-user-ca-keys.pem + CASignatureAlgorithms ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,ssh-ed25519,rsa-sha2-512,rsa-sha2-256,ssh-rsa + append: true + +- path: /etc/docker/daemon.json + permissions: "0644" + user: root + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /etc/crictl.yaml + permissions: "0644" + user: root + content: | + runtime-endpoint: unix:///run/containerd/containerd.sock diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index 2a5d8328e..fb3aceddd 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -398,7 +398,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.21.10.json b/pkg/userdata/flatcar/testdata/ignition_v1.21.10.json new file mode 100644 index 000000000..35fe8e7f1 --- /dev/null +++ b/pkg/userdata/flatcar/testdata/ignition_v1.21.10.json @@ -0,0 +1 @@ +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.10%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json b/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json new file mode 100644 index 000000000..ca64195b9 --- /dev/null +++ b/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json @@ -0,0 +1 @@ +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.7%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json b/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json new file mode 100644 index 000000000..c7a4c3bff --- /dev/null +++ b/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json @@ -0,0 +1 @@ +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --network-plugin=cni \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/common_test.go b/pkg/userdata/helper/common_test.go index 2b8a79edd..0bac1429e 100644 --- a/pkg/userdata/helper/common_test.go +++ b/pkg/userdata/helper/common_test.go @@ -26,9 +26,8 @@ var update = flag.Bool("update", false, "update testdata files") var ( versions = []*semver.Version{ - semver.MustParse("v1.20.14"), - semver.MustParse("v1.21.8"), - semver.MustParse("v1.22.5"), - semver.MustParse("v1.23.0"), + semver.MustParse("v1.21.10"), + semver.MustParse("v1.22.7"), + semver.MustParse("v1.23.5"), } ) diff --git a/pkg/userdata/helper/kubelet_test.go b/pkg/userdata/helper/kubelet_test.go index 9e4f6acac..5b7326a1c 100644 --- a/pkg/userdata/helper/kubelet_test.go +++ b/pkg/userdata/helper/kubelet_test.go @@ -63,7 +63,7 @@ func TestKubeletSystemdUnit(t *testing.T) { tests = append(tests, []kubeletFlagTestCase{ { name: "multiple-dns-servers", - version: semver.MustParse("v1.20.14"), + version: semver.MustParse("v1.23.5"), dnsIPs: []net.IP{ net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), @@ -73,14 +73,14 @@ func TestKubeletSystemdUnit(t *testing.T) { }, { name: "cloud-provider-set", - version: semver.MustParse("v1.20.14"), + version: semver.MustParse("v1.23.5"), dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, hostname: "some-test-node", cloudProvider: "aws", }, { name: "pause-image-set", - version: semver.MustParse("v1.20.14"), + version: semver.MustParse("v1.23.5"), dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, hostname: "some-test-node", cloudProvider: "aws", @@ -88,7 +88,7 @@ func TestKubeletSystemdUnit(t *testing.T) { }, { name: "taints-set", - version: semver.MustParse("v1.20.14"), + version: semver.MustParse("v1.23.5"), dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, hostname: "some-test-node", cloudProvider: "aws", diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.21.10.golden b/pkg/userdata/helper/testdata/download_binaries_v1.21.10.golden new file mode 100644 index 000000000..75ed990ec --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.21.10.golden @@ -0,0 +1,17 @@ +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - +fi +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.21.10/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.22.7.golden b/pkg/userdata/helper/testdata/download_binaries_v1.22.7.golden new file mode 100644 index 000000000..02b36f747 --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.22.7.golden @@ -0,0 +1,17 @@ +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - +fi +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.22.7/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.23.5.golden b/pkg/userdata/helper/testdata/download_binaries_v1.23.5.golden new file mode 100644 index 000000000..e16afc69b --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.23.5.golden @@ -0,0 +1,17 @@ +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - +fi +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.23.5/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden index 42484e368..b42e530f2 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden @@ -30,8 +30,6 @@ ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden index 2c7c03898..8cd283a84 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden @@ -29,8 +29,6 @@ ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --hostname-override=some-test-node \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_pause-image-set.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_pause-image-set.golden index 0d21c5dd6..d7c4dec1b 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_pause-image-set.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_pause-image-set.golden @@ -31,8 +31,6 @@ ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_taints-set.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_taints-set.golden index 0db8bf8ae..9e6f21093 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_taints-set.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_taints-set.golden @@ -31,8 +31,6 @@ ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --register-with-taints=key1=value1:NoSchedule,key2=value2:NoExecute \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10-external.golden new file mode 100644 index 000000000..50c334831 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10-external.golden @@ -0,0 +1,38 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10.golden new file mode 100644 index 000000000..2c7c03898 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10.golden @@ -0,0 +1,37 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.7-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.7-external.golden new file mode 100644 index 000000000..50c334831 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.7-external.golden @@ -0,0 +1,38 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.7.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.7.golden new file mode 100644 index 000000000..2c7c03898 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.7.golden @@ -0,0 +1,37 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5-external.golden new file mode 100644 index 000000000..62ee82ac5 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5-external.golden @@ -0,0 +1,36 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5.golden new file mode 100644 index 000000000..8cd283a84 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5.golden @@ -0,0 +1,35 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index 6270ad01a..0a11d1ec0 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -99,31 +99,12 @@ func TestUserDataGeneration(t *testing.T) { t.Parallel() tests := []userDataTestCase{ - { - name: "kubelet-v1.20-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.20.14", - }, - }, - }, - { - name: "kubelet-containerd-v1.20-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.20.14", - }, - }, - containerruntime: "containerd", - }, { name: "kubelet-v1.21-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.8", + Kubelet: "1.21.10", }, }, }, @@ -132,7 +113,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.5", + Kubelet: "1.22.7", }, }, }, @@ -141,7 +122,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.0", + Kubelet: "1.23.5", }, }, }, @@ -150,7 +131,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.0", + Kubelet: "1.23.5", }, }, externalCloudProvider: true, @@ -160,7 +141,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.0", + Kubelet: "1.23.5", }, }, cloudProviderName: stringPtr("vsphere"), @@ -170,7 +151,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.0", + Kubelet: "1.23.5", }, }, cloudProviderName: stringPtr("vsphere"), @@ -184,7 +165,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.0", + Kubelet: "1.23.5", }, }, cloudProviderName: stringPtr("vsphere"), @@ -198,7 +179,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.2", + Kubelet: "1.22.7", }, }, cloudProviderName: stringPtr("nutanix"), @@ -208,7 +189,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.2", + Kubelet: "1.22.7", }, }, cloudProviderName: stringPtr("azure"), diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml index a17711a82..7502c44ef 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index a527bf430..bb46a0ab8 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml index 695b02ba3..f9e905e55 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml @@ -152,7 +152,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index 4dd5ca128..97cff977e 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index db44f8876..48a4479c4 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index e5368bc62..d3c8af5cb 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -158,7 +158,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index 9f899a025..b11c82709 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -158,7 +158,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index b067a10b5..19048b01a 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index ab6b831a3..c753a3d14 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/provider_test.go b/pkg/userdata/rockylinux/provider_test.go index ad23a2f1b..db89c9fb6 100644 --- a/pkg/userdata/rockylinux/provider_test.go +++ b/pkg/userdata/rockylinux/provider_test.go @@ -99,31 +99,12 @@ func TestUserDataGeneration(t *testing.T) { t.Parallel() tests := []userDataTestCase{ - { - name: "kubelet-v1.20-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.20.14", - }, - }, - }, - { - name: "kubelet-containerd-v1.20-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.20.14", - }, - }, - containerruntime: "containerd", - }, { name: "kubelet-v1.21-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.8", + Kubelet: "1.21.10", }, }, }, @@ -132,7 +113,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.8", + Kubelet: "1.21.10", }, }, externalCloudProvider: true, @@ -142,7 +123,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.8", + Kubelet: "1.21.10", }, }, cloudProviderName: stringPtr("vsphere"), @@ -152,7 +133,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.8", + Kubelet: "1.21.10", }, }, cloudProviderName: stringPtr("vsphere"), @@ -166,7 +147,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.8", + Kubelet: "1.21.10", }, }, cloudProviderName: stringPtr("vsphere"), @@ -180,7 +161,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.5", + Kubelet: "1.22.7", }, }, }, @@ -189,16 +170,16 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.0", + Kubelet: "1.23.5", }, }, }, { - name: "kubelet-v1.21-nutanix", + name: "kubelet-v1.23-nutanix", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.23.5", }, }, cloudProviderName: stringPtr("nutanix"), diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml index f5bc91a24..22d01bf21 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml index abb4be043..a3d3422e1 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 30d48ac51..33a5c2834 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -157,7 +157,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml index b96a049a2..6720f3545 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -157,7 +157,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml index 3736aee85..2e9edf240 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml index e336d03bc..a70be1fd9 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml index 65a534e8b..d5852f0fb 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml new file mode 100644 index 000000000..9cf6040c1 --- /dev/null +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml @@ -0,0 +1,445 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + +hostname: node1 + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + hostnamectl set-hostname node1 + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + tar \ + iscsi-initiator-utils \ + ipvsadm + systemctl enable --now iscsid + + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=nutanix \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/sles/provider_test.go b/pkg/userdata/sles/provider_test.go index b04304099..f6225ebca 100644 --- a/pkg/userdata/sles/provider_test.go +++ b/pkg/userdata/sles/provider_test.go @@ -126,10 +126,9 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.20.14"), - semver.MustParse("v1.21.8"), - semver.MustParse("v1.22.5"), - semver.MustParse("v1.23.0"), + semver.MustParse("v1.21.10"), + semver.MustParse("v1.22.7"), + semver.MustParse("v1.23.5"), } var tests []userDataTestCase diff --git a/pkg/userdata/sles/testdata/version-1.21.10.yaml b/pkg/userdata/sles/testdata/version-1.21.10.yaml new file mode 100644 index 000000000..5c7a9a845 --- /dev/null +++ b/pkg/userdata/sles/testdata/version-1.21.10.yaml @@ -0,0 +1,424 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + systemctl restart systemd-modules-load.service + sysctl --system + + zypper --non-interactive --quiet --color install ebtables \ + ceph-common \ + e2fsprogs \ + jq \ + socat \ + ipvsadm + + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v0.8.7}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + +- path: /etc/systemd/system/docker.service.d/environment.conf + permissions: "0644" + content: | + [Service] + EnvironmentFile=-/etc/environment + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.22.7.yaml b/pkg/userdata/sles/testdata/version-1.22.7.yaml new file mode 100644 index 000000000..f15590efb --- /dev/null +++ b/pkg/userdata/sles/testdata/version-1.22.7.yaml @@ -0,0 +1,424 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + systemctl restart systemd-modules-load.service + sysctl --system + + zypper --non-interactive --quiet --color install ebtables \ + ceph-common \ + e2fsprogs \ + jq \ + socat \ + ipvsadm + + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v0.8.7}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + +- path: /etc/systemd/system/docker.service.d/environment.conf + permissions: "0644" + content: | + [Service] + EnvironmentFile=-/etc/environment + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.23.5.yaml b/pkg/userdata/sles/testdata/version-1.23.5.yaml new file mode 100644 index 000000000..6e4904368 --- /dev/null +++ b/pkg/userdata/sles/testdata/version-1.23.5.yaml @@ -0,0 +1,422 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + systemctl restart systemd-modules-load.service + sysctl --system + + zypper --non-interactive --quiet --color install ebtables \ + ceph-common \ + e2fsprogs \ + jq \ + socat \ + ipvsadm + + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v0.8.7}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + +- path: /etc/systemd/system/docker.service.d/environment.conf + permissions: "0644" + content: | + [Service] + EnvironmentFile=-/etc/environment + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 6c1ecd00e..63e85822b 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -92,7 +92,7 @@ kPe6XoSbiLm/kxk32T0= ) const ( - defaultVersion = "1.22.5" + defaultVersion = "1.22.7" ) type fakeCloudConfigProvider struct { @@ -127,10 +127,9 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.20.14"), - semver.MustParse("v1.21.8"), - semver.MustParse("v1.22.5"), - semver.MustParse("v1.23.0"), + semver.MustParse("v1.21.10"), + semver.MustParse("v1.22.7"), + semver.MustParse("v1.23.5"), } var tests []userDataTestCase @@ -233,7 +232,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.5", + Kubelet: "1.22.7", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -258,7 +257,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.5", + Kubelet: "1.22.7", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -309,7 +308,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.5", + Kubelet: "1.22.7", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -335,7 +334,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.5", + Kubelet: "1.22.7", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -361,7 +360,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.5", + Kubelet: "1.22.7", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -391,7 +390,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.5", + Kubelet: "1.22.7", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -457,7 +456,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.5", + Kubelet: "1.21.10", }, }, ccProvider: &fakeCloudConfigProvider{ diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index d451999dc..b4e6b55b9 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 22f215094..95e3ec8fc 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index 470ae4581..b7ceb1c17 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index cdef7de4e..32d7d24e1 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 067868e0e..7652fc67c 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 41ca26fff..b7d9c4dc1 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -152,7 +152,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index 9bf32e2e6..8be555a3a 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index f4ca3a745..65ece0806 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml b/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml new file mode 100644 index 000000000..9f698da58 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml @@ -0,0 +1,446 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml b/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml new file mode 100644 index 000000000..b7ceb1c17 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml @@ -0,0 +1,446 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml new file mode 100644 index 000000000..71de2cf2c --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml @@ -0,0 +1,444 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --network-plugin=cni \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index 749469a8c..c93345dc5 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -159,7 +159,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 4a767f33a..281d9f21e 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -159,7 +159,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 3f4a95a91..9cbff06ec 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index c49bc5f0b..9cb39d9c7 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,10 +33,9 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.20.14"), - semver.MustParse("v1.21.8"), - semver.MustParse("v1.22.5"), - semver.MustParse("v1.23.0"), + semver.MustParse("v1.21.10"), + semver.MustParse("v1.22.7"), + semver.MustParse("v1.23.5"), } operatingSystems = []providerconfigtypes.OperatingSystem{ From 8fa0ac2851bb1759883d175178eedca8c1095538 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Thu, 5 May 2022 18:06:46 +0200 Subject: [PATCH 131/489] Add support for Kubernetes 1.24 and containerd 1.5 (#1272) * Update testdata generation Signed-off-by: Marvin Beckers * Use containerd 1.5 for k8s 1.24, do not set removed kubelet flag Signed-off-by: Marvin Beckers * Update fixtures Signed-off-by: Marvin Beckers * Use containerd 1.4 on amazon linux 2 Signed-off-by: Marvin Beckers * Update README with k8s 1.24 Signed-off-by: Marvin Beckers --- README.md | 1 + pkg/containerruntime/containerd.go | 13 +- pkg/containerruntime/containerruntime.go | 1 + pkg/containerruntime/docker.go | 7 +- pkg/userdata/amzn2/provider_test.go | 9 + .../containerd-kubelet-v1.20-aws.yaml | 2 +- .../amzn2/testdata/kubelet-v1.20-aws.yaml | 2 +- .../testdata/kubelet-v1.21-aws-external.yaml | 2 +- .../amzn2/testdata/kubelet-v1.21-aws.yaml | 2 +- .../kubelet-v1.21-vsphere-mirrors.yaml | 2 +- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 2 +- .../amzn2/testdata/kubelet-v1.21-vsphere.yaml | 2 +- .../amzn2/testdata/kubelet-v1.22-aws.yaml | 2 +- .../amzn2/testdata/kubelet-v1.23-aws.yaml | 2 +- .../amzn2/testdata/kubelet-v1.24-aws.yaml | 451 ++++++++++++++++ pkg/userdata/centos/provider_test.go | 9 + .../kubelet-containerd-v1.20-aws.yaml | 2 +- .../centos/testdata/kubelet-v1.20-aws.yaml | 2 +- .../testdata/kubelet-v1.21-aws-external.yaml | 2 +- .../centos/testdata/kubelet-v1.21-aws.yaml | 2 +- .../testdata/kubelet-v1.21-nutanix.yaml | 2 +- .../kubelet-v1.21-vsphere-mirrors.yaml | 2 +- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 2 +- .../testdata/kubelet-v1.21-vsphere.yaml | 2 +- .../centos/testdata/kubelet-v1.22-aws.yaml | 2 +- .../centos/testdata/kubelet-v1.23-aws.yaml | 2 +- .../testdata/kubelet-v1.23-nutanix.yaml | 2 +- .../centos/testdata/kubelet-v1.24-aws.yaml | 457 ++++++++++++++++ pkg/userdata/flatcar/provider_test.go | 94 ++++ .../flatcar/testdata/cloud-init_v1.20.14.yaml | 2 +- .../flatcar/testdata/cloud-init_v1.21.10.yaml | 2 +- .../flatcar/testdata/cloud-init_v1.21.8.yaml | 2 +- .../flatcar/testdata/cloud-init_v1.22.5.yaml | 2 +- .../flatcar/testdata/cloud-init_v1.22.7.yaml | 2 +- .../flatcar/testdata/cloud-init_v1.23.0.yaml | 2 +- .../flatcar/testdata/cloud-init_v1.23.5.yaml | 2 +- .../flatcar/testdata/cloud-init_v1.24.0.yaml | 501 ++++++++++++++++++ pkg/userdata/flatcar/testdata/containerd.yaml | 2 +- .../flatcar/testdata/ignition_v1.20.14.json | 2 +- .../flatcar/testdata/ignition_v1.21.10.json | 2 +- .../flatcar/testdata/ignition_v1.21.8.json | 2 +- .../flatcar/testdata/ignition_v1.22.5.json | 2 +- .../flatcar/testdata/ignition_v1.22.7.json | 2 +- .../flatcar/testdata/ignition_v1.23.0.json | 2 +- .../flatcar/testdata/ignition_v1.23.5.json | 2 +- .../flatcar/testdata/ignition_v1.24.0.json | 1 + pkg/userdata/helper/common_test.go | 1 + pkg/userdata/helper/kubelet.go | 14 +- .../testdata/download_binaries_v1.24.0.golden | 17 + ...let_systemd_unit_cloud-provider-set.golden | 2 +- ...t_systemd_unit_multiple-dns-servers.golden | 2 +- ...kublet_systemd_unit_pause-image-set.golden | 2 +- .../kublet_systemd_unit_taints-set.golden | 2 +- ...temd_unit_version-v1.20.14-external.golden | 2 +- ...ublet_systemd_unit_version-v1.20.14.golden | 2 +- ...temd_unit_version-v1.21.10-external.golden | 2 +- ...ublet_systemd_unit_version-v1.21.10.golden | 2 +- ...stemd_unit_version-v1.21.8-external.golden | 2 +- ...kublet_systemd_unit_version-v1.21.8.golden | 2 +- ...stemd_unit_version-v1.22.5-external.golden | 2 +- ...kublet_systemd_unit_version-v1.22.5.golden | 2 +- ...stemd_unit_version-v1.22.7-external.golden | 2 +- ...kublet_systemd_unit_version-v1.22.7.golden | 2 +- ...stemd_unit_version-v1.23.0-external.golden | 2 +- ...kublet_systemd_unit_version-v1.23.0.golden | 2 +- ...stemd_unit_version-v1.23.5-external.golden | 2 +- ...kublet_systemd_unit_version-v1.23.5.golden | 2 +- ...stemd_unit_version-v1.24.0-external.golden | 35 ++ ...kublet_systemd_unit_version-v1.24.0.golden | 34 ++ pkg/userdata/rhel/provider_test.go | 19 + .../kubelet-containerd-v1.20-aws.yaml | 2 +- .../rhel/testdata/kubelet-v1.20-aws.yaml | 2 +- .../rhel/testdata/kubelet-v1.21-aws.yaml | 2 +- .../rhel/testdata/kubelet-v1.22-aws.yaml | 2 +- .../rhel/testdata/kubelet-v1.22-nutanix.yaml | 2 +- .../testdata/kubelet-v1.23-aws-external.yaml | 2 +- .../rhel/testdata/kubelet-v1.23-aws.yaml | 2 +- .../kubelet-v1.23-vsphere-mirrors.yaml | 2 +- .../testdata/kubelet-v1.23-vsphere-proxy.yaml | 2 +- .../rhel/testdata/kubelet-v1.23-vsphere.yaml | 2 +- .../testdata/kubelet-v1.24-aws-external.yaml | 486 +++++++++++++++++ .../rhel/testdata/kubelet-v1.24-aws.yaml | 486 +++++++++++++++++ .../rhel/testdata/pod-cidr-azure-rhel.yaml | 2 +- pkg/userdata/rockylinux/provider_test.go | 9 + .../kubelet-containerd-v1.20-aws.yaml | 2 +- .../testdata/kubelet-v1.20-aws.yaml | 2 +- .../testdata/kubelet-v1.21-aws-external.yaml | 2 +- .../testdata/kubelet-v1.21-aws.yaml | 2 +- .../testdata/kubelet-v1.21-nutanix.yaml | 2 +- .../kubelet-v1.21-vsphere-mirrors.yaml | 2 +- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 2 +- .../testdata/kubelet-v1.21-vsphere.yaml | 2 +- .../testdata/kubelet-v1.22-aws.yaml | 2 +- .../testdata/kubelet-v1.23-aws.yaml | 2 +- .../testdata/kubelet-v1.23-nutanix.yaml | 2 +- .../testdata/kubelet-v1.24-aws.yaml | 452 ++++++++++++++++ pkg/userdata/sles/provider_test.go | 1 + .../sles/testdata/dist-upgrade-on-boot.yaml | 2 +- .../kubelet-version-without-v-prefix.yaml | 2 +- .../sles/testdata/multiple-dns-servers.yaml | 2 +- .../sles/testdata/multiple-ssh-keys.yaml | 2 +- .../openstack-overwrite-cloud-config.yaml | 2 +- pkg/userdata/sles/testdata/openstack.yaml | 2 +- .../sles/testdata/version-1.20.14.yaml | 2 +- .../sles/testdata/version-1.21.10.yaml | 2 +- .../sles/testdata/version-1.21.8.yaml | 2 +- .../sles/testdata/version-1.22.5.yaml | 2 +- .../sles/testdata/version-1.22.7.yaml | 2 +- .../sles/testdata/version-1.23.0.yaml | 2 +- .../sles/testdata/version-1.23.5.yaml | 2 +- .../sles/testdata/version-1.24.0.yaml | 438 +++++++++++++++ .../sles/testdata/vsphere-mirrors.yaml | 2 +- pkg/userdata/sles/testdata/vsphere-proxy.yaml | 2 +- pkg/userdata/sles/testdata/vsphere.yaml | 2 +- pkg/userdata/ubuntu/provider_test.go | 1 + pkg/userdata/ubuntu/testdata/containerd.yaml | 2 +- .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 2 +- .../kubelet-version-without-v-prefix.yaml | 2 +- .../ubuntu/testdata/multiple-dns-servers.yaml | 2 +- .../ubuntu/testdata/multiple-ssh-keys.yaml | 2 +- pkg/userdata/ubuntu/testdata/nutanix.yaml | 2 +- .../openstack-overwrite-cloud-config.yaml | 2 +- pkg/userdata/ubuntu/testdata/openstack.yaml | 2 +- .../ubuntu/testdata/version-1.20.14.yaml | 2 +- .../ubuntu/testdata/version-1.21.10.yaml | 2 +- .../ubuntu/testdata/version-1.21.8.yaml | 2 +- .../ubuntu/testdata/version-1.22.5.yaml | 2 +- .../ubuntu/testdata/version-1.22.7.yaml | 2 +- .../ubuntu/testdata/version-1.23.0.yaml | 2 +- .../ubuntu/testdata/version-1.23.5.yaml | 2 +- .../ubuntu/testdata/version-1.24.0.yaml | 460 ++++++++++++++++ .../ubuntu/testdata/vsphere-mirrors.yaml | 2 +- .../ubuntu/testdata/vsphere-proxy.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere.yaml | 2 +- test/e2e/provisioning/helper.go | 1 + 135 files changed, 4102 insertions(+), 114 deletions(-) create mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml create mode 100644 pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml create mode 100644 pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml create mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.24.0.json create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.24.0.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.0-external.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.0.golden create mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml create mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml create mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml create mode 100644 pkg/userdata/sles/testdata/version-1.24.0.yaml create mode 100644 pkg/userdata/ubuntu/testdata/version-1.24.0.yaml diff --git a/README.md b/README.md index 775c7ff99..370e76a23 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ machine-controller tries to follow the Kubernetes version Currently supported K8S versions are: +- 1.24 - 1.23 - 1.22 - 1.21 diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index 05599b23a..07280ab31 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -27,7 +27,8 @@ import ( ) const ( - DefaultContainerdVersion = "1.4" + LegacyContainerdVersion = "1.4" + DefaultContainerdVersion = "1.5" ) type Containerd struct { @@ -35,6 +36,7 @@ type Containerd struct { registryMirrors map[string][]string sandboxImage string registryCredentials map[string]AuthConfig + version string } func (eng *Containerd) ConfigFileName() string { @@ -57,6 +59,15 @@ func (eng *Containerd) ScriptFor(os types.OperatingSystem) (string, error) { ContainerdVersion: DefaultContainerdVersion, } + if eng.version != "" { + args.ContainerdVersion = eng.version + } + + // Amazon Linux 2 does not have containerd 1.5 + if eng.version == "" && os == types.OperatingSystemAmazonLinux2 { + args.ContainerdVersion = LegacyContainerdVersion + } + switch os { case types.OperatingSystemAmazonLinux2: err := containerdAmzn2Template.Execute(&buf, args) diff --git a/pkg/containerruntime/containerruntime.go b/pkg/containerruntime/containerruntime.go index 1cc2ea650..f9ce1934c 100644 --- a/pkg/containerruntime/containerruntime.go +++ b/pkg/containerruntime/containerruntime.go @@ -122,6 +122,7 @@ func (cfg Config) Engine(kubeletVersion *semver.Version) Engine { case cfg.Docker != nil: return docker case cfg.Containerd != nil: + containerd.version = LegacyContainerdVersion return containerd } diff --git a/pkg/containerruntime/docker.go b/pkg/containerruntime/docker.go index cb3a2c9ca..d12dfa11e 100644 --- a/pkg/containerruntime/docker.go +++ b/pkg/containerruntime/docker.go @@ -26,8 +26,9 @@ import ( ) const ( - DefaultDockerVersion = "19.03" - LegacyDockerVersion = "18.09" + DefaultDockerContainerdVersion = "1.4" + DefaultDockerVersion = "19.03" + LegacyDockerVersion = "18.09" ) type Docker struct { @@ -60,7 +61,7 @@ func (eng *Docker) ScriptFor(os types.OperatingSystem) (string, error) { ContainerdVersion string }{ DockerVersion: DefaultDockerVersion, - ContainerdVersion: DefaultContainerdVersion, + ContainerdVersion: DefaultDockerContainerdVersion, } switch os { diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index bb42d0651..9cd2057b2 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -174,6 +174,15 @@ func TestUserDataGeneration(t *testing.T) { }, }, }, + { + name: "kubelet-v1.24-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.24.0", + }, + }, + }, } defaultCloudProvider := &fakeCloudConfigProvider{ diff --git a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml index 4d153c419..78793e32b 100644 --- a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml @@ -219,7 +219,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -229,6 +228,7 @@ write_files: --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml index a02201885..70498fbfb 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml @@ -216,7 +216,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -226,6 +225,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml index 485632d9b..27d9eb15b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml @@ -216,7 +216,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=external \ --hostname-override=${KUBELET_HOSTNAME} \ @@ -226,6 +225,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml index 02d52a5a5..b2b7f81ad 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml @@ -216,7 +216,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -226,6 +225,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml index b8290b7cd..91b61d1c2 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -231,7 +231,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=vsphere \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -243,6 +242,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml index ccc154543..8a71ab415 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -231,7 +231,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=vsphere \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -243,6 +242,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml index 390fc9784..004b4504f 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml @@ -223,7 +223,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=vsphere \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -234,6 +233,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml index 581b817e6..5f0b13b30 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml @@ -216,7 +216,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -226,6 +225,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml index cf51102d6..d957ee4c3 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml @@ -216,7 +216,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -224,6 +223,7 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml new file mode 100644 index 000000000..6c2ae7203 --- /dev/null +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml @@ -0,0 +1,451 @@ +#cloud-config + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + + mkdir -p /etc/systemd/system/containerd.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index 532b9f561..37f36be41 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -184,6 +184,15 @@ func TestUserDataGeneration(t *testing.T) { }, cloudProviderName: stringPtr("nutanix"), }, + { + name: "kubelet-v1.24-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.24.0", + }, + }, + }, } defaultCloudProvider := &fakeCloudConfigProvider{ diff --git a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml index 3b61e8445..f1d71da8a 100644 --- a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml @@ -225,7 +225,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -235,6 +234,7 @@ write_files: --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml index 0fdc64ab5..c20c27f8a 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml @@ -226,7 +226,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -236,6 +235,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml index b320e4525..8084e5f05 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml @@ -226,7 +226,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=external \ --hostname-override=${KUBELET_HOSTNAME} \ @@ -236,6 +235,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml index d72e3d0a9..55a72e4fc 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml @@ -226,7 +226,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -236,6 +235,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml index 711776a8f..442979500 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml @@ -233,7 +233,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=nutanix \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -244,6 +243,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml index df9d20754..24a680856 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -241,7 +241,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=vsphere \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -253,6 +252,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml index 641e97670..513051ec2 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -241,7 +241,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=vsphere \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -253,6 +252,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml index 6e6e7d3bd..9bda409e9 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml @@ -233,7 +233,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=vsphere \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -244,6 +243,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index 6b886b62d..2f0466221 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -226,7 +226,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -236,6 +235,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index 744b6273c..bb6785c09 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -226,7 +226,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -234,6 +233,7 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml index e353cb0e0..a5119df13 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml @@ -233,7 +233,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=nutanix \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -242,6 +241,7 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml new file mode 100644 index 000000000..fb7f7adfa --- /dev/null +++ b/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml @@ -0,0 +1,457 @@ +#cloud-config + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index bd98b82de..18688dd67 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -214,6 +214,68 @@ func TestUserDataGeneration(t *testing.T) { ProvisioningUtility: Ignition, }, }, + { + name: "ignition_v1.23.5", + providerSpec: &providerconfigtypes.Config{ + CloudProvider: "vsphere", + SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, + CAPublicKey: "ssh-rsa AAABBB", + Network: &providerconfigtypes.NetworkConfig{ + CIDR: "192.168.81.4/24", + Gateway: "192.168.81.1", + DNS: providerconfigtypes.DNSConfig{ + Servers: []string{"8.8.8.8"}, + }, + }, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "v1.23.5", + }, + }, + ccProvider: &fakeCloudConfigProvider{ + name: "vsphere", + config: "{vsphere-config:true}", + err: nil, + }, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, + osConfig: &Config{ + DisableAutoUpdate: true, + ProvisioningUtility: Ignition, + }, + }, + { + name: "ignition_v1.24.0", + providerSpec: &providerconfigtypes.Config{ + CloudProvider: "vsphere", + SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, + CAPublicKey: "ssh-rsa AAABBB", + Network: &providerconfigtypes.NetworkConfig{ + CIDR: "192.168.81.4/24", + Gateway: "192.168.81.1", + DNS: providerconfigtypes.DNSConfig{ + Servers: []string{"8.8.8.8"}, + }, + }, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "v1.24.0", + }, + }, + ccProvider: &fakeCloudConfigProvider{ + name: "vsphere", + config: "{vsphere-config:true}", + err: nil, + }, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, + osConfig: &Config{ + DisableAutoUpdate: true, + ProvisioningUtility: Ignition, + }, + }, { name: "cloud-init_v1.21.10", providerSpec: &providerconfigtypes.Config{ @@ -307,6 +369,38 @@ func TestUserDataGeneration(t *testing.T) { ProvisioningUtility: CloudInit, }, }, + { + name: "cloud-init_v1.24.0", + providerSpec: &providerconfigtypes.Config{ + CloudProvider: "anexia", + SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, + CAPublicKey: "ssh-rsa AAABBB", + Network: &providerconfigtypes.NetworkConfig{ + CIDR: "192.168.81.4/24", + Gateway: "192.168.81.1", + DNS: providerconfigtypes.DNSConfig{ + Servers: []string{"8.8.8.8"}, + }, + }, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "v1.24.0", + }, + }, + ccProvider: &fakeCloudConfigProvider{ + name: "anexia", + config: "{anexia-config:true}", + err: nil, + }, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, + osConfig: &Config{ + DisableAutoUpdate: true, + ProvisioningUtility: CloudInit, + }, + }, + { name: "containerd", containerruntime: "containerd", diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml index 9bb1f3186..127f76dc9 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml @@ -118,7 +118,6 @@ coreos: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=anexia \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -129,6 +128,7 @@ coreos: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.10.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.10.yaml index 23526cb04..1c01066bb 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.10.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.10.yaml @@ -118,7 +118,6 @@ coreos: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=anexia \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -129,6 +128,7 @@ coreos: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml index 4a23fb10b..00ba54450 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml @@ -118,7 +118,6 @@ coreos: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=anexia \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -129,6 +128,7 @@ coreos: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml index 30c242335..7083a9070 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml @@ -118,7 +118,6 @@ coreos: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=anexia \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -129,6 +128,7 @@ coreos: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml index c94066c34..61378ccea 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml @@ -118,7 +118,6 @@ coreos: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=anexia \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -129,6 +128,7 @@ coreos: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml index 4a2251c00..a10722280 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml @@ -118,7 +118,6 @@ coreos: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=anexia \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -127,6 +126,7 @@ coreos: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml index 708e6f6be..e78ec28c6 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml @@ -118,7 +118,6 @@ coreos: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=anexia \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -127,6 +126,7 @@ coreos: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml new file mode 100644 index 000000000..91fc82387 --- /dev/null +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml @@ -0,0 +1,501 @@ +#cloud-config + +users: +- name: core + ssh_authorized_keys: + - ssh-rsa AAABBB + - ssh-rsa CCCDDD + + +coreos: + units: + - name: static-nic.network + content: | + [Match] + # Because of difficulty predicting specific NIC names on different cloud providers, + # we only support static addressing on VSphere. There should be a single NIC attached + # that we will match by name prefix 'en' which denotes ethernet devices. + Name=en* + + [Network] + DHCP=no + Address=192.168.81.4/24 + Gateway=192.168.81.1 + DNS=8.8.8.8 + + - name: update-engine.service + command: stop + mask: true + - name: locksmithd.service + command: stop + mask: true + - name: download-script.service + enable: true + command: start + content: | + [Unit] + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/download.sh + [Install] + WantedBy=multi-user.target + + - name: kubelet-healthcheck.service + enable: true + command: start + drop-ins: + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + + - name: nodeip.service + enable: true + command: start + content: | + [Unit] + Description=Setup Kubelet Node IP Env + Requires=network-online.target + After=network-online.target + + [Service] + ExecStart=/opt/bin/setup_net_env.sh + RemainAfterExit=yes + Type=oneshot + [Install] + WantedBy=multi-user.target + + - name: kubelet.service + enable: true + command: start + drop-ins: + - name: 10-nodeip.conf + content: | + [Service] + EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=anexia \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + + - name: apply-sysctl-settings.service + enable: true + command: start + content: | + [Unit] + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + ExecStart=/opt/bin/apply_sysctl_settings.sh + [Install] + WantedBy=multi-user.target + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + permissions: "0644" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/etc/kubernetes/kubelet.conf" + permissions: "0644" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /opt/load-kernel-modules.sh + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: /etc/sysctl.d/k8s.conf + permissions: "0644" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: /etc/kubernetes/bootstrap-kubelet.conf + permissions: "0400" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: /etc/kubernetes/cloud-config + permissions: "0400" + content: | + {anexia-config:true} + +- path: /etc/kubernetes/pki/ca.crt + permissions: "0644" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + +- path: /etc/hostname + permissions: "0600" + content: 'node1' + +- path: /etc/ssh/sshd_config + permissions: "0600" + user: root + content: | + # Use most defaults for sshd configuration. + Subsystem sftp internal-sftp + ClientAliveInterval 180 + UseDNS no + UsePAM yes + PrintLastLog no # handled by PAM + PrintMotd no # handled by PAM + PasswordAuthentication no + ChallengeResponseAuthentication no + +- path: /opt/bin/download.sh + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v0.8.7}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.24.0}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + mkdir -p /etc/systemd/system/containerd.service.d + + cat < /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=${KUBELET_HOSTNAME} \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + +rh_subscription: + username: "" + password: "" + auto-attach: false + +runcmd: +- systemctl start setup.service +- systemctl start disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml new file mode 100644 index 000000000..6a3f85086 --- /dev/null +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml @@ -0,0 +1,486 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + +rh_subscription: + username: "" + password: "" + auto-attach: false + +runcmd: +- systemctl start setup.service +- systemctl start disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index c753a3d14..eba41a222 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -228,7 +228,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=azure \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -239,6 +238,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/rockylinux/provider_test.go b/pkg/userdata/rockylinux/provider_test.go index db89c9fb6..3927c846f 100644 --- a/pkg/userdata/rockylinux/provider_test.go +++ b/pkg/userdata/rockylinux/provider_test.go @@ -174,6 +174,15 @@ func TestUserDataGeneration(t *testing.T) { }, }, }, + { + name: "kubelet-v1.24-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.24.0", + }, + }, + }, { name: "kubelet-v1.23-nutanix", spec: clusterv1alpha1.MachineSpec{ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml index 575c8915d..9e1fe6ad2 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml @@ -220,7 +220,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -230,6 +229,7 @@ write_files: --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml index 7cb5b1ca4..632c44b74 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml @@ -221,7 +221,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -231,6 +230,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml index 22d01bf21..937197439 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml @@ -221,7 +221,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=external \ --hostname-override=${KUBELET_HOSTNAME} \ @@ -231,6 +230,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml index a3d3422e1..b5f566c62 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml @@ -221,7 +221,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -231,6 +230,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml index 9b53ab40d..d10782447 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml @@ -228,7 +228,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=nutanix \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -239,6 +238,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 33a5c2834..a9f97caef 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -236,7 +236,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=vsphere \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -248,6 +247,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml index 6720f3545..43bc2130e 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -236,7 +236,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=vsphere \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -248,6 +247,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml index 2e9edf240..551beb181 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml @@ -228,7 +228,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=vsphere \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -239,6 +238,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml index a70be1fd9..dadbb77a5 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml @@ -221,7 +221,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -231,6 +230,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml index d5852f0fb..0c78587d7 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml @@ -221,7 +221,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=aws \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -229,6 +228,7 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml index 9cf6040c1..49fb9bf58 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml @@ -228,7 +228,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=nutanix \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -237,6 +236,7 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml new file mode 100644 index 000000000..73060d423 --- /dev/null +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml @@ -0,0 +1,452 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + tar \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/sles/provider_test.go b/pkg/userdata/sles/provider_test.go index f6225ebca..302344db9 100644 --- a/pkg/userdata/sles/provider_test.go +++ b/pkg/userdata/sles/provider_test.go @@ -129,6 +129,7 @@ func simpleVersionTests() []userDataTestCase { semver.MustParse("v1.21.10"), semver.MustParse("v1.22.7"), semver.MustParse("v1.23.5"), + semver.MustParse("v1.24.0"), } var tests []userDataTestCase diff --git a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml index 99ceebed7..97b60ae7c 100644 --- a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml @@ -183,7 +183,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -192,6 +191,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml index 54e826d01..de832d974 100644 --- a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml @@ -181,7 +181,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -190,6 +189,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml index c5dcbd600..affbbf41e 100644 --- a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml @@ -181,7 +181,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -190,6 +189,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml index e22248039..9d4ed5312 100644 --- a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml @@ -183,7 +183,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -192,6 +191,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml index 8858a7ce1..096835dbf 100644 --- a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml @@ -181,7 +181,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=openstack \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -192,6 +191,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/testdata/openstack.yaml b/pkg/userdata/sles/testdata/openstack.yaml index f67ac1ba0..02f52bef2 100644 --- a/pkg/userdata/sles/testdata/openstack.yaml +++ b/pkg/userdata/sles/testdata/openstack.yaml @@ -181,7 +181,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=openstack \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -192,6 +191,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/testdata/version-1.20.14.yaml b/pkg/userdata/sles/testdata/version-1.20.14.yaml index 70ed85fd0..6c50c9958 100644 --- a/pkg/userdata/sles/testdata/version-1.20.14.yaml +++ b/pkg/userdata/sles/testdata/version-1.20.14.yaml @@ -181,7 +181,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -190,6 +189,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/testdata/version-1.21.10.yaml b/pkg/userdata/sles/testdata/version-1.21.10.yaml index 5c7a9a845..fb99d31fe 100644 --- a/pkg/userdata/sles/testdata/version-1.21.10.yaml +++ b/pkg/userdata/sles/testdata/version-1.21.10.yaml @@ -181,7 +181,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -190,6 +189,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/testdata/version-1.21.8.yaml b/pkg/userdata/sles/testdata/version-1.21.8.yaml index 53c2958d0..726859301 100644 --- a/pkg/userdata/sles/testdata/version-1.21.8.yaml +++ b/pkg/userdata/sles/testdata/version-1.21.8.yaml @@ -181,7 +181,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -190,6 +189,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/testdata/version-1.22.5.yaml b/pkg/userdata/sles/testdata/version-1.22.5.yaml index 54e826d01..de832d974 100644 --- a/pkg/userdata/sles/testdata/version-1.22.5.yaml +++ b/pkg/userdata/sles/testdata/version-1.22.5.yaml @@ -181,7 +181,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -190,6 +189,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/testdata/version-1.22.7.yaml b/pkg/userdata/sles/testdata/version-1.22.7.yaml index f15590efb..5020276b6 100644 --- a/pkg/userdata/sles/testdata/version-1.22.7.yaml +++ b/pkg/userdata/sles/testdata/version-1.22.7.yaml @@ -181,7 +181,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -190,6 +189,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/testdata/version-1.23.0.yaml b/pkg/userdata/sles/testdata/version-1.23.0.yaml index a24f95948..080177f7e 100644 --- a/pkg/userdata/sles/testdata/version-1.23.0.yaml +++ b/pkg/userdata/sles/testdata/version-1.23.0.yaml @@ -181,13 +181,13 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/testdata/version-1.23.5.yaml b/pkg/userdata/sles/testdata/version-1.23.5.yaml index 6e4904368..3cef64351 100644 --- a/pkg/userdata/sles/testdata/version-1.23.5.yaml +++ b/pkg/userdata/sles/testdata/version-1.23.5.yaml @@ -181,13 +181,13 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/testdata/version-1.24.0.yaml b/pkg/userdata/sles/testdata/version-1.24.0.yaml new file mode 100644 index 000000000..afbb0faef --- /dev/null +++ b/pkg/userdata/sles/testdata/version-1.24.0.yaml @@ -0,0 +1,438 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + systemctl restart systemd-modules-load.service + sysctl --system + + zypper --non-interactive --quiet --color install ebtables \ + ceph-common \ + e2fsprogs \ + jq \ + socat \ + ipvsadm + + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v0.8.7}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.24.0}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=containerd.service + After=containerd.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + +- path: /etc/systemd/system/docker.service.d/environment.conf + permissions: "0644" + content: | + [Service] + EnvironmentFile=-/etc/environment + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml index ed88682b4..092a051c4 100644 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml @@ -191,7 +191,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=vsphere \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -203,6 +202,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml index cafb9cfd9..2a706d7ee 100644 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/sles/testdata/vsphere-proxy.yaml @@ -191,7 +191,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=vsphere \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -203,6 +202,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml index 5e377e19b..2c7a9d155 100644 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ b/pkg/userdata/sles/testdata/vsphere.yaml @@ -182,7 +182,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=vsphere \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -193,6 +192,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 63e85822b..8f564b845 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -130,6 +130,7 @@ func simpleVersionTests() []userDataTestCase { semver.MustParse("v1.21.10"), semver.MustParse("v1.22.7"), semver.MustParse("v1.23.5"), + semver.MustParse("v1.24.0"), } var tests []userDataTestCase diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index b4e6b55b9..593abf213 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -224,7 +224,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -233,6 +232,7 @@ write_files: --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 95e3ec8fc..848ed03bb 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -224,7 +224,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -233,6 +232,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index b7ceb1c17..036278ac9 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -222,7 +222,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -231,6 +230,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 32d7d24e1..c9ac3fc14 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -222,7 +222,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -231,6 +230,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 7652fc67c..2600324f9 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -224,7 +224,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -233,6 +232,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index b7d9c4dc1..0255b4787 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -225,7 +225,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=nutanix \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -236,6 +235,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index 8be555a3a..3a87ca5cb 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -222,7 +222,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=openstack \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -233,6 +232,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 65ece0806..b2e9a633a 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -222,7 +222,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=openstack \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -233,6 +232,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml b/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml index 4b5e69de5..eaea76513 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml @@ -222,7 +222,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -231,6 +230,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml b/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml index 9f698da58..59f3a7727 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml @@ -222,7 +222,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -231,6 +230,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml b/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml index 3f914dbf5..a7a0e0649 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml @@ -222,7 +222,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -231,6 +230,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml index 470ae4581..be45b45dc 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml @@ -222,7 +222,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -231,6 +230,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml b/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml index b7ceb1c17..036278ac9 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml @@ -222,7 +222,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ @@ -231,6 +230,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml index a304d0815..106228754 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml @@ -222,13 +222,13 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml index 71de2cf2c..a161862a5 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml @@ -222,13 +222,13 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml new file mode 100644 index 000000000..d825aba93 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml @@ -0,0 +1,460 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index c93345dc5..3019f8c48 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -232,7 +232,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=vsphere \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -244,6 +243,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 281d9f21e..f8ffd4b8e 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -232,7 +232,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=vsphere \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -244,6 +243,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 9cbff06ec..16eb24573 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -223,7 +223,6 @@ write_files: --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ - --network-plugin=cni \ --cert-dir=/etc/kubernetes/pki \ --cloud-provider=vsphere \ --cloud-config=/etc/kubernetes/cloud-config \ @@ -234,6 +233,7 @@ write_files: --container-runtime-endpoint=unix:///var/run/dockershim.sock \ --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 9cb39d9c7..8fc4f362a 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -36,6 +36,7 @@ var ( semver.MustParse("v1.21.10"), semver.MustParse("v1.22.7"), semver.MustParse("v1.23.5"), + semver.MustParse("v1.24.0"), } operatingSystems = []providerconfigtypes.OperatingSystem{ From 42b079cd6fccb3bfbbb269ee36f9829e4cfd5e4e Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Fri, 6 May 2022 10:04:47 +0200 Subject: [PATCH 132/489] Remove obsolete testdata that was not cleaned up (#1275) Signed-off-by: Marvin Beckers --- .../containerd-kubelet-v1.20-aws.yaml | 454 ---------------- .../amzn2/testdata/kubelet-v1.20-aws.yaml | 434 ---------------- .../kubelet-containerd-v1.20-aws.yaml | 460 ---------------- .../centos/testdata/kubelet-v1.20-aws.yaml | 444 ---------------- .../testdata/kubelet-v1.21-nutanix.yaml | 452 ---------------- .../flatcar/testdata/cloud-init_v1.20.14.yaml | 477 ----------------- .../flatcar/testdata/cloud-init_v1.21.8.yaml | 477 ----------------- .../flatcar/testdata/cloud-init_v1.22.5.yaml | 477 ----------------- .../flatcar/testdata/cloud-init_v1.23.0.yaml | 475 ----------------- .../flatcar/testdata/ignition_v1.20.14.json | 1 - .../flatcar/testdata/ignition_v1.21.8.json | 1 - .../flatcar/testdata/ignition_v1.22.5.json | 1 - .../flatcar/testdata/ignition_v1.23.0.json | 1 - .../download_binaries_v1.20.14.golden | 17 - .../testdata/download_binaries_v1.21.8.golden | 17 - .../testdata/download_binaries_v1.22.5.golden | 17 - .../testdata/download_binaries_v1.23.0.golden | 17 - ...temd_unit_version-v1.20.14-external.golden | 38 -- ...ublet_systemd_unit_version-v1.20.14.golden | 37 -- ...stemd_unit_version-v1.21.8-external.golden | 38 -- ...kublet_systemd_unit_version-v1.21.8.golden | 37 -- ...stemd_unit_version-v1.22.5-external.golden | 38 -- ...kublet_systemd_unit_version-v1.22.5.golden | 37 -- ...stemd_unit_version-v1.23.0-external.golden | 36 -- ...kublet_systemd_unit_version-v1.23.0.golden | 35 -- .../kubelet-containerd-v1.20-aws.yaml | 489 ------------------ .../rhel/testdata/kubelet-v1.20-aws.yaml | 473 ----------------- .../kubelet-containerd-v1.20-aws.yaml | 455 ---------------- .../testdata/kubelet-v1.20-aws.yaml | 439 ---------------- .../testdata/kubelet-v1.21-nutanix.yaml | 447 ---------------- .../sles/testdata/version-1.20.14.yaml | 424 --------------- .../sles/testdata/version-1.21.8.yaml | 424 --------------- .../sles/testdata/version-1.22.5.yaml | 424 --------------- .../sles/testdata/version-1.23.0.yaml | 422 --------------- .../ubuntu/testdata/version-1.20.14.yaml | 446 ---------------- .../ubuntu/testdata/version-1.21.8.yaml | 446 ---------------- .../ubuntu/testdata/version-1.22.5.yaml | 446 ---------------- .../ubuntu/testdata/version-1.23.0.yaml | 444 ---------------- 38 files changed, 10297 deletions(-) delete mode 100644 pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml delete mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml delete mode 100644 pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml delete mode 100644 pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml delete mode 100644 pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml delete mode 100644 pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml delete mode 100644 pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml delete mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.20.14.json delete mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.21.8.json delete mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.22.5.json delete mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.23.0.json delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.20.14.golden delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.21.8.golden delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.22.5.golden delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.23.0.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0.golden delete mode 100644 pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml delete mode 100644 pkg/userdata/sles/testdata/version-1.20.14.yaml delete mode 100644 pkg/userdata/sles/testdata/version-1.21.8.yaml delete mode 100644 pkg/userdata/sles/testdata/version-1.22.5.yaml delete mode 100644 pkg/userdata/sles/testdata/version-1.23.0.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.20.14.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.21.8.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.22.5.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.23.0.yaml diff --git a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml deleted file mode 100644 index 78793e32b..000000000 --- a/pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml +++ /dev/null @@ -1,454 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml deleted file mode 100644 index 70498fbfb..000000000 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml +++ /dev/null @@ -1,434 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml deleted file mode 100644 index f1d71da8a..000000000 --- a/pkg/userdata/centos/testdata/kubelet-containerd-v1.20-aws.yaml +++ /dev/null @@ -1,460 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml deleted file mode 100644 index c20c27f8a..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.20-aws.yaml +++ /dev/null @@ -1,444 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml deleted file mode 100644 index 442979500..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-nutanix.yaml +++ /dev/null @@ -1,452 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - iscsi-initiator-utils \ - ipvsadm - systemctl enable --now iscsid - - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=nutanix \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml deleted file mode 100644 index 127f76dc9..000000000 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.20.14.yaml +++ /dev/null @@ -1,477 +0,0 @@ -#cloud-config - -users: -- name: core - ssh_authorized_keys: - - ssh-rsa AAABBB - - ssh-rsa CCCDDD - - -coreos: - units: - - name: static-nic.network - content: | - [Match] - # Because of difficulty predicting specific NIC names on different cloud providers, - # we only support static addressing on VSphere. There should be a single NIC attached - # that we will match by name prefix 'en' which denotes ethernet devices. - Name=en* - - [Network] - DHCP=no - Address=192.168.81.4/24 - Gateway=192.168.81.1 - DNS=8.8.8.8 - - - name: update-engine.service - command: stop - mask: true - - name: locksmithd.service - command: stop - mask: true - - name: download-script.service - enable: true - command: start - content: | - [Unit] - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/download.sh - [Install] - WantedBy=multi-user.target - - - name: kubelet-healthcheck.service - enable: true - command: start - drop-ins: - - name: 40-download.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - - - name: nodeip.service - enable: true - command: start - content: | - [Unit] - Description=Setup Kubelet Node IP Env - Requires=network-online.target - After=network-online.target - - [Service] - ExecStart=/opt/bin/setup_net_env.sh - RemainAfterExit=yes - Type=oneshot - [Install] - WantedBy=multi-user.target - - - name: kubelet.service - enable: true - command: start - drop-ins: - - name: 10-nodeip.conf - content: | - [Service] - EnvironmentFile=/etc/kubernetes/nodeip.conf - - name: 40-download.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=anexia \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - - - name: apply-sysctl-settings.service - enable: true - command: start - content: | - [Unit] - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - ExecStart=/opt/bin/apply_sysctl_settings.sh - [Install] - WantedBy=multi-user.target - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - permissions: "0644" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/etc/kubernetes/kubelet.conf" - permissions: "0644" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /opt/load-kernel-modules.sh - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: /etc/sysctl.d/k8s.conf - permissions: "0644" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: /etc/kubernetes/bootstrap-kubelet.conf - permissions: "0400" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: /etc/kubernetes/cloud-config - permissions: "0400" - content: | - {anexia-config:true} - -- path: /etc/kubernetes/pki/ca.crt - permissions: "0644" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - -- path: /etc/hostname - permissions: "0600" - content: 'node1' - -- path: /etc/ssh/sshd_config - permissions: "0600" - user: root - content: | - # Use most defaults for sshd configuration. - Subsystem sftp internal-sftp - ClientAliveInterval 180 - UseDNS no - UsePAM yes - PrintLastLog no # handled by PAM - PrintMotd no # handled by PAM - PasswordAuthentication no - ChallengeResponseAuthentication no - -- path: /opt/bin/download.sh - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.20.14}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - systemctl daemon-reload - systemctl enable --now docker - - systemctl disable download-script.service - -- path: /opt/bin/apply_sysctl_settings.sh - permissions: "0755" - user: root - content: | - #!/bin/bash - set -xeuo pipefail - sysctl --system - systemctl disable apply-sysctl-settings.service - -- path: "/etc/ssh/trusted-user-ca-keys.pem" - content: | - ssh-rsa AAABBB - -- path: "/etc/ssh/sshd_config" - content: | - TrustedUserCAKeys /etc/ssh/trusted-user-ca-keys.pem - CASignatureAlgorithms ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,ssh-ed25519,rsa-sha2-512,rsa-sha2-256,ssh-rsa - append: true - -- path: /etc/docker/daemon.json - permissions: "0644" - user: root - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/crictl.yaml - permissions: "0644" - user: root - content: | - runtime-endpoint: unix:///run/containerd/containerd.sock diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml deleted file mode 100644 index 00ba54450..000000000 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.8.yaml +++ /dev/null @@ -1,477 +0,0 @@ -#cloud-config - -users: -- name: core - ssh_authorized_keys: - - ssh-rsa AAABBB - - ssh-rsa CCCDDD - - -coreos: - units: - - name: static-nic.network - content: | - [Match] - # Because of difficulty predicting specific NIC names on different cloud providers, - # we only support static addressing on VSphere. There should be a single NIC attached - # that we will match by name prefix 'en' which denotes ethernet devices. - Name=en* - - [Network] - DHCP=no - Address=192.168.81.4/24 - Gateway=192.168.81.1 - DNS=8.8.8.8 - - - name: update-engine.service - command: stop - mask: true - - name: locksmithd.service - command: stop - mask: true - - name: download-script.service - enable: true - command: start - content: | - [Unit] - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/download.sh - [Install] - WantedBy=multi-user.target - - - name: kubelet-healthcheck.service - enable: true - command: start - drop-ins: - - name: 40-download.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - - - name: nodeip.service - enable: true - command: start - content: | - [Unit] - Description=Setup Kubelet Node IP Env - Requires=network-online.target - After=network-online.target - - [Service] - ExecStart=/opt/bin/setup_net_env.sh - RemainAfterExit=yes - Type=oneshot - [Install] - WantedBy=multi-user.target - - - name: kubelet.service - enable: true - command: start - drop-ins: - - name: 10-nodeip.conf - content: | - [Service] - EnvironmentFile=/etc/kubernetes/nodeip.conf - - name: 40-download.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=anexia \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - - - name: apply-sysctl-settings.service - enable: true - command: start - content: | - [Unit] - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - ExecStart=/opt/bin/apply_sysctl_settings.sh - [Install] - WantedBy=multi-user.target - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - permissions: "0644" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/etc/kubernetes/kubelet.conf" - permissions: "0644" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /opt/load-kernel-modules.sh - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: /etc/sysctl.d/k8s.conf - permissions: "0644" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: /etc/kubernetes/bootstrap-kubelet.conf - permissions: "0400" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: /etc/kubernetes/cloud-config - permissions: "0400" - content: | - {anexia-config:true} - -- path: /etc/kubernetes/pki/ca.crt - permissions: "0644" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - -- path: /etc/hostname - permissions: "0600" - content: 'node1' - -- path: /etc/ssh/sshd_config - permissions: "0600" - user: root - content: | - # Use most defaults for sshd configuration. - Subsystem sftp internal-sftp - ClientAliveInterval 180 - UseDNS no - UsePAM yes - PrintLastLog no # handled by PAM - PrintMotd no # handled by PAM - PasswordAuthentication no - ChallengeResponseAuthentication no - -- path: /opt/bin/download.sh - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - systemctl daemon-reload - systemctl enable --now docker - - systemctl disable download-script.service - -- path: /opt/bin/apply_sysctl_settings.sh - permissions: "0755" - user: root - content: | - #!/bin/bash - set -xeuo pipefail - sysctl --system - systemctl disable apply-sysctl-settings.service - -- path: "/etc/ssh/trusted-user-ca-keys.pem" - content: | - ssh-rsa AAABBB - -- path: "/etc/ssh/sshd_config" - content: | - TrustedUserCAKeys /etc/ssh/trusted-user-ca-keys.pem - CASignatureAlgorithms ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,ssh-ed25519,rsa-sha2-512,rsa-sha2-256,ssh-rsa - append: true - -- path: /etc/docker/daemon.json - permissions: "0644" - user: root - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/crictl.yaml - permissions: "0644" - user: root - content: | - runtime-endpoint: unix:///run/containerd/containerd.sock diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml deleted file mode 100644 index 7083a9070..000000000 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.5.yaml +++ /dev/null @@ -1,477 +0,0 @@ -#cloud-config - -users: -- name: core - ssh_authorized_keys: - - ssh-rsa AAABBB - - ssh-rsa CCCDDD - - -coreos: - units: - - name: static-nic.network - content: | - [Match] - # Because of difficulty predicting specific NIC names on different cloud providers, - # we only support static addressing on VSphere. There should be a single NIC attached - # that we will match by name prefix 'en' which denotes ethernet devices. - Name=en* - - [Network] - DHCP=no - Address=192.168.81.4/24 - Gateway=192.168.81.1 - DNS=8.8.8.8 - - - name: update-engine.service - command: stop - mask: true - - name: locksmithd.service - command: stop - mask: true - - name: download-script.service - enable: true - command: start - content: | - [Unit] - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/download.sh - [Install] - WantedBy=multi-user.target - - - name: kubelet-healthcheck.service - enable: true - command: start - drop-ins: - - name: 40-download.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - - - name: nodeip.service - enable: true - command: start - content: | - [Unit] - Description=Setup Kubelet Node IP Env - Requires=network-online.target - After=network-online.target - - [Service] - ExecStart=/opt/bin/setup_net_env.sh - RemainAfterExit=yes - Type=oneshot - [Install] - WantedBy=multi-user.target - - - name: kubelet.service - enable: true - command: start - drop-ins: - - name: 10-nodeip.conf - content: | - [Service] - EnvironmentFile=/etc/kubernetes/nodeip.conf - - name: 40-download.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=anexia \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - - - name: apply-sysctl-settings.service - enable: true - command: start - content: | - [Unit] - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - ExecStart=/opt/bin/apply_sysctl_settings.sh - [Install] - WantedBy=multi-user.target - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - permissions: "0644" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/etc/kubernetes/kubelet.conf" - permissions: "0644" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /opt/load-kernel-modules.sh - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: /etc/sysctl.d/k8s.conf - permissions: "0644" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: /etc/kubernetes/bootstrap-kubelet.conf - permissions: "0400" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: /etc/kubernetes/cloud-config - permissions: "0400" - content: | - {anexia-config:true} - -- path: /etc/kubernetes/pki/ca.crt - permissions: "0644" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - -- path: /etc/hostname - permissions: "0600" - content: 'node1' - -- path: /etc/ssh/sshd_config - permissions: "0600" - user: root - content: | - # Use most defaults for sshd configuration. - Subsystem sftp internal-sftp - ClientAliveInterval 180 - UseDNS no - UsePAM yes - PrintLastLog no # handled by PAM - PrintMotd no # handled by PAM - PasswordAuthentication no - ChallengeResponseAuthentication no - -- path: /opt/bin/download.sh - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - systemctl daemon-reload - systemctl enable --now docker - - systemctl disable download-script.service - -- path: /opt/bin/apply_sysctl_settings.sh - permissions: "0755" - user: root - content: | - #!/bin/bash - set -xeuo pipefail - sysctl --system - systemctl disable apply-sysctl-settings.service - -- path: "/etc/ssh/trusted-user-ca-keys.pem" - content: | - ssh-rsa AAABBB - -- path: "/etc/ssh/sshd_config" - content: | - TrustedUserCAKeys /etc/ssh/trusted-user-ca-keys.pem - CASignatureAlgorithms ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,ssh-ed25519,rsa-sha2-512,rsa-sha2-256,ssh-rsa - append: true - -- path: /etc/docker/daemon.json - permissions: "0644" - user: root - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/crictl.yaml - permissions: "0644" - user: root - content: | - runtime-endpoint: unix:///run/containerd/containerd.sock diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml deleted file mode 100644 index a10722280..000000000 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.0.yaml +++ /dev/null @@ -1,475 +0,0 @@ -#cloud-config - -users: -- name: core - ssh_authorized_keys: - - ssh-rsa AAABBB - - ssh-rsa CCCDDD - - -coreos: - units: - - name: static-nic.network - content: | - [Match] - # Because of difficulty predicting specific NIC names on different cloud providers, - # we only support static addressing on VSphere. There should be a single NIC attached - # that we will match by name prefix 'en' which denotes ethernet devices. - Name=en* - - [Network] - DHCP=no - Address=192.168.81.4/24 - Gateway=192.168.81.1 - DNS=8.8.8.8 - - - name: update-engine.service - command: stop - mask: true - - name: locksmithd.service - command: stop - mask: true - - name: download-script.service - enable: true - command: start - content: | - [Unit] - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/download.sh - [Install] - WantedBy=multi-user.target - - - name: kubelet-healthcheck.service - enable: true - command: start - drop-ins: - - name: 40-download.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - - - name: nodeip.service - enable: true - command: start - content: | - [Unit] - Description=Setup Kubelet Node IP Env - Requires=network-online.target - After=network-online.target - - [Service] - ExecStart=/opt/bin/setup_net_env.sh - RemainAfterExit=yes - Type=oneshot - [Install] - WantedBy=multi-user.target - - - name: kubelet.service - enable: true - command: start - drop-ins: - - name: 10-nodeip.conf - content: | - [Service] - EnvironmentFile=/etc/kubernetes/nodeip.conf - - name: 40-download.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=anexia \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - - - name: apply-sysctl-settings.service - enable: true - command: start - content: | - [Unit] - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - ExecStart=/opt/bin/apply_sysctl_settings.sh - [Install] - WantedBy=multi-user.target - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - permissions: "0644" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/etc/kubernetes/kubelet.conf" - permissions: "0644" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /opt/load-kernel-modules.sh - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: /etc/sysctl.d/k8s.conf - permissions: "0644" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: /etc/kubernetes/bootstrap-kubelet.conf - permissions: "0400" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: /etc/kubernetes/cloud-config - permissions: "0400" - content: | - {anexia-config:true} - -- path: /etc/kubernetes/pki/ca.crt - permissions: "0644" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - -- path: /etc/hostname - permissions: "0600" - content: 'node1' - -- path: /etc/ssh/sshd_config - permissions: "0600" - user: root - content: | - # Use most defaults for sshd configuration. - Subsystem sftp internal-sftp - ClientAliveInterval 180 - UseDNS no - UsePAM yes - PrintLastLog no # handled by PAM - PrintMotd no # handled by PAM - PasswordAuthentication no - ChallengeResponseAuthentication no - -- path: /opt/bin/download.sh - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - systemctl daemon-reload - systemctl enable --now docker - - systemctl disable download-script.service - -- path: /opt/bin/apply_sysctl_settings.sh - permissions: "0755" - user: root - content: | - #!/bin/bash - set -xeuo pipefail - sysctl --system - systemctl disable apply-sysctl-settings.service - -- path: "/etc/ssh/trusted-user-ca-keys.pem" - content: | - ssh-rsa AAABBB - -- path: "/etc/ssh/sshd_config" - content: | - TrustedUserCAKeys /etc/ssh/trusted-user-ca-keys.pem - CASignatureAlgorithms ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,ssh-ed25519,rsa-sha2-512,rsa-sha2-256,ssh-rsa - append: true - -- path: /etc/docker/daemon.json - permissions: "0644" - user: root - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/crictl.yaml - permissions: "0644" - user: root - content: | - runtime-endpoint: unix:///run/containerd/containerd.sock diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json b/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json deleted file mode 100644 index f56bbe14c..000000000 --- a/pkg/userdata/flatcar/testdata/ignition_v1.20.14.json +++ /dev/null @@ -1 +0,0 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.20.14%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json b/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json deleted file mode 100644 index 102dd7b03..000000000 --- a/pkg/userdata/flatcar/testdata/ignition_v1.21.8.json +++ /dev/null @@ -1 +0,0 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.8%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json b/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json deleted file mode 100644 index 12115eba8..000000000 --- a/pkg/userdata/flatcar/testdata/ignition_v1.22.5.json +++ /dev/null @@ -1 +0,0 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json deleted file mode 100644 index c2fb0662d..000000000 --- a/pkg/userdata/flatcar/testdata/ignition_v1.23.0.json +++ /dev/null @@ -1 +0,0 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.20.14.golden b/pkg/userdata/helper/testdata/download_binaries_v1.20.14.golden deleted file mode 100644 index 3d8518a34..000000000 --- a/pkg/userdata/helper/testdata/download_binaries_v1.20.14.golden +++ /dev/null @@ -1,17 +0,0 @@ -mkdir -p /opt/bin/ -mkdir -p /var/lib/calico -mkdir -p /etc/kubernetes/manifests -mkdir -p /etc/cni/net.d -mkdir -p /opt/cni/bin -if [ ! -f /opt/cni/bin/loopback ]; then - curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - -fi -if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.20.14/bin/linux/amd64/kubelet - chmod +x /opt/bin/kubelet -fi - -if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh -fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.21.8.golden b/pkg/userdata/helper/testdata/download_binaries_v1.21.8.golden deleted file mode 100644 index e31636457..000000000 --- a/pkg/userdata/helper/testdata/download_binaries_v1.21.8.golden +++ /dev/null @@ -1,17 +0,0 @@ -mkdir -p /opt/bin/ -mkdir -p /var/lib/calico -mkdir -p /etc/kubernetes/manifests -mkdir -p /etc/cni/net.d -mkdir -p /opt/cni/bin -if [ ! -f /opt/cni/bin/loopback ]; then - curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - -fi -if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.21.8/bin/linux/amd64/kubelet - chmod +x /opt/bin/kubelet -fi - -if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh -fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.22.5.golden b/pkg/userdata/helper/testdata/download_binaries_v1.22.5.golden deleted file mode 100644 index 5a6d5e8b8..000000000 --- a/pkg/userdata/helper/testdata/download_binaries_v1.22.5.golden +++ /dev/null @@ -1,17 +0,0 @@ -mkdir -p /opt/bin/ -mkdir -p /var/lib/calico -mkdir -p /etc/kubernetes/manifests -mkdir -p /etc/cni/net.d -mkdir -p /opt/cni/bin -if [ ! -f /opt/cni/bin/loopback ]; then - curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - -fi -if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.22.5/bin/linux/amd64/kubelet - chmod +x /opt/bin/kubelet -fi - -if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh -fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.23.0.golden b/pkg/userdata/helper/testdata/download_binaries_v1.23.0.golden deleted file mode 100644 index c93028eb3..000000000 --- a/pkg/userdata/helper/testdata/download_binaries_v1.23.0.golden +++ /dev/null @@ -1,17 +0,0 @@ -mkdir -p /opt/bin/ -mkdir -p /var/lib/calico -mkdir -p /etc/kubernetes/manifests -mkdir -p /etc/cni/net.d -mkdir -p /opt/cni/bin -if [ ! -f /opt/cni/bin/loopback ]; then - curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - -fi -if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.23.0/bin/linux/amd64/kubelet - chmod +x /opt/bin/kubelet -fi - -if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh -fi diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14-external.golden deleted file mode 100644 index 9a73edd63..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14-external.golden +++ /dev/null @@ -1,38 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14.golden deleted file mode 100644 index 4cbdf3e13..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.20.14.golden +++ /dev/null @@ -1,37 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8-external.golden deleted file mode 100644 index 9a73edd63..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8-external.golden +++ /dev/null @@ -1,38 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8.golden deleted file mode 100644 index 4cbdf3e13..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.8.golden +++ /dev/null @@ -1,37 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5-external.golden deleted file mode 100644 index 9a73edd63..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5-external.golden +++ /dev/null @@ -1,38 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5.golden deleted file mode 100644 index 4cbdf3e13..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.5.golden +++ /dev/null @@ -1,37 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0-external.golden deleted file mode 100644 index 04ba38dbf..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0-external.golden +++ /dev/null @@ -1,36 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0.golden deleted file mode 100644 index be191df0e..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.0.golden +++ /dev/null @@ -1,35 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml deleted file mode 100644 index 7e6bb2656..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-containerd-v1.20-aws.yaml +++ /dev/null @@ -1,489 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl start setup.service -- systemctl start disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml deleted file mode 100644 index e2ac287eb..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.20-aws.yaml +++ /dev/null @@ -1,473 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl start setup.service -- systemctl start disable-nm-cloud-setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml deleted file mode 100644 index 9e1fe6ad2..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-containerd-v1.20-aws.yaml +++ /dev/null @@ -1,455 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml deleted file mode 100644 index 632c44b74..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.20-aws.yaml +++ /dev/null @@ -1,439 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml deleted file mode 100644 index d10782447..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-nutanix.yaml +++ /dev/null @@ -1,447 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - iscsi-initiator-utils \ - ipvsadm - systemctl enable --now iscsid - - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=nutanix \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.20.14.yaml b/pkg/userdata/sles/testdata/version-1.20.14.yaml deleted file mode 100644 index 6c50c9958..000000000 --- a/pkg/userdata/sles/testdata/version-1.20.14.yaml +++ /dev/null @@ -1,424 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.20.14}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.21.8.yaml b/pkg/userdata/sles/testdata/version-1.21.8.yaml deleted file mode 100644 index 726859301..000000000 --- a/pkg/userdata/sles/testdata/version-1.21.8.yaml +++ /dev/null @@ -1,424 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.22.5.yaml b/pkg/userdata/sles/testdata/version-1.22.5.yaml deleted file mode 100644 index de832d974..000000000 --- a/pkg/userdata/sles/testdata/version-1.22.5.yaml +++ /dev/null @@ -1,424 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.23.0.yaml b/pkg/userdata/sles/testdata/version-1.23.0.yaml deleted file mode 100644 index 080177f7e..000000000 --- a/pkg/userdata/sles/testdata/version-1.23.0.yaml +++ /dev/null @@ -1,422 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml b/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml deleted file mode 100644 index eaea76513..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.20.14.yaml +++ /dev/null @@ -1,446 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml b/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml deleted file mode 100644 index a7a0e0649..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.21.8.yaml +++ /dev/null @@ -1,446 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml deleted file mode 100644 index be45b45dc..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.22.5.yaml +++ /dev/null @@ -1,446 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml deleted file mode 100644 index 106228754..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.23.0.yaml +++ /dev/null @@ -1,444 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl start setup.service From fce1320fd139f6ee48f762a2e8696f02aa87ed0c Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 6 May 2022 14:18:49 +0500 Subject: [PATCH 133/489] flatcar: explicitly stop masked units on first boot (#1271) Signed-off-by: Waleed Malik --- hack/lib.sh | 59 +++++++++++++++++++ hack/update-fixtures.sh | 7 +++ pkg/userdata/flatcar/provider.go | 42 +++++++++++++ .../flatcar/testdata/ignition_v1.21.10.json | 2 +- .../flatcar/testdata/ignition_v1.22.7.json | 2 +- .../flatcar/testdata/ignition_v1.23.5.json | 2 +- .../flatcar/testdata/ignition_v1.24.0.json | 2 +- 7 files changed, 112 insertions(+), 4 deletions(-) create mode 100644 hack/lib.sh diff --git a/hack/lib.sh b/hack/lib.sh new file mode 100644 index 000000000..313119c90 --- /dev/null +++ b/hack/lib.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +### Contains commonly used functions for the other scripts. + +# Required for signal propagation to work so +# the cleanup trap gets executed when a script +# receives a SIGINT +set -o monitor + +echodate() { + # do not use -Is to keep this compatible with macOS + echo "[$(date +%Y-%m-%dT%H:%M:%S%:z)]" "$@" +} + +containerize() { + local cmd="$1" + local image="${CONTAINERIZE_IMAGE:-quay.io/kubermatic/util:2.0.0}" + local gocache="${CONTAINERIZE_GOCACHE:-/tmp/.gocache}" + local gomodcache="${CONTAINERIZE_GOMODCACHE:-/tmp/.gomodcache}" + local skip="${NO_CONTAINERIZE:-}" + + # short-circuit containerize when in some cases it needs to be avoided + [ -n "$skip" ] && return + + if ! [ -f /.dockerenv ]; then + echodate "Running $cmd in a Docker container using $image..." + mkdir -p "$gocache" + mkdir -p "$gomodcache" + + exec docker run \ + -v "$PWD":/go/src/k8c.io/kubermatic \ + -v "$gocache":"$gocache" \ + -v "$gomodcache":"$gomodcache" \ + -w /go/src/k8c.io/kubermatic \ + -e "GOCACHE=$gocache" \ + -e "GOMODCACHE=$gomodcache" \ + -u "$(id -u):$(id -g)" \ + --entrypoint="$cmd" \ + --rm \ + -it \ + $image $@ + + exit $? + fi +} diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 3826851ec..6873ee5dd 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -14,6 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -euo pipefail + +cd $(dirname $0)/.. +source hack/lib.sh + +CONTAINERIZE_IMAGE=golang:1.18.1 containerize ./hack/update-fixtures.sh + go test ./... -v -update || go test ./... if [[ $? -eq 0 ]]; then echo "Successfully updated fixtures"; else "Failed to update fixtures"; fi diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index d87c5ff59..6047db7e9 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -200,12 +200,34 @@ systemd: Environment=ALL_PROXY={{ .HTTPProxy }} {{- end }} + - name: setup.service + enabled: true + contents: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + Requires=nodeip.service + After=network-online.target + After=nodeip.service + + Description=Service responsible for configuring the flatcar machine + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/setup.sh + - name: download-script.service enabled: true contents: | [Unit] Requires=network-online.target + Requires=setup.service After=network-online.target + After=setup.service [Service] Type=oneshot EnvironmentFile=-/etc/environment @@ -426,6 +448,26 @@ storage: }); {{- end }} + - path: /opt/bin/setup.sh + filesystem: root + mode: 0755 + contents: + inline: | + #!/bin/bash + set -xeuo pipefail + + # We stop these services here explicitly since masking only removes the symlinks for these services so that they can't be started. + # But that wouldn't "stop" the already running services on the first boot. + + {{- if or .FlatcarConfig.DisableUpdateEngine .FlatcarConfig.DisableAutoUpdate }} + systemctl stop update-engine.service + {{- end }} + + {{- if or .FlatcarConfig.DisableLocksmithD .FlatcarConfig.DisableAutoUpdate }} + systemctl stop locksmithd.service + {{- end }} + systemctl disable setup.service + - path: /opt/bin/download.sh filesystem: root mode: 0755 diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.21.10.json b/pkg/userdata/flatcar/testdata/ignition_v1.21.10.json index 1b2aa2838..9422cbf51 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.21.10.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.21.10.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.10%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.10%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json b/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json index f4954dc0b..86790330c 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.7%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.7%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json b/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json index c7c04fca9..f470beb95 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json index fc2fb3d19..9c669c995 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Unit]\nRequires=network-online.target\nAfter=network-online.target\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file From 3bcd78385917fcebdab5e86b35b6524eeaa3ce51 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Fri, 6 May 2022 15:51:46 +0200 Subject: [PATCH 134/489] Update containerd to 1.5 for all Kubernetes versions (#1277) * Update containerd to 1.5 Signed-off-by: Marvin Beckers * Update fixtures Signed-off-by: Marvin Beckers --- pkg/containerruntime/containerd.go | 7 ++----- pkg/containerruntime/containerruntime.go | 6 ++---- pkg/userdata/ubuntu/testdata/containerd.yaml | 2 +- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index 07280ab31..29f3e8012 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -63,13 +63,10 @@ func (eng *Containerd) ScriptFor(os types.OperatingSystem) (string, error) { args.ContainerdVersion = eng.version } - // Amazon Linux 2 does not have containerd 1.5 - if eng.version == "" && os == types.OperatingSystemAmazonLinux2 { - args.ContainerdVersion = LegacyContainerdVersion - } - switch os { case types.OperatingSystemAmazonLinux2: + // Amazon Linux 2 does not have containerd 1.5 + args.ContainerdVersion = LegacyContainerdVersion err := containerdAmzn2Template.Execute(&buf, args) return buf.String(), err case types.OperatingSystemCentOS, types.OperatingSystemRHEL, types.OperatingSystemRockyLinux: diff --git a/pkg/containerruntime/containerruntime.go b/pkg/containerruntime/containerruntime.go index f9ce1934c..f8c23ca26 100644 --- a/pkg/containerruntime/containerruntime.go +++ b/pkg/containerruntime/containerruntime.go @@ -117,13 +117,11 @@ func (cfg Config) Engine(kubeletVersion *semver.Version) Engine { moreThan124, _ := semver.NewConstraint(">= 1.24") switch { - case moreThan124.Check(kubeletVersion): + case moreThan124.Check(kubeletVersion) || cfg.Containerd != nil: + // docker support has been removed in Kubernetes 1.24 return containerd case cfg.Docker != nil: return docker - case cfg.Containerd != nil: - containerd.version = LegacyContainerdVersion - return containerd } return docker diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index 593abf213..b3ccf4b3b 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -103,7 +103,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y --allow-downgrades containerd.io=1.4* + apt-get install -y --allow-downgrades containerd.io=1.5* apt-mark hold containerd.io systemctl daemon-reload From 135da76f5cd64ca02c50042fba319f8d6e47f67b Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Fri, 13 May 2022 17:52:12 +0300 Subject: [PATCH 135/489] refactor kubevirt tests (#1284) Signed-off-by: Moath Qasim --- test/e2e/provisioning/all_e2e_test.go | 2 +- .../e2e/provisioning/testdata/machinedeployment-kubevirt.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index bc7050f42..fcadb91bc 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -281,7 +281,7 @@ func addCAToDeployment(ctx context.Context, client ctrlruntimeclient.Client, nam func TestKubevirtProvisioningE2E(t *testing.T) { t.Parallel() - kubevirtKubeconfig := os.Getenv("KUBEVIRT_E2E_TESTS_KUBECONFIG") + kubevirtKubeconfig := os.Getenv("KUBEVIRT_E2E_TESTS_KUBECONFIG_JSON") if kubevirtKubeconfig == "" { t.Fatalf("Unable to run kubevirt tests, KUBEVIRT_E2E_TESTS_KUBECONFIG must be set") diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index f22f8be4c..07bb38b40 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -34,9 +34,9 @@ spec: cpus: "1" memory: "4096M" primaryDisk: - osImage: http://10.107.208.71/<< OS_NAME >>.img + osImage: http://10.244.1.19/<< OS_NAME >>.img size: "25Gi" - storageClassName: local-path + storageClassName: longhorn dnsPolicy: "None" dnsConfig: nameservers: From 2a55801c426d3b7cb9d2d98b52a38cd7998bea14 Mon Sep 17 00:00:00 2001 From: Pratik Deoghare Date: Fri, 13 May 2022 23:08:12 +0530 Subject: [PATCH 136/489] enable DHCPv6 on RHEL (#1280) * enable DHCPv6 in RHEL * update tests * adapt for changing default interface names * remove sudo usage, we already have superpowers --- pkg/userdata/rhel/provider.go | 6 ++++++ pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml | 6 ++++++ pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml | 6 ++++++ pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml | 6 ++++++ pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml | 6 ++++++ pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml | 6 ++++++ .../rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml | 6 ++++++ pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml | 6 ++++++ pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml | 6 ++++++ pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml | 6 ++++++ pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml | 6 ++++++ pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml | 6 ++++++ 12 files changed, 72 insertions(+) diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 12861a2e0..41aa68e46 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -232,6 +232,12 @@ write_files: {{ end }} {{ .ContainerRuntimeScript | indent 4 }} {{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml index 71970eb9f..5c1466da9 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml @@ -168,6 +168,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index 2455fb24f..e6fdfe082 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -168,6 +168,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml index db8c84527..669402bb0 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml @@ -176,6 +176,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index dd5cb14ca..6a5b4afb8 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -168,6 +168,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index 55b369f14..ed0a5ebfb 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -168,6 +168,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index f304693a7..faf4f9533 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -182,6 +182,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index 4e6cff774..7d390cc9c 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -182,6 +182,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index 6152a9f4b..8c9789045 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -174,6 +174,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml index e21c199b7..89a7a7478 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml @@ -167,6 +167,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml index 6a3f85086..8dc58fd1a 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml @@ -167,6 +167,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index eba41a222..6cb508a59 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -175,6 +175,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh From d17b28516bf58d493627134e6af9a62f63aaf511 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 16 May 2022 08:49:58 +0500 Subject: [PATCH 137/489] Remove use-osm flag from run-machine-controller script (#1282) OSM is not GA yet, it's confusing for people who are trying to run machine-controller locally since it's enabled by default. We should disable it for now. Signed-off-by: Waleed Malik --- hack/run-machine-controller.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hack/run-machine-controller.sh b/hack/run-machine-controller.sh index 58b7edb29..69cc68125 100755 --- a/hack/run-machine-controller.sh +++ b/hack/run-machine-controller.sh @@ -17,9 +17,12 @@ set -e # Use a special env variable for machine-controller only +# This kubeconfig should point to the cluster where machinedeployments, machines are installed. MC_KUBECONFIG=${MC_KUBECONFIG:-$(dirname $0)/../.kubeconfig} # If you want to use the default kubeconfig `export MC_KUBECONFIG=$KUBECONFIG` +# `-use-osm` flag can be specified if https://github.com/kubermatic/operating-system-manager is used to manage user data. + make -C $(dirname $0)/.. build-machine-controller $(dirname $0)/../machine-controller \ -kubeconfig=$MC_KUBECONFIG \ @@ -29,6 +32,5 @@ $(dirname $0)/../machine-controller \ -cluster-dns=172.16.0.10 \ -enable-profiling \ -metrics-address=0.0.0.0:8080 \ - -use-osm \ -health-probe-address=0.0.0.0:8085 \ -node-container-runtime=containerd From 64e3e9aff1c2a3703fd3df82469308fa2ca442dd Mon Sep 17 00:00:00 2001 From: Pratik Deoghare Date: Mon, 16 May 2022 09:49:58 +0530 Subject: [PATCH 138/489] remove check for dualstack unsupported zones in GCP (#1283) --- pkg/cloudprovider/provider/gce/provider.go | 23 ------------------- .../provider/gce/provider_test.go | 14 ----------- 2 files changed, 37 deletions(-) diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 6c5db721f..ba3df6677 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -24,7 +24,6 @@ import ( "fmt" "net/http" "strconv" - "strings" "cloud.google.com/go/logging" monitoring "cloud.google.com/go/monitoring/apiv3" @@ -49,7 +48,6 @@ const ( errOperatingSystem = "Invalid or not supported operating system specified %q: %v" errConnect = "Failed to connect: %v" errInvalidServiceAccount = "Service account is missing" - errIPv6UnsupportedZone = "IPv6 is not supported in zone: %s" errInvalidZone = "Zone is missing" errInvalidMachineType = "Machine type is missing" errInvalidDiskSize = "Disk size must be a positive number" @@ -122,9 +120,6 @@ func (p *Provider) Validate(spec clusterv1alpha1.MachineSpec) error { case util.IPv6: return newError(common.InvalidConfigurationMachineError, util.ErrIPv6OnlyUnsupported) case util.DualStack: - if !isIPv6Supported(cfg.zone) { - return newError(common.InvalidConfigurationMachineError, errIPv6UnsupportedZone, cfg.zone) - } default: return newError(common.InvalidConfigurationMachineError, util.ErrUnknownNetworkFamily, cfg.providerConfig.Network.GetIPFamily()) } @@ -145,24 +140,6 @@ func (p *Provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func isIPv6Supported(zone string) bool { - supportedRegions := []string{ - "asia-east1", - "asia-south1", - "europe-west2", - "us-west2", - } - - for _, region := range supportedRegions { - // this is fine since zones are constructed from region + zone suffix - if strings.HasPrefix(zone, region) { - return true - } - } - - return false -} - // Get retrieves a node instance that is associated with the given machine. func (p *Provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { return p.get(machine) diff --git a/pkg/cloudprovider/provider/gce/provider_test.go b/pkg/cloudprovider/provider/gce/provider_test.go index 036918699..4f5fc405b 100644 --- a/pkg/cloudprovider/provider/gce/provider_test.go +++ b/pkg/cloudprovider/provider/gce/provider_test.go @@ -157,20 +157,6 @@ func TestValidate(t *testing.T) { }, false, }, - { - "with unsupported zone", - v1alpha1.MachineSpec{ - ProviderSpec: v1alpha1.ProviderSpec{ - Value: &runtime.RawExtension{ - Raw: rawBytes(testMap(testProviderSpec()). - with("network.ipFamily", "IPv4+IPv6"). - with("cloudProviderSpec.zone", "europe-west3-a"), - ), - }, - }, - }, - true, - }, } for _, test := range tests { From 6b5d02b0fe3e11f5f0037ed8812ef0a7f9487825 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 16 May 2022 16:40:11 +0500 Subject: [PATCH 139/489] Fix machine cleanup on azure (#1286) Signed-off-by: Waleed Malik --- go.mod | 2 +- go.sum | 4 ++-- pkg/cloudprovider/provider/azure/provider.go | 12 +----------- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 297672d94..73404312d 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.18 require ( cloud.google.com/go/logging v1.1.2 cloud.google.com/go/monitoring v1.4.0 - github.com/Azure/azure-sdk-for-go v62.0.0+incompatible + github.com/Azure/azure-sdk-for-go v64.1.0+incompatible github.com/Azure/go-autorest/autorest/azure/auth v0.5.5 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/BurntSushi/toml v0.3.1 diff --git a/go.sum b/go.sum index 7a47f6e2f..9ef4daf42 100644 --- a/go.sum +++ b/go.sum @@ -56,8 +56,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go v62.0.0+incompatible h1:8N2k27SYtc12qj5nTsuFMFJPZn5CGmgMWqTy4y9I7Jw= -github.com/Azure/azure-sdk-for-go v62.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v64.1.0+incompatible h1:FpsZmWR9FfEr9hP6K9S7RP0EkSFgGd6P1F2scHtbhnU= +github.com/Azure/azure-sdk-for-go v64.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 949d65ef9..c651c0b3d 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -730,19 +730,9 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider return false, fmt.Errorf("failed to parse MachineSpec: %v", err) } - _, err = p.get(machine) - // If a defunct VM got created, the `Get` call returns an error - But not because the request - // failed but because the VM has an invalid config hence always delete except on err == cloudprovidererrors.ErrInstanceNotFound - if err != nil { - if err == cloudprovidererrors.ErrInstanceNotFound { - return util.RemoveFinalizerOnInstanceNotFound(finalizerVM, machine, data) - } - return false, err - } - klog.Infof("deleting VM %q", machine.Name) if err = deleteVMsByMachineUID(context.TODO(), config, machine.UID); err != nil { - return false, fmt.Errorf("failed to delete instance for machine %q: %v", machine.Name, err) + return false, fmt.Errorf("failed to delete instance for machine %q: %v", machine.Name, err) } if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { From 6781c1a2833f2ad540e03589270f92499551d1b0 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Mon, 16 May 2022 16:06:14 +0200 Subject: [PATCH 140/489] Only attempt resource deletion in azure provider if finalizer is set (#1287) Signed-off-by: Marvin Beckers --- .../provider/azure/create_delete_resources.go | 3 +- pkg/cloudprovider/provider/azure/provider.go | 78 ++++++++++--------- 2 files changed, 45 insertions(+), 36 deletions(-) diff --git a/pkg/cloudprovider/provider/azure/create_delete_resources.go b/pkg/cloudprovider/provider/azure/create_delete_resources.go index 45248e1b1..4bf719a17 100644 --- a/pkg/cloudprovider/provider/azure/create_delete_resources.go +++ b/pkg/cloudprovider/provider/azure/create_delete_resources.go @@ -115,7 +115,8 @@ func deleteVMsByMachineUID(ctx context.Context, c *config, machineUID types.UID) return err } - list, err := vmClient.ListAll(ctx, "", "") + list, err := vmClient.List(ctx, c.ResourceGroup, "") + if err != nil { return err } diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index c651c0b3d..5cde5322c 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -730,45 +730,53 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider return false, fmt.Errorf("failed to parse MachineSpec: %v", err) } - klog.Infof("deleting VM %q", machine.Name) - if err = deleteVMsByMachineUID(context.TODO(), config, machine.UID); err != nil { - return false, fmt.Errorf("failed to delete instance for machine %q: %v", machine.Name, err) - } + if kuberneteshelper.HasFinalizer(machine, finalizerVM) { + klog.Infof("deleting VM %q", machine.Name) + if err = deleteVMsByMachineUID(context.TODO(), config, machine.UID); err != nil { + return false, fmt.Errorf("failed to delete instance for machine %q: %v", machine.Name, err) + } - if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { - updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerVM) - }); err != nil { - return false, err + if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { + updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerVM) + }); err != nil { + return false, err + } } - klog.Infof("deleting disks of VM %q", machine.Name) - if err := deleteDisksByMachineUID(context.TODO(), config, machine.UID); err != nil { - return false, fmt.Errorf("failed to remove disks of machine %q: %v", machine.Name, err) - } - if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { - updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerDisks) - }); err != nil { - return false, err + if kuberneteshelper.HasFinalizer(machine, finalizerDisks) { + klog.Infof("deleting disks of VM %q", machine.Name) + if err := deleteDisksByMachineUID(context.TODO(), config, machine.UID); err != nil { + return false, fmt.Errorf("failed to remove disks of machine %q: %v", machine.Name, err) + } + if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { + updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerDisks) + }); err != nil { + return false, err + } } - klog.Infof("deleting network interfaces of VM %q", machine.Name) - if err := deleteInterfacesByMachineUID(context.TODO(), config, machine.UID); err != nil { - return false, fmt.Errorf("failed to remove network interfaces of machine %q: %v", machine.Name, err) - } - if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { - updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerNIC) - }); err != nil { - return false, err + if kuberneteshelper.HasFinalizer(machine, finalizerNIC) { + klog.Infof("deleting network interfaces of VM %q", machine.Name) + if err := deleteInterfacesByMachineUID(context.TODO(), config, machine.UID); err != nil { + return false, fmt.Errorf("failed to remove network interfaces of machine %q: %v", machine.Name, err) + } + if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { + updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerNIC) + }); err != nil { + return false, err + } } - klog.Infof("deleting public IP addresses of VM %q", machine.Name) - if err := deleteIPAddressesByMachineUID(context.TODO(), config, machine.UID); err != nil { - return false, fmt.Errorf("failed to remove public IP addresses of machine %q: %v", machine.Name, err) - } - if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { - updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerPublicIP) - }); err != nil { - return false, err + if kuberneteshelper.HasFinalizer(machine, finalizerPublicIP) { + klog.Infof("deleting public IP addresses of VM %q", machine.Name) + if err := deleteIPAddressesByMachineUID(context.TODO(), config, machine.UID); err != nil { + return false, fmt.Errorf("failed to remove public IP addresses of machine %q: %v", machine.Name, err) + } + if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { + updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerPublicIP) + }); err != nil { + return false, err + } } return true, nil @@ -780,7 +788,7 @@ func getVMByUID(ctx context.Context, c *config, uid types.UID) (*compute.Virtual return nil, err } - list, err := vmClient.ListAll(ctx, "", "") + list, err := vmClient.List(ctx, c.ResourceGroup, "") if err != nil { return nil, err } @@ -1019,9 +1027,9 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return fmt.Errorf("failed to (create) vm client: %v", err.Error()) } - _, err = vmClient.ListAll(context.TODO(), "", "") + _, err = vmClient.List(context.TODO(), c.ResourceGroup, "") if err != nil { - return fmt.Errorf("failed to list all: %v", err.Error()) + return fmt.Errorf("failed to list virtual machines: %v", err.Error()) } if _, err := getVirtualNetwork(context.TODO(), c); err != nil { From 8c6b61cc7c9a695914835e1ef90ca89abdabace0 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Mon, 16 May 2022 17:58:26 +0200 Subject: [PATCH 141/489] update to controller-runtime 0.12, k8s 1.24.0 (#1289) --- go.mod | 55 +++++++++--------- go.sum | 175 +++++++++++++++++++++++++++++++++++++++------------------ 2 files changed, 146 insertions(+), 84 deletions(-) diff --git a/go.mod b/go.mod index 73404312d..ad668bc56 100644 --- a/go.mod +++ b/go.mod @@ -30,12 +30,12 @@ require ( github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 - github.com/prometheus/client_golang v1.11.0 + github.com/prometheus/client_golang v1.12.1 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7 github.com/sethvargo/go-password v0.2.0 github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda github.com/vmware/govmomi v0.23.1 - golang.org/x/crypto v0.0.0-20211202192323-5770296d904e + golang.org/x/crypto v0.0.0-20220214200702-86341886e292 golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a gomodules.xyz/jsonpatch/v2 v2.2.0 google.golang.org/api v0.74.0 @@ -43,16 +43,16 @@ require ( gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b k8c.io/operating-system-manager v0.4.0 - k8s.io/api v0.23.6 - k8s.io/apiextensions-apiserver v0.23.6 - k8s.io/apimachinery v0.23.6 + k8s.io/api v0.24.0 + k8s.io/apiextensions-apiserver v0.24.0 + k8s.io/apimachinery v0.24.0 k8s.io/client-go v12.0.0+incompatible k8s.io/klog v1.0.0 - k8s.io/kubelet v0.23.6 - k8s.io/utils v0.0.0-20211116205334-6203023598ed + k8s.io/kubelet v0.24.0 + k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 kubevirt.io/api v0.48.1 - kubevirt.io/containerized-data-importer-api v1.41.0 - sigs.k8s.io/controller-runtime v0.11.2 + kubevirt.io/containerized-data-importer-api v1.49.0 + sigs.k8s.io/controller-runtime v0.12.0 sigs.k8s.io/yaml v1.3.0 ) @@ -74,30 +74,29 @@ require ( github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd // indirect github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/ignition v0.35.0 // indirect github.com/dimchansky/utfbom v1.1.0 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect - github.com/emicklei/go-restful v2.11.2+incompatible // indirect + github.com/emicklei/go-restful v2.15.0+incompatible // indirect github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect - github.com/go-logr/logr v1.2.0 // indirect + github.com/go-logr/logr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.19.5 // indirect - github.com/go-openapi/spec v0.19.15 // indirect - github.com/go-openapi/swag v0.19.15 // indirect + github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/swag v0.21.1 // indirect github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.7 // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/googleapis/gax-go/v2 v2.2.0 // indirect - github.com/googleapis/gnostic v0.5.5 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.12 // indirect @@ -112,11 +111,12 @@ require ( github.com/mitchellh/reflectwalk v1.0.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect - github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca // indirect + github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.28.0 // indirect - github.com/prometheus/procfs v0.6.0 // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/smartystreets/assertions v1.2.0 // indirect github.com/spf13/cast v1.3.1 // indirect @@ -129,7 +129,7 @@ require ( golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect + golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb // indirect google.golang.org/protobuf v1.27.1 // indirect @@ -138,18 +138,17 @@ require ( gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/component-base v0.23.6 // indirect - k8s.io/klog/v2 v2.30.0 // indirect - k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect - kubevirt.io/controller-lifecycle-operator-sdk v0.2.1 // indirect - sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect + k8s.io/component-base v0.24.0 // indirect + k8s.io/klog/v2 v2.60.1 // indirect + k8s.io/kube-openapi v0.0.0-20220413171646-5e7f5fdc6da6 // indirect + kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect + sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect ) replace ( github.com/packethost/packngo => github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888 - k8s.io/client-go => k8s.io/client-go v0.23.6 - k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.23.6 - k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd + k8s.io/client-go => k8s.io/client-go v0.24.0 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.24.0 ) diff --git a/go.sum b/go.sum index 9ef4daf42..b59717ee2 100644 --- a/go.sum +++ b/go.sum @@ -59,7 +59,6 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 github.com/Azure/azure-sdk-for-go v64.1.0+incompatible h1:FpsZmWR9FfEr9hP6K9S7RP0EkSFgGd6P1F2scHtbhnU= github.com/Azure/azure-sdk-for-go v64.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= @@ -103,8 +102,10 @@ github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXn github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -153,8 +154,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/briandowns/spinner v1.8.0/go.mod h1://Zf9tMcxfRUA36V23M6YGEAv+kECGfvpnLTnb8n4XQ= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= @@ -162,8 +163,9 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -206,9 +208,11 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -243,8 +247,8 @@ github.com/embik/nutanix-client-go v0.0.0-20220106131900-50b8f27e5f60 h1:0FVKOkp github.com/embik/nutanix-client-go v0.0.0-20220106131900-50b8f27e5f60/go.mod h1:gkKNSxfEt3QtYG3S/wKiN8OmrJ4fpU7JbTlbnrMDOL8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.11.2+incompatible h1:Z4Z0K2AuOw+QtgwkkJnwpT165MBr12qS8rnBwjP/Pzs= -github.com/emicklei/go-restful v2.11.2+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.15.0+incompatible h1:8KpYO/Xl/ZudZs5RNOEhWMBY4hmzlZhhRd9cu+jrZP4= +github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -277,6 +281,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -297,31 +302,35 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= -github.com/go-openapi/jsonreference v0.19.5 h1:1WJP/wi4OjB4iV8KVbH73rQaoialJrqv8gitZLxGLtM= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= -github.com/go-openapi/spec v0.19.15 h1:uxh8miNJEfMm8l8ekpY7i39LcORm1xSRtoipEGl1JPk= -github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.12/go.mod h1:eFdyEBkTdoAf/9RXBvj4cr1nH7GD8Kzo5HTt47gr72M= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= -github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU= +github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= @@ -333,6 +342,7 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= @@ -372,6 +382,7 @@ github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -395,8 +406,10 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/cel-go v0.9.0/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= +github.com/google/cel-go v0.10.1/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA= +github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -413,6 +426,7 @@ github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= @@ -432,6 +446,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -447,10 +462,10 @@ github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pf github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gax-go/v2 v2.2.0 h1:s7jOdKSaksJVOxE0Y/S32otcfiP+UQ0cL8/GTKaONwE= github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= -github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gophercloud/gophercloud v0.24.0 h1:jDsIMGJ1KZpAjYfQgGI2coNQj5Q83oPzuiGJRFWgMzw= github.com/gophercloud/gophercloud v0.24.0/go.mod h1:Q8fZtyi5zZxPS/j9aj3sSxtvj41AdQMDwyo1myduD5c= @@ -459,6 +474,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -527,6 +543,7 @@ github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUB github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -572,7 +589,7 @@ github.com/linode/linodego v0.24.0/go.mod h1:GSBKPpjoQfxEfryoCRcgkuUOCuVtGHWhzI8 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= @@ -615,10 +632,11 @@ github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY7 github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= -github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= @@ -626,6 +644,7 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -640,6 +659,7 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -654,7 +674,9 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -663,14 +685,17 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ= -github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca h1:F1MEnOMwSrTA0YAkO0he9ip9w0JhYzI/iCB2mXmaSPg= github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca/go.mod h1:GDjWl0tX6FNIj82vIxeudWeSx2Ff6nDZ8uJn0ohUFvo= +github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= +github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -691,7 +716,6 @@ github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= @@ -702,6 +726,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -712,8 +737,9 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -729,8 +755,8 @@ github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.28.0 h1:vGVfV9KrDTvWt5boZO0I19g2E3CsWfpPPKZM9dt3mEw= -github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -739,8 +765,9 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= @@ -754,6 +781,7 @@ github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtIS github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= @@ -795,9 +823,8 @@ github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tL github.com/spf13/cobra v1.0.1-0.20200713175500-884edc58ad08/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= +github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -805,7 +832,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stormcat24/protodep v0.0.0-20200505140716-b02c9ba62816/go.mod h1:mBd5PI4uI6NkqJpCyiWiYzWyTFs4QRDss/JTMC2b4kc= @@ -815,6 +841,7 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -849,6 +876,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -857,9 +885,12 @@ go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= +go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q= go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= @@ -929,9 +960,10 @@ golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211202192323-5770296d904e h1:MUP6MR3rJ7Gk9LEia0LP2ytiH6MuCfs7qYz+47jGdD8= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -969,6 +1001,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1016,10 +1051,13 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -1037,7 +1075,6 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= @@ -1058,6 +1095,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1110,7 +1148,6 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1118,6 +1155,7 @@ golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1126,6 +1164,7 @@ golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1139,18 +1178,20 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 h1:eJv7u3ksNXoLbGSKuv2s/SIO4tJVxc/A+MTpzxDgz/Q= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1167,12 +1208,13 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= -golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= +golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180810170437-e96c4e24768d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1234,6 +1276,7 @@ golang.org/x/tools v0.0.0-20201105220310-78b158585360/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= @@ -1243,6 +1286,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1277,7 +1322,6 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= @@ -1371,6 +1415,7 @@ google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= @@ -1492,56 +1537,72 @@ k8c.io/operating-system-manager v0.4.0/go.mod h1:pJImhsLb5GJdZunZ47r5Db0ydBwhWxh k8s.io/api v0.0.0-20190725062911-6607c48751ae/go.mod h1:1O0xzX/RAtnm7l+5VEUxZ1ysO2ghatfq/OZED4zM9kA= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= -k8s.io/api v0.23.6 h1:yOK34wbYECH4RsJbQ9sfkFK3O7f/DUHRlzFehkqZyVw= -k8s.io/api v0.23.6/go.mod h1:1kFaYxGCFHYp3qd6a85DAj/yW8aVD6XLZMqJclkoi9g= +k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= +k8s.io/api v0.24.0 h1:J0hann2hfxWr1hinZIDefw7Q96wmCBx6SSB8IY0MdDg= +k8s.io/api v0.24.0/go.mod h1:5Jl90IUrJHUJYEMANRURMiVvJ0g7Ax7r3R1bqO8zx8I= k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= k8s.io/apiextensions-apiserver v0.20.2/go.mod h1:F6TXp389Xntt+LUq3vw6HFOLttPa0V8821ogLGwb6Zs= -k8s.io/apiextensions-apiserver v0.23.6 h1:v58cQ6Z0/GK1IXYr+oW0fnYl52o9LTY0WgoWvI8uv5Q= -k8s.io/apiextensions-apiserver v0.23.6/go.mod h1:YVh17Mphv183THQJA5spNFp9XfoidFyL3WoDgZxQIZU= +k8s.io/apiextensions-apiserver v0.24.0 h1:JfgFqbA8gKJ/uDT++feAqk9jBIwNnL9YGdQvaI9DLtY= +k8s.io/apiextensions-apiserver v0.24.0/go.mod h1:iuVe4aEpe6827lvO6yWQVxiPSpPoSKVjkq+MIdg84cM= k8s.io/apimachinery v0.0.0-20190719140911-bfcf53abc9f8/go.mod h1:sBJWIJZfxLhp7mRsRyuAE/NfKTr3kXGR1iaqg8O0gJo= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.23.6 h1:RH1UweWJkWNTlFx0D8uxOpaU1tjIOvVVWV/bu5b3/NQ= -k8s.io/apimachinery v0.23.6/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= +k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= +k8s.io/apimachinery v0.24.0 h1:ydFCyC/DjCvFCHK5OPMKBlxayQytB8pxy8YQInd5UyQ= +k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.2/go.mod h1:2nKd93WyMhZx4Hp3RfgH2K5PhwyTrprrkWYnI7id7jA= -k8s.io/apiserver v0.23.6/go.mod h1:5PU32F82tfErXPmf7FXhd/UcuLfh97tGepjKUgJ2atg= -k8s.io/client-go v0.23.6 h1:7h4SctDVQAQbkHQnR4Kzi7EyUyvla5G1pFWf4+Od7hQ= -k8s.io/client-go v0.23.6/go.mod h1:Umt5icFOMLV/+qbtZ3PR0D+JA6lvvb3syzodv4irpK4= +k8s.io/apiserver v0.24.0/go.mod h1:WFx2yiOMawnogNToVvUYT9nn1jaIkMKj41ZYCVycsBA= +k8s.io/client-go v0.24.0 h1:lbE4aB1gTHvYFSwm6eD3OF14NhFDKCejlnsGYlSJe5U= +k8s.io/client-go v0.24.0/go.mod h1:VFPQET+cAFpYxh6Bq6f4xyMY80G6jKKktU6G0m00VDw= k8s.io/code-generator v0.0.0-20190717022600-77f3a1fe56bb/go.mod h1:cDx5jQmWH25Ff74daM7NVYty9JWw9dvIS9zT9eIubCY= k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.20.2/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= -k8s.io/code-generator v0.23.6/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= +k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= +k8s.io/code-generator v0.24.0/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= -k8s.io/component-base v0.23.6 h1:8dhVZ4VrRcNdV2EGjl8tj8YOHwX6ysgCGMJ2Oyy0NW8= -k8s.io/component-base v0.23.6/go.mod h1:FGMPeMrjYu0UZBSAFcfloVDplj9IvU+uRMTOdE23Fj0= +k8s.io/component-base v0.24.0 h1:h5jieHZQoHrY/lHG+HyrSbJeyfuitheBvqvKwKHVC0g= +k8s.io/component-base v0.24.0/go.mod h1:Dgazgon0i7KYUsS8krG8muGiMVtUZxG037l1MKyXgrA= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.30.0 h1:bUO6drIvCIsvZ/XFgfxoGFQU/a4Qkh0iAlvUR7vlHJw= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd h1:sOHNzJIkytDF6qadMNKhhDRpc6ODik8lVC6nOur7B2c= +k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc= +k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20190709113604-33be087ad058/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4= k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= -k8s.io/kubelet v0.23.6 h1:tuscMqYCt9cxWursmTU9OJ2tPLv66Ji+AGbuV1Z/lug= -k8s.io/kubelet v0.23.6/go.mod h1:ROttmKIUkB9in4NyX/SfnAoXGfW/Dju3VCGFP34F5ac= +k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= +k8s.io/kube-openapi v0.0.0-20220413171646-5e7f5fdc6da6 h1:nBQrWPlrNIiw0BsX6a6MKr1itkm0ZS0Nl97kNLitFfI= +k8s.io/kube-openapi v0.0.0-20220413171646-5e7f5fdc6da6/go.mod h1:daOouuuwd9JXpv1L7Y34iV3yf6nxzipkKMWWlqlvK9M= +k8s.io/kubelet v0.24.0 h1:fH+D6mSr4DGIeHp/O2+mCEJhkVq3Gpgv9BVOHI+GrWY= +k8s.io/kubelet v0.24.0/go.mod h1:p3BBacmHTCMpUf+nluhlyzuGHmONKAspqCvpu9oPAyA= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20211116205334-6203023598ed h1:ck1fRPWPJWsMd8ZRFsWc6mh/zHp5fZ/shhbrgPUxDAE= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= kubevirt.io/api v0.48.1 h1:C5i9h8ea7Xy3fJMoKEuzjRP74GnVMF7u2mQV8FGf2XE= kubevirt.io/api v0.48.1/go.mod h1:RoYMmFt76vWvFtw/FSiL0YUHZ2Ao6UfXlgpZAQnRswo= -kubevirt.io/containerized-data-importer-api v1.41.0 h1:VdEwYP36N+4asMnTBSadVH4SF7OVPvvraEQMtOd7Vlk= kubevirt.io/containerized-data-importer-api v1.41.0/go.mod h1:0xadDFtaMd8iy+/oD2+dYoPxACZ/YizKqay5QIrQ6cw= -kubevirt.io/controller-lifecycle-operator-sdk v0.2.1 h1:I1b14fnhwrVvQLmgksMo9vgje42hmH4QN5kqyYDqbMA= +kubevirt.io/containerized-data-importer-api v1.49.0 h1:V3eUSKL/kRoJSpQ3FA12vj1jod/QYVXlQEOsv8Cg7mc= +kubevirt.io/containerized-data-importer-api v1.49.0/go.mod h1:yjD8pGZVMCeqcN46JPUQdZ2JwRVoRCOXrTVyNuFvrLo= kubevirt.io/controller-lifecycle-operator-sdk v0.2.1/go.mod h1:ZJhLceiY2Gl5CXFGSp5eMGt/sksOiJP0289nAZFCQf0= +kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= +kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= @@ -1553,11 +1614,13 @@ rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= -sigs.k8s.io/controller-runtime v0.11.2 h1:H5GTxQl0Mc9UjRJhORusqfJCIjBO8UtUxGggCwL1rLA= -sigs.k8s.io/controller-runtime v0.11.2/go.mod h1:P6QCzrEjLaZGqHsfd+os7JQ+WFZhvB8MRFsn4dWF7O4= +sigs.k8s.io/controller-runtime v0.12.0 h1:gA4zphrmHFc7ihmY/+GyyE0BxKD+OYdb5+DjD2azFAQ= +sigs.k8s.io/controller-runtime v0.12.0/go.mod h1:BKhxlA4l7FPK4AQcsuL4X6vZeWnKDXez/vp1Y8dxTU0= sigs.k8s.io/controller-tools v0.5.0/go.mod h1:JTsstrMpxs+9BUj6eGuAaEb6SDSPTeVtUyp0jmnAM/I= -sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 h1:fD1pz4yfdADVNfFmcP2aBEtudwUQ1AlLnRBALr33v3s= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y= +sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= From 73a62b5a8acd0c073b006f908fbce93935223e71 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 17 May 2022 13:46:32 +0500 Subject: [PATCH 142/489] docs(anexia): add info for templates (#1288) * docs(anexia): add info for templates Signed-off-by: Waleed Malik * Update anexia.md Signed-off-by: Waleed Malik --- docs/anexia.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/anexia.md b/docs/anexia.md index 4d186831d..5a224f532 100644 --- a/docs/anexia.md +++ b/docs/anexia.md @@ -6,4 +6,14 @@ This provider implementation is currently in **alpha** state. Only flatcar linux is currently supported and you explicitly have to set the provisioning mechanism to cloud-init by setting `machine.spec.providerSpec.value.operatingSystemSpec.provisioningUtility` to "cloud-init". -An example machine deployment can be found here: [examples/anexia-machinedeployment.yaml](../examples/anexia-machinedeployment.yaml) \ No newline at end of file +An example machine deployment can be found here: [examples/anexia-machinedeployment.yaml](../examples/anexia-machinedeployment.yaml) + +## Templates + +To retrieve all available templates against a given location: + +``` +https://engine.anexia-it.com/api/vsphere/v1/provisioning/templates.json//templates?page=1&limit=50&api_key= +``` + +Templates are rotated pretty often, to include updates and latest security patches. Outdated versions of templates are not retained as a result and they get removed after some time. From 82ccf48bb02e8963679b3a5a7ec68fd5aad59ff1 Mon Sep 17 00:00:00 2001 From: Mattia Lavacca Date: Tue, 17 May 2022 13:35:37 +0200 Subject: [PATCH 143/489] rh_subscriprion bootstrap script (#1265) * rh_subscriprion cloud-init This Commit adds the rh_subscription cloud-init module to the yum version of the bootstrap script. Signed-off-by: Mattia Lavacca * /etc/hostname patched also for rhel in Openstack /etc/hostname is patched so that is contains the machineName not only for Centos under Openstack, but also for RHEL. Signed-off-by: Mattia Lavacca * RHEL updated to 8.5 in Azure Rhel has been updated to the 8.5 version. Beside this change, the part of removing and adding the RHUI repo from yum isn't needed anymore, as seems the problem has been solved by just updating the RHEL version, thus it has been removed. Signed-off-by: Mattia Lavacca * yum update change in Azure Signed-off-by: Mattia Lavacca * update fixtures Signed-off-by: Mattia Lavacca --- pkg/cloudprovider/provider/azure/provider.go | 8 ++--- pkg/controller/machine/bootstrap.go | 30 +++++++++++++++++-- pkg/userdata/rhel/provider.go | 3 -- .../rhel/testdata/kubelet-v1.21-aws.yaml | 1 - .../rhel/testdata/kubelet-v1.22-aws.yaml | 1 - .../rhel/testdata/kubelet-v1.22-nutanix.yaml | 1 - .../testdata/kubelet-v1.23-aws-external.yaml | 1 - .../rhel/testdata/kubelet-v1.23-aws.yaml | 1 - .../kubelet-v1.23-vsphere-mirrors.yaml | 1 - .../testdata/kubelet-v1.23-vsphere-proxy.yaml | 1 - .../rhel/testdata/kubelet-v1.23-vsphere.yaml | 1 - .../testdata/kubelet-v1.24-aws-external.yaml | 1 - .../rhel/testdata/kubelet-v1.24-aws.yaml | 1 - .../rhel/testdata/pod-cidr-azure-rhel.yaml | 3 -- 14 files changed, 31 insertions(+), 23 deletions(-) diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 5cde5322c..c002224b7 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -142,8 +142,8 @@ var imageReferences = map[providerconfigtypes.OperatingSystem]compute.ImageRefer providerconfigtypes.OperatingSystemRHEL: { Publisher: to.StringPtr("RedHat"), Offer: to.StringPtr("rhel-byos"), - Sku: to.StringPtr("rhel-lvm83"), - Version: to.StringPtr("8.3.20201109"), + Sku: to.StringPtr("rhel-lvm85"), + Version: to.StringPtr("8.5.20220316"), }, providerconfigtypes.OperatingSystemFlatcar: { Publisher: to.StringPtr("kinvolk"), @@ -166,7 +166,7 @@ var osPlans = map[providerconfigtypes.OperatingSystem]*compute.Plan{ Product: pointer.StringPtr("flatcar-container-linux"), }, providerconfigtypes.OperatingSystemRHEL: { - Name: pointer.StringPtr("rhel-lvm83"), + Name: pointer.StringPtr("rhel-lvm85"), Publisher: pointer.StringPtr("redhat"), Product: pointer.StringPtr("rhel-byos"), }, @@ -221,7 +221,7 @@ func getOSImageReference(c *config, os providerconfigtypes.OperatingSystem) (*co return &ref, nil } -// New returns a digitalocean provider +// New returns a new azure provider func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } diff --git a/pkg/controller/machine/bootstrap.go b/pkg/controller/machine/bootstrap.go index d68705583..92916f4a6 100644 --- a/pkg/controller/machine/bootstrap.go +++ b/pkg/controller/machine/bootstrap.go @@ -31,6 +31,7 @@ import ( providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" "github.com/kubermatic/machine-controller/pkg/userdata/convert" "github.com/kubermatic/machine-controller/pkg/userdata/helper" + "github.com/kubermatic/machine-controller/pkg/userdata/rhel" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -123,6 +124,7 @@ func getOSMBootstrapUserDataForCloudInit(req plugin.UserDataRequest, pconfig *pr MachineName string EnterpriseLinux bool ProviderSpec *providerconfigtypes.Config + RHELConfig rhel.Config }{ Token: token, SecretName: secretName, @@ -132,8 +134,9 @@ func getOSMBootstrapUserDataForCloudInit(req plugin.UserDataRequest, pconfig *pr } var ( - bsScript *template.Template - err error + rhelConfig *rhel.Config + bsScript *template.Template + err error ) switch pconfig.OperatingSystem { @@ -159,6 +162,10 @@ func getOSMBootstrapUserDataForCloudInit(req plugin.UserDataRequest, pconfig *pr return "", fmt.Errorf("failed to parse bootstrapZypperBinContentTemplate template: %v", err) } case providerconfigtypes.OperatingSystemRHEL: + rhelConfig, err = rhel.LoadConfig(pconfig.OperatingSystemSpec) + if err != nil { + return "", fmt.Errorf("failed to parse OperatingSystemSpec: %w", err) + } bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapYumBinContentTemplate) if err != nil { return "", fmt.Errorf("failed to parse bootstrapYumBinContentTemplate template: %v", err) @@ -182,12 +189,14 @@ func getOSMBootstrapUserDataForCloudInit(req plugin.UserDataRequest, pconfig *pr plugin.UserDataRequest ProviderSpec *providerconfigtypes.Config BootstrapKubeconfig string + RHELConfig *rhel.Config }{ Script: base64.StdEncoding.EncodeToString(script.Bytes()), Service: base64.StdEncoding.EncodeToString([]byte(bootstrapServiceContentTemplate)), UserDataRequest: req, ProviderSpec: pconfig, BootstrapKubeconfig: base64.StdEncoding.EncodeToString([]byte(bootstrapKfg)), + RHELConfig: rhelConfig, }) if err != nil { return "", fmt.Errorf("failed to execute cloudInitTemplate template: %v", err) @@ -238,7 +247,9 @@ fi {{- if .EnterpriseLinux }} yum install epel-release -y {{- end }} + yum install -y curl jq + curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-config"]' -r| base64 -d > /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg cloud-init clean cloud-init --file /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg init @@ -305,7 +316,7 @@ write_files: encoding: b64 content: | {{ .BootstrapKubeconfig }} -{{- if and (eq .ProviderSpec.CloudProvider "openstack") (eq .ProviderSpec.OperatingSystem "centos") }} +{{- if and (eq .ProviderSpec.CloudProvider "openstack") (or (eq .ProviderSpec.OperatingSystem "centos") (eq .ProviderSpec.OperatingSystem "rhel")) }} {{- /* The normal way of setting it via cloud-init is broken, see */}} {{- /* https://bugs.launchpad.net/cloud-init/+bug/1662542 */}} - path: /etc/hostname @@ -328,6 +339,19 @@ write_files: runcmd: - systemctl restart bootstrap.service - systemctl daemon-reload +{{- if .RHELConfig }} +rh_subscription: +{{- if .RHELConfig.RHELUseSatelliteServer }} + org: "{{.RHELConfig.RHELOrganizationName}}" + activation-key: "{{.RHELConfig.RHELActivationKey}}" + server-hostname: {{ .RHELConfig.RHELSatelliteServer }} + rhsm-baseurl: https://{{ .RHELConfig.RHELSatelliteServer }}/pulp/repos +{{- else }} + username: "{{.RHELConfig.RHELSubscriptionManagerUser}}" + password: "{{.RHELConfig.RHELSubscriptionManagerPassword}}" + auto-attach: {{.RHELConfig.AttachSubscription}} +{{- end }} +{{- end }} ` ignitionBootstrapBinContentTemplate = `#!/bin/bash diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 41aa68e46..1d3821376 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -204,9 +204,6 @@ write_files: {{- /* https://bugs.launchpad.net/cloud-init/+bug/1662542 */}} hostnamectl set-hostname {{ .MachineSpec.Name }} {{ end }} - {{ if eq .CloudProviderName "azure" }} - yum update -y --disablerepo='*' --enablerepo='*microsoft*' - {{ end }} yum install -y \ device-mapper-persistent-data \ lvm2 \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml index 5c1466da9..db6c4a381 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml @@ -67,7 +67,6 @@ write_files: systemctl restart systemd-modules-load.service sysctl --system - yum install -y \ device-mapper-persistent-data \ lvm2 \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index e6fdfe082..7f25ec49d 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -67,7 +67,6 @@ write_files: systemctl restart systemd-modules-load.service sysctl --system - yum install -y \ device-mapper-persistent-data \ lvm2 \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml index 669402bb0..6ba2ce6e5 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml @@ -72,7 +72,6 @@ write_files: hostnamectl set-hostname node1 - yum install -y \ device-mapper-persistent-data \ lvm2 \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index 6a5b4afb8..2ab3000dd 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -67,7 +67,6 @@ write_files: systemctl restart systemd-modules-load.service sysctl --system - yum install -y \ device-mapper-persistent-data \ lvm2 \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index ed0a5ebfb..c43fa2702 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -67,7 +67,6 @@ write_files: systemctl restart systemd-modules-load.service sysctl --system - yum install -y \ device-mapper-persistent-data \ lvm2 \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index faf4f9533..01e0d1df5 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -80,7 +80,6 @@ write_files: hostnamectl set-hostname node1 - yum install -y \ device-mapper-persistent-data \ lvm2 \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index 7d390cc9c..38e98cb96 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -80,7 +80,6 @@ write_files: hostnamectl set-hostname node1 - yum install -y \ device-mapper-persistent-data \ lvm2 \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index 8c9789045..693e29a41 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -72,7 +72,6 @@ write_files: hostnamectl set-hostname node1 - yum install -y \ device-mapper-persistent-data \ lvm2 \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml index 89a7a7478..b1e1e8845 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml @@ -67,7 +67,6 @@ write_files: systemctl restart systemd-modules-load.service sysctl --system - yum install -y \ device-mapper-persistent-data \ lvm2 \ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml index 8dc58fd1a..b91e171b8 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml @@ -67,7 +67,6 @@ write_files: systemctl restart systemd-modules-load.service sysctl --system - yum install -y \ device-mapper-persistent-data \ lvm2 \ diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index 6cb508a59..f804908c5 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -72,9 +72,6 @@ write_files: hostnamectl set-hostname node1 - - yum update -y --disablerepo='*' --enablerepo='*microsoft*' - yum install -y \ device-mapper-persistent-data \ lvm2 \ From e2efd1d303920599ad3e5a3afee944d80af365c0 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 17 May 2022 16:35:44 +0500 Subject: [PATCH 144/489] Split prowjobs; move them to .prow directory (#1290) Signed-off-by: Waleed Malik --- .prow.yaml | 929 ------------------------------ .prow/e2e-features.yaml | 110 ++++ .prow/postsubmits.yaml | 63 ++ .prow/provider-alibaba.yaml | 38 ++ .prow/provider-anexia.yaml | 36 ++ .prow/provider-aws.yaml | 214 +++++++ .prow/provider-azure.yaml | 84 +++ .prow/provider-digitalocean.yaml | 36 ++ .prow/provider-equinix-metal.yaml | 37 ++ .prow/provider-gcp.yaml | 37 ++ .prow/provider-hetzner.yaml | 35 ++ .prow/provider-kubevirt.yaml | 38 ++ .prow/provider-linode.yaml | 37 ++ .prow/provider-nutanix.yaml | 38 ++ .prow/provider-openstack.yaml | 60 ++ .prow/provider-scaleway.yaml | 36 ++ .prow/provider-vsphere.yaml | 83 +++ .prow/verify.yaml | 160 +++++ 18 files changed, 1142 insertions(+), 929 deletions(-) delete mode 100644 .prow.yaml create mode 100644 .prow/e2e-features.yaml create mode 100644 .prow/postsubmits.yaml create mode 100644 .prow/provider-alibaba.yaml create mode 100644 .prow/provider-anexia.yaml create mode 100644 .prow/provider-aws.yaml create mode 100644 .prow/provider-azure.yaml create mode 100644 .prow/provider-digitalocean.yaml create mode 100644 .prow/provider-equinix-metal.yaml create mode 100644 .prow/provider-gcp.yaml create mode 100644 .prow/provider-hetzner.yaml create mode 100644 .prow/provider-kubevirt.yaml create mode 100644 .prow/provider-linode.yaml create mode 100644 .prow/provider-nutanix.yaml create mode 100644 .prow/provider-openstack.yaml create mode 100644 .prow/provider-scaleway.yaml create mode 100644 .prow/provider-vsphere.yaml create mode 100644 .prow/verify.yaml diff --git a/.prow.yaml b/.prow.yaml deleted file mode 100644 index f9ea92b65..000000000 --- a/.prow.yaml +++ /dev/null @@ -1,929 +0,0 @@ ---- -presubmits: - - name: pull-machine-controller-build - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - make - args: - - download-gocache - - all - resources: - requests: - cpu: 1 - - - name: pull-machine-controller-dependencies - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - make - args: - - check-dependencies - resources: - requests: - cpu: 800m - - - name: pull-machine-controller-lint - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-goproxy: "true" - spec: - containers: - - image: golangci/golangci-lint:v1.42.1 - command: - - make - args: - - lint - resources: - requests: - cpu: 800m - memory: 6Gi - - - name: pull-machine-controller-yamllint - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-goproxy: "true" - spec: - containers: - - image: quay.io/kubermatic/yamllint:0.1 - command: - - "sh" - - "-c" - - "yamllint -c .yamllint.conf ." - resources: - requests: - cpu: 200m - - - name: pre-machine-controller-verify-shfmt - run_if_changed: "^hack/" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - spec: - containers: - - image: docker.io/mvdan/shfmt:v3.3.1 - command: - - "/bin/shfmt" - args: - # -l list files whose formatting differs from shfmt's - # -d error with a diff when the formatting differs - # -i uint indent: 0 for tabs (default), >0 for number of spaces - # -sr redirect operators will be followed by a space - - "-l" - - "-sr" - - "-i" - - "2" - - "-d" - - "hack" - resources: - requests: - memory: 32Mi - cpu: 50m - limits: - memory: 256Mi - cpu: 250m - - - name: pull-machine-controller-verify-boilerplate - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - spec: - containers: - - image: quay.io/kubermatic-labs/boilerplate:v0.2.0 - command: - - "./hack/verify-boilerplate.sh" - resources: - requests: - memory: 64Mi - cpu: 100m - - - name: pull-machine-controller-license-validation - run_if_changed: "^go.(mod|sum)$" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-goproxy: "true" - spec: - containers: - - image: quay.io/kubermatic/wwhrd:0.4.0-1 - command: - - ./hack/verify-licenses.sh - resources: - requests: - memory: 512Mi - cpu: 1 - - - name: pull-machine-controller-test - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - make - args: - - download-gocache - - test-unit - resources: - requests: - cpu: 800m - - - name: pull-machine-controller-e2e-invalid-objects-get-rejected - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-azure: "true" - preset-digitalocean: "true" - preset-gce: "true" - preset-e2e-ssh: "true" - preset-hetzner: "true" - preset-openstack: "true" - preset-vsphere: "true" - preset-kubevirt: "true" - preset-alibaba: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestInvalidObjectsGetRejected" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-kubevirt - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - max_concurrency: 1 - labels: - preset-kubevirt: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestKubevirtProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-alibaba - optional: true - always_run: false - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - max_concurrency: 1 - labels: - preset-alibaba: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestAlibabaProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-custom-ca - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-openstack: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestCustomCAsAreApplied" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-openstack - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-openstack: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestOpenstackProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-openstack-project-auth - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-openstack: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestOpenstackProjectAuthProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-aws - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestAWSProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-aws-arm - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestAWSARMProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-digitalocean - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-digitalocean: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestDigitalOceanProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-azure - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-azure: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestAzureProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-azure-custom-image-reference - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-azure: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestAzureCustomImageReferenceProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-azure-redhat-satellite - optional: true - always_run: false - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-azure: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestAzureProvisioningE2ERedhatSatellite" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-gce - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-gce: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestGCEProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-hetzner - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestHetznerProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-linode - always_run: false - optional: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-linode: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestLinodeProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-equinix-metal - optional: true - run_if_changed: pkg\/cloudprovider\/provider\/equinixmetal\/.* - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-equinix-metal: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestEquinixMetalProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-cherryservers - optional: true - run_if_changed: pkg\/cloudprovider\/provider\/cherryservers\/.* - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-cherryservers: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestCherryServersProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-vsphere - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-vsphere: "true" - preset-rhel: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestVsphereProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-nutanix - optional: true - always_run: false - run_if_changed: "(pkg/cloudprovider/provider/nutanix/)" - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-nutanix: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestNutanixProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-anexia - always_run: false - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-anexia: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestAnexiaProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-ubuntu-upgrade - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-openstack: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestUbuntuProvisioningWithUpgradeE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-deployment-upgrade - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestDeploymentControllerUpgradesMachineE2E" - env: - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-aws-ebs-encryption-enabled - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestAWSEbsEncryptionEnabledProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-aws-flatcar-containerd - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestAWSFlatcarContainerdProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-aws-spot-instance - always_run: true - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-rhel: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestAWSSpotInstanceProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-aws-sles - always_run: false - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestAWSSLESProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-aws-flatcar-coreos-cloud-init - always_run: false - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestAWSFlatcarCoreOSCloudInit8ProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-aws-centos8 - always_run: false - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestAWSCentOS8ProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-vsphere-datastore-cluster - always_run: false - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-vsphere: "true" - preset-rhel: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestVsphereDatastoreClusterProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-vsphere-resource-pool - always_run: false - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-vsphere: "true" - preset-rhel: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestVsphereResourcePoolProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-scaleway - always_run: false - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-scaleway: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestScalewayProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - - - name: pull-machine-controller-e2e-aws-assume-role - always_run: false - decorate: true - error_on_eviction: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws-assume-role: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - spec: - containers: - - image: golang:1.18.1 - command: - - "./hack/ci-e2e-test.sh" - args: - - "TestAWSAssumeRoleProvisioningE2E" - resources: - requests: - memory: 1Gi - cpu: 500m - -postsubmits: - - name: ci-push-machine-controller-image - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - branches: - - ^master$ - # Match on tags - - ^v\d+\.\d+\.\d+.* - labels: - preset-docker-push: "true" - preset-goproxy: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.12-1 - command: - - /bin/bash - - -c - - | - set -euo pipefail - start-docker.sh - docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD - docker login -u $QUAY_IO_USERNAME -p $QUAY_IO_PASSWORD quay.io - make download-gocache docker-image-publish - # docker-in-docker needs privileged mode - securityContext: - privileged: true - resources: - requests: - cpu: 2 - memory: 1Gi - - - name: ci-push-machine-controller-upload-gocache - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - branches: - - ^master$ - labels: - preset-goproxy: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.12-1 - command: - - "./hack/ci-upload-gocache.sh" - resources: - requests: - cpu: 2 - memory: 1Gi diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml new file mode 100644 index 000000000..ed4bb04f5 --- /dev/null +++ b/.prow/e2e-features.yaml @@ -0,0 +1,110 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-e2e-invalid-objects-get-rejected + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-aws: "true" + preset-azure: "true" + preset-digitalocean: "true" + preset-gce: "true" + preset-e2e-ssh: "true" + preset-hetzner: "true" + preset-openstack: "true" + preset-vsphere: "true" + preset-kubevirt: "true" + preset-alibaba: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestInvalidObjectsGetRejected" + resources: + requests: + memory: 1Gi + cpu: 500m + + - name: pull-machine-controller-e2e-custom-ca + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-openstack: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-rhel: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestCustomCAsAreApplied" + resources: + requests: + memory: 1Gi + cpu: 500m + + - name: pull-machine-controller-e2e-ubuntu-upgrade + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-openstack: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestUbuntuProvisioningWithUpgradeE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + + - name: pull-machine-controller-e2e-deployment-upgrade + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestDeploymentControllerUpgradesMachineE2E" + env: + resources: + requests: + memory: 1Gi + cpu: 500m diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml new file mode 100644 index 000000000..e76fef473 --- /dev/null +++ b/.prow/postsubmits.yaml @@ -0,0 +1,63 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +postsubmits: + - name: ci-push-machine-controller-image + always_run: true + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + branches: + - ^master$ + # Match on tags + - ^v\d+\.\d+\.\d+.* + labels: + preset-docker-push: "true" + preset-goproxy: "true" + spec: + containers: + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.12-1 + command: + - /bin/bash + - -c + - | + set -euo pipefail + start-docker.sh + docker login -u $DOCKERHUB_USERNAME -p $DOCKERHUB_PASSWORD + docker login -u $QUAY_IO_USERNAME -p $QUAY_IO_PASSWORD quay.io + make download-gocache docker-image-publish + # docker-in-docker needs privileged mode + securityContext: + privileged: true + resources: + requests: + cpu: 2 + memory: 1Gi + + - name: ci-push-machine-controller-upload-gocache + always_run: true + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + branches: + - ^master$ + labels: + preset-goproxy: "true" + spec: + containers: + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.12-1 + command: + - "./hack/ci-upload-gocache.sh" + resources: + requests: + cpu: 2 + memory: 1Gi diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml new file mode 100644 index 000000000..e1dfd9eb9 --- /dev/null +++ b/.prow/provider-alibaba.yaml @@ -0,0 +1,38 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-e2e-alibaba + optional: true + always_run: false + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + max_concurrency: 1 + labels: + preset-alibaba: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestAlibabaProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml new file mode 100644 index 000000000..de9a4b926 --- /dev/null +++ b/.prow/provider-anexia.yaml @@ -0,0 +1,36 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-e2e-anexia + always_run: false + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-anexia: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestAnexiaProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml new file mode 100644 index 000000000..4fc458e8c --- /dev/null +++ b/.prow/provider-aws.yaml @@ -0,0 +1,214 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-e2e-aws + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-aws: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-rhel: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestAWSProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + + - name: pull-machine-controller-e2e-aws-arm + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-aws: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestAWSARMProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + + - name: pull-machine-controller-e2e-aws-ebs-encryption-enabled + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-aws: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestAWSEbsEncryptionEnabledProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + + - name: pull-machine-controller-e2e-aws-flatcar-containerd + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-aws: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestAWSFlatcarContainerdProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + + - name: pull-machine-controller-e2e-aws-spot-instance + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-aws: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + preset-rhel: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestAWSSpotInstanceProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + + - name: pull-machine-controller-e2e-aws-sles + always_run: false + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-aws: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestAWSSLESProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + + - name: pull-machine-controller-e2e-aws-flatcar-coreos-cloud-init + always_run: false + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-aws: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestAWSFlatcarCoreOSCloudInit8ProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + + - name: pull-machine-controller-e2e-aws-centos8 + always_run: false + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-aws: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestAWSCentOS8ProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + + - name: pull-machine-controller-e2e-aws-assume-role + always_run: false + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-aws-assume-role: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestAWSAssumeRoleProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml new file mode 100644 index 000000000..ee0999f30 --- /dev/null +++ b/.prow/provider-azure.yaml @@ -0,0 +1,84 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-e2e-azure + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-azure: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-rhel: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestAzureProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + + - name: pull-machine-controller-e2e-azure-custom-image-reference + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-azure: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-rhel: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestAzureCustomImageReferenceProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + + - name: pull-machine-controller-e2e-azure-redhat-satellite + optional: true + always_run: false + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-azure: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-rhel: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestAzureProvisioningE2ERedhatSatellite" + resources: + requests: + memory: 1Gi + cpu: 500m diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml new file mode 100644 index 000000000..e2a869ddc --- /dev/null +++ b/.prow/provider-digitalocean.yaml @@ -0,0 +1,36 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-e2e-digitalocean + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-digitalocean: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestDigitalOceanProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml new file mode 100644 index 000000000..f18630ea2 --- /dev/null +++ b/.prow/provider-equinix-metal.yaml @@ -0,0 +1,37 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-e2e-equinix-metal + optional: true + run_if_changed: pkg\/cloudprovider\/provider\/equinixmetal\/.* + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-equinix-metal: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestEquinixMetalProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml new file mode 100644 index 000000000..09a4bacc0 --- /dev/null +++ b/.prow/provider-gcp.yaml @@ -0,0 +1,37 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-e2e-gce + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-gce: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-rhel: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestGCEProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml new file mode 100644 index 000000000..ffd7b61b0 --- /dev/null +++ b/.prow/provider-hetzner.yaml @@ -0,0 +1,35 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-e2e-hetzner + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestHetznerProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml new file mode 100644 index 000000000..6a044a7e8 --- /dev/null +++ b/.prow/provider-kubevirt.yaml @@ -0,0 +1,38 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-e2e-kubevirt + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + max_concurrency: 1 + labels: + preset-kubevirt: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-rhel: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestKubevirtProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml new file mode 100644 index 000000000..0c330dc54 --- /dev/null +++ b/.prow/provider-linode.yaml @@ -0,0 +1,37 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-e2e-linode + always_run: false + optional: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-linode: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestLinodeProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml new file mode 100644 index 000000000..7853dee2f --- /dev/null +++ b/.prow/provider-nutanix.yaml @@ -0,0 +1,38 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-e2e-nutanix + optional: true + always_run: false + run_if_changed: "(pkg/cloudprovider/provider/nutanix/)" + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-nutanix: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestNutanixProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml new file mode 100644 index 000000000..49dfd4bd5 --- /dev/null +++ b/.prow/provider-openstack.yaml @@ -0,0 +1,60 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-e2e-openstack + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-openstack: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-rhel: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestOpenstackProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + + - name: pull-machine-controller-e2e-openstack-project-auth + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-openstack: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-rhel: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestOpenstackProjectAuthProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml new file mode 100644 index 000000000..34f732fd8 --- /dev/null +++ b/.prow/provider-scaleway.yaml @@ -0,0 +1,36 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-e2e-scaleway + always_run: false + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-scaleway: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestScalewayProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml new file mode 100644 index 000000000..a9357ccd9 --- /dev/null +++ b/.prow/provider-vsphere.yaml @@ -0,0 +1,83 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-e2e-vsphere + always_run: true + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-vsphere: "true" + preset-rhel: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestVsphereProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + + - name: pull-machine-controller-e2e-vsphere-datastore-cluster + always_run: false + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-vsphere: "true" + preset-rhel: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestVsphereDatastoreClusterProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m + + - name: pull-machine-controller-e2e-vsphere-resource-pool + always_run: false + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-vsphere: "true" + preset-rhel: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestVsphereResourcePoolProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m diff --git a/.prow/verify.yaml b/.prow/verify.yaml new file mode 100644 index 000000000..ca05ea61c --- /dev/null +++ b/.prow/verify.yaml @@ -0,0 +1,160 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-build + always_run: true + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - make + args: + - download-gocache + - all + resources: + requests: + cpu: 1 + + - name: pull-machine-controller-dependencies + always_run: true + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - make + args: + - check-dependencies + resources: + requests: + cpu: 800m + + - name: pull-machine-controller-lint + always_run: true + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-goproxy: "true" + spec: + containers: + - image: golangci/golangci-lint:v1.42.1 + command: + - make + args: + - lint + resources: + requests: + cpu: 800m + memory: 6Gi + + - name: pull-machine-controller-yamllint + always_run: true + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-goproxy: "true" + spec: + containers: + - image: quay.io/kubermatic/yamllint:0.1 + command: + - "sh" + - "-c" + - "yamllint -c .yamllint.conf ." + resources: + requests: + cpu: 200m + + - name: pre-machine-controller-verify-shfmt + run_if_changed: "^hack/" + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + spec: + containers: + - image: docker.io/mvdan/shfmt:v3.3.1 + command: + - "/bin/shfmt" + args: + # -l list files whose formatting differs from shfmt's + # -d error with a diff when the formatting differs + # -i uint indent: 0 for tabs (default), >0 for number of spaces + # -sr redirect operators will be followed by a space + - "-l" + - "-sr" + - "-i" + - "2" + - "-d" + - "hack" + resources: + requests: + memory: 32Mi + cpu: 50m + limits: + memory: 256Mi + cpu: 250m + + - name: pull-machine-controller-verify-boilerplate + always_run: true + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + spec: + containers: + - image: quay.io/kubermatic-labs/boilerplate:v0.2.0 + command: + - "./hack/verify-boilerplate.sh" + resources: + requests: + memory: 64Mi + cpu: 100m + + - name: pull-machine-controller-license-validation + run_if_changed: "^go.(mod|sum)$" + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-goproxy: "true" + spec: + containers: + - image: quay.io/kubermatic/wwhrd:0.4.0-1 + command: + - ./hack/verify-licenses.sh + resources: + requests: + memory: 512Mi + cpu: 1 + + - name: pull-machine-controller-test + always_run: true + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.1 + command: + - make + args: + - download-gocache + - test-unit + resources: + requests: + cpu: 800m From 6cd9dbc0cd4efabd772685cb0979fee5cea4c84e Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Tue, 17 May 2022 15:47:38 +0200 Subject: [PATCH 145/489] Upgrade to Go 1.18.2 (#1293) * Bump prowjobs to Go 1.18.2 Signed-off-by: Marvin Beckers * Also update Go in Dockerfile and Makefile Signed-off-by: Marvin Beckers --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 18 +++++++++--------- .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vsphere.yaml | 6 +++--- .prow/verify.yaml | 6 +++--- Dockerfile | 2 +- Makefile | 2 +- hack/update-fixtures.sh | 2 +- 20 files changed, 39 insertions(+), 39 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index ed4bb04f5..49d3d8ea0 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -32,7 +32,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: @@ -55,7 +55,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: @@ -77,7 +77,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: @@ -98,7 +98,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index e76fef473..61dfdda06 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.12-1 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.13-4 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.12-1 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.13-4 command: - "./hack/ci-upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index e1dfd9eb9..5680da163 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -27,7 +27,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index de9a4b926..0da566fef 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -25,7 +25,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 4fc458e8c..ec52a2400 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -26,7 +26,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: @@ -48,7 +48,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: @@ -70,7 +70,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: @@ -115,7 +115,7 @@ presubmits: preset-rhel: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: @@ -137,7 +137,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: @@ -159,7 +159,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: @@ -181,7 +181,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: @@ -203,7 +203,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index ee0999f30..a88e0b8e0 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -26,7 +26,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: @@ -49,7 +49,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: @@ -73,7 +73,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index e2a869ddc..9617a8e84 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -25,7 +25,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index f18630ea2..ead732a44 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -26,7 +26,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 09a4bacc0..7dc6f44b9 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -26,7 +26,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index ffd7b61b0..9fcd5c1e1 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -24,7 +24,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 6a044a7e8..61cdd508f 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -27,7 +27,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 0c330dc54..fbfe4e97f 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -26,7 +26,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 7853dee2f..234dd205e 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 49dfd4bd5..7437288a2 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -26,7 +26,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: @@ -49,7 +49,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 34f732fd8..6eba49268 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -25,7 +25,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index a9357ccd9..3e9750aed 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -26,7 +26,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: @@ -49,7 +49,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: @@ -72,7 +72,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - "./hack/ci-e2e-test.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index ca05ea61c..1fc33888b 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - make args: @@ -39,7 +39,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - make args: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.1 + - image: golang:1.18.2 command: - make args: diff --git a/Dockerfile b/Dockerfile index 6af23a884..a059d4b38 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.18.1 +ARG GO_VERSION=1.18.2 FROM golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index a27616783..1df761943 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.18.1 +GO_VERSION ?= 1.18.2 GOOS ?= $(shell go env GOOS) diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 6873ee5dd..a85226f41 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.18.1 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.18.2 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... From 8f9e914433e5e03c7c3e6737960a241e217dd432 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 17 May 2022 20:52:36 +0500 Subject: [PATCH 146/489] Enable more golangci-linters (#1292) * Add asciicheck, bidichk, bodyclose, deadcode, depguard, durationcheck linters Signed-off-by: Waleed Malik * Add errname linter Signed-off-by: Waleed Malik * Enable errorlint linter Signed-off-by: Waleed Malik * Enable exportloopref linter Signed-off-by: Waleed Malik * Enable godot linter Signed-off-by: Waleed Malik * Enable importas, noctx, nolintlint linters Signed-off-by: Waleed Malik * Add nosprintfhostport linter Signed-off-by: Waleed Malik * Enable predeclared linter Signed-off-by: Waleed Malik * Enable promlinter linter Signed-off-by: Waleed Malik * Add staticcheck linter Signed-off-by: Waleed Malik * Enable tenv linter Signed-off-by: Waleed Malik * Enable wastedassign, whitespace linter Signed-off-by: Waleed Malik * Enable nakedret linter Signed-off-by: Waleed Malik * Bump golangci-lint to v1.46.1 in CI Signed-off-by: Waleed Malik * refactor: fix error condition checks Signed-off-by: Waleed Malik * Add comment regarding ignoring SA1019 Signed-off-by: Waleed Malik --- .golangci.yml | 70 ++++++--- .prow/verify.yaml | 2 +- cmd/machine-controller/main.go | 54 +++---- go.mod | 2 +- go.sum | 15 +- pkg/admission/admission.go | 8 +- pkg/admission/machinedeployments.go | 6 +- .../machinedeployments_validation.go | 6 +- pkg/admission/machines.go | 30 ++-- pkg/admission/util.go | 4 +- pkg/apis/cluster/common/consts.go | 16 +- pkg/apis/cluster/v1alpha1/cluster_types.go | 6 +- .../v1alpha1/conversions/conversions.go | 9 +- .../providerconfig_to_providerspec.go | 18 +-- pkg/apis/cluster/v1alpha1/defaults.go | 2 +- pkg/apis/cluster/v1alpha1/machine_types.go | 7 +- .../cluster/v1alpha1/machineclass_types.go | 2 +- .../v1alpha1/machinedeployment_types.go | 6 +- pkg/apis/cluster/v1alpha1/machineset_types.go | 10 +- .../cluster/v1alpha1/migrations/migrations.go | 66 ++++---- pkg/cloudprovider/cache/cloudprovidercache.go | 10 +- pkg/cloudprovider/common/ssh/ssh.go | 8 +- pkg/cloudprovider/errors/errors.go | 12 +- pkg/cloudprovider/instance/instance.go | 2 +- pkg/cloudprovider/provider.go | 4 +- .../provider/alibaba/provider.go | 73 +++++---- .../provider/anexia/helper_test.go | 1 - pkg/cloudprovider/provider/anexia/provider.go | 16 +- .../provider/anexia/provider_test.go | 2 - .../provider/anexia/types/errors.go | 6 +- pkg/cloudprovider/provider/aws/provider.go | 72 +++++---- .../provider/aws/types/cloudconfig.go | 4 +- pkg/cloudprovider/provider/aws/types/types.go | 2 +- .../provider/azure/create_delete_resources.go | 51 +++---- .../provider/azure/get_client.go | 14 +- pkg/cloudprovider/provider/azure/provider.go | 144 +++++++++--------- .../provider/azure/types/cloudconfig.go | 2 +- .../provider/azure/types/types.go | 4 +- .../baremetal/plugins/tinkerbell/driver.go | 27 ++-- .../plugins/tinkerbell/driver_test.go | 12 +- .../plugins/tinkerbell/metadata/client.go | 8 +- .../provider/baremetal/provider.go | 40 ++--- .../provider/digitalocean/provider.go | 38 ++--- .../provider/equinixmetal/provider.go | 36 +++-- pkg/cloudprovider/provider/fake/provider.go | 8 +- pkg/cloudprovider/provider/gce/config.go | 42 ++--- pkg/cloudprovider/provider/gce/provider.go | 21 +-- .../provider/gce/provider_test.go | 4 +- pkg/cloudprovider/provider/gce/service.go | 10 +- .../provider/gce/types/cloudconfig.go | 4 +- .../provider/hetzner/provider.go | 38 ++--- .../provider/kubevirt/provider.go | 99 ++++++------ .../provider/kubevirt/types/types.go | 18 +-- pkg/cloudprovider/provider/linode/provider.go | 29 ++-- pkg/cloudprovider/provider/nutanix/client.go | 10 +- .../provider/nutanix/provider.go | 24 +-- .../provider/openstack/helper.go | 6 +- .../provider/openstack/provider.go | 134 ++++++++-------- .../provider/openstack/provider_test.go | 12 +- .../provider/openstack/types/cloudconfig.go | 6 +- .../provider/scaleway/provider.go | 43 +++--- pkg/cloudprovider/provider/vsphere/client.go | 12 +- pkg/cloudprovider/provider/vsphere/helper.go | 84 +++++----- pkg/cloudprovider/provider/vsphere/network.go | 4 +- .../provider/vsphere/provider.go | 98 ++++++------ .../provider/vsphere/provider_test.go | 6 +- .../provider/vsphere/types/cloudconfig.go | 6 +- .../provider/vsphere/types/types.go | 2 +- pkg/cloudprovider/types/types.go | 16 +- pkg/cloudprovider/util/cloud_init_settings.go | 15 +- .../util/cloud_init_settings_test.go | 9 +- pkg/cloudprovider/util/http.go | 4 +- pkg/cloudprovider/util/net.go | 6 +- pkg/cloudprovider/util/util.go | 5 +- pkg/cloudprovider/util/util_test.go | 9 +- pkg/cloudprovider/validationwrapper.go | 26 ++-- pkg/clusterinfo/configmap.go | 7 +- pkg/containerruntime/config.go | 2 +- pkg/containerruntime/containerd.go | 2 +- pkg/controller/machine/bootstrap.go | 37 +++-- pkg/controller/machine/kubeconfig.go | 24 +-- pkg/controller/machine/kubeconfig_test.go | 9 +- pkg/controller/machine/machine_controller.go | 114 +++++++------- pkg/controller/machine/machine_test.go | 45 ++++-- pkg/controller/machine/metrics.go | 18 +-- .../machinedeployment_controller.go | 4 +- pkg/controller/machinedeployment/rolling.go | 2 +- pkg/controller/machinedeployment/sync.go | 4 +- pkg/controller/machineset/delete_policy.go | 2 +- .../machineset/machineset_controller.go | 6 +- .../nodecsrapprover/node_csr_approver.go | 28 ++-- pkg/controller/util/machine_deployment.go | 20 +-- pkg/health/readiness.go | 4 +- pkg/ini/duration.go | 6 +- pkg/ini/escape.go | 4 +- pkg/ini/escape_test.go | 2 +- pkg/kubernetes/helper.go | 4 +- pkg/node/eviction/eviction.go | 15 +- pkg/node/eviction/eviction_test.go | 4 +- pkg/node/nodemanager/node_manager.go | 2 +- pkg/node/poddeletion/pod_deletion.go | 37 +++-- pkg/providerconfig/types.go | 13 +- pkg/providerconfig/types/types.go | 12 +- pkg/providerconfig/types/types_test.go | 4 - pkg/providerconfig/types_test.go | 1 - pkg/rhsm/satellite_subscription_manager.go | 6 +- pkg/rhsm/subscription_manager.go | 25 ++- pkg/userdata/convert/ignition-converter.go | 2 +- pkg/userdata/flatcar/provider.go | 18 +-- .../helper/download_binaries_script.go | 8 +- pkg/userdata/helper/helper.go | 14 +- pkg/userdata/helper/kubelet.go | 22 +-- pkg/userdata/helper/template_functions.go | 2 +- pkg/userdata/manager/plugin.go | 4 +- pkg/userdata/sles/provider.go | 15 +- pkg/userdata/ubuntu/provider.go | 12 +- test/e2e/provisioning/all_e2e_test.go | 26 ++-- test/e2e/provisioning/deploymentscenario.go | 9 +- test/e2e/provisioning/helper.go | 3 +- test/e2e/provisioning/migrateuidscenario.go | 37 ++--- test/e2e/provisioning/verify.go | 51 +++---- 121 files changed, 1217 insertions(+), 1206 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 834cf8c47..5425522a1 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -8,33 +8,59 @@ run: linters: enable: - - revive - - govet - - gofmt - - structcheck - - varcheck - - unconvert - - ineffassign + - asciicheck + - bidichk + - bodyclose + - deadcode + - depguard + - durationcheck + - errcheck + - errname + - errorlint + - exportloopref - goconst - gocyclo - - misspell + - godot + - gofmt - gosimple + - govet + - importas + - ineffassign + - misspell + - nakedret + - noctx + - nolintlint + - nosprintfhostport + - predeclared + - promlinter + - revive + - staticcheck + - structcheck + - tenv + - unconvert - unused - - errcheck + - varcheck + - wastedassign + - whitespace disable-all: true issues: exclude: - - should have comment or be unexported - - should have comment \\(or a comment on this block\\) or be unexported - - func Convert_MachinesV1alpha1Machine_To_ClusterV1alpha1Machine should be ConvertMachinesV1alpha1MachineToClusterV1alpha1Machine - - func Convert_MachineDeployment_ProviderConfig_To_ProviderSpec should be ConvertMachineDeploymentProviderConfigToProviderSpec - - func Convert_MachineSet_ProviderConfig_To_ProviderSpec should be ConvertMachineSetProviderConfigToProviderSpec - - func Convert_Machine_ProviderConfig_To_ProviderSpec should be ConvertMachineProviderConfigToProviderSpec - - 'counter\.Set is deprecated: Use NewConstMetric' - - 'eviction\.go:221:4: the surrounding loop is unconditionally terminated' - - 'cyclomatic complexity 31 of func `verifyMigrateUID` is high' - - 'cyclomatic complexity 31 of func `main` is high' - - 'cyclomatic complexity 34 of func `\(\*provider\)\.getConfig` is high' - - 'cyclomatic complexity 31 of func `\(\*provider\)\.Validate` is high' - - 'cyclomatic complexity 31 of func `\(\*provider\)\.Create` is high' + - should have comment or be unexported + - should have comment \\(or a comment on this block\\) or be unexported + - func Convert_MachinesV1alpha1Machine_To_ClusterV1alpha1Machine should be ConvertMachinesV1alpha1MachineToClusterV1alpha1Machine + - func Convert_MachineDeployment_ProviderConfig_To_ProviderSpec should be ConvertMachineDeploymentProviderConfigToProviderSpec + - func Convert_MachineSet_ProviderConfig_To_ProviderSpec should be ConvertMachineSetProviderConfigToProviderSpec + - func Convert_Machine_ProviderConfig_To_ProviderSpec should be ConvertMachineProviderConfigToProviderSpec + - 'counter\.Set is deprecated: Use NewConstMetric' + - 'eviction\.go:221:4: the surrounding loop is unconditionally terminated' + - "cyclomatic complexity 31 of func `verifyMigrateUID` is high" + - "cyclomatic complexity 31 of func `main` is high" + - 'cyclomatic complexity 34 of func `\(\*provider\)\.getConfig` is high' + - 'cyclomatic complexity 31 of func `\(\*provider\)\.Validate` is high' + - 'cyclomatic complexity 31 of func `\(\*provider\)\.Create` is high' + # SA1019: node.Spec.ConfigSource is deprecated: Previously used to specify the source of the node's configuration for the DynamicKubeletConfig feature. + # This feature is removed from Kubelets as of 1.24 and will be fully removed in 1.26. +optional + # We still support setting dynamic kubelet config feature in machine-controller. Hence, ignoring this error. + # TODO: remove this once we remove support for the feature in 1.23 + - "SA1019: node.Spec.ConfigSource is deprecated" diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 1fc33888b..dc256eec7 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -56,7 +56,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golangci/golangci-lint:v1.42.1 + - image: golangci/golangci-lint:v1.46.1 command: - make args: diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 14f5d3234..20b6b2055 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -89,35 +89,35 @@ const ( defaultLeaderElectionNamespace = "kube-system" ) -// controllerRunOptions holds data that are required to create and run machine controller +// controllerRunOptions holds data that are required to create and run machine controller. type controllerRunOptions struct { - // kubeClient a client that knows how to consume kubernetes API + // kubeClient a client that knows how to consume kubernetes API. kubeClient *kubernetes.Clientset - // metrics a struct that holds all metrics we want to collect + // metrics a struct that holds all metrics we want to collect. metrics *machinecontroller.MetricsCollection - // kubeconfigProvider knows how to get cluster information stored under a ConfigMap + // kubeconfigProvider knows how to get cluster information stored under a ConfigMap. kubeconfigProvider machinecontroller.KubeconfigProvider - // name of the controller. When set the controller will only process machines with the label "machine.k8s.io/controller": name + // name of the controller. When set the controller will only process machines with the label "machine.k8s.io/controller": name. name string - // Name of the ServiceAccount from which the bootstrap token secret will be fetched. A bootstrap token will be created + // Name of the ServiceAccount from which the bootstrap token secret will be fetched. A bootstrap token will be created. // if this is nil bootstrapTokenServiceAccountName *types.NamespacedName - // prometheusRegisterer is used by the MachineController instance to register its metrics + // prometheusRegisterer is used by the MachineController instance to register its metrics. prometheusRegisterer prometheus.Registerer - // The cfg is used by the migration to conditionally spawn additional clients + // The cfg is used by the migration to conditionally spawn additional clients. cfg *restclient.Config - // The timeout in which machines owned by a MachineSet must join the cluster to avoid being + // The timeout in which machines owned by a MachineSet must join the cluster to avoid being. // deleted by the machine-controller joinClusterTimeout *time.Duration - // Will instruct the machine-controller to skip the eviction if the machine deletion is older than skipEvictionAfter + // Will instruct the machine-controller to skip the eviction if the machine deletion is older than skipEvictionAfter. skipEvictionAfter time.Duration // Enable NodeCSRApprover controller to automatically approve node serving certificate requests. @@ -127,7 +127,7 @@ type controllerRunOptions struct { useOSM bool - // A port range to reserve for services with NodePort visibility + // A port range to reserve for services with NodePort visibility. nodePortRange string } @@ -136,7 +136,7 @@ func main() { klog.InitFlags(nil) // This is also being registered in kubevirt.io/kubevirt/pkg/kubecli/kubecli.go so - // we have to guard it + // we have to guard it. // TODO: Evaluate alternatives to importing the CLI. Generate our own client? Use a dynamic client? if flag.Lookup("kubeconfig") == nil { flag.StringVar(&kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") @@ -311,19 +311,19 @@ func createManager(syncPeriod time.Duration, options controllerRunOptions) (mana MetricsBindAddress: metricsAddress, }) if err != nil { - return nil, fmt.Errorf("error building ctrlruntime manager: %v", err) + return nil, fmt.Errorf("error building ctrlruntime manager: %w", err) } if err := mgr.AddReadyzCheck("alive", healthz.Ping); err != nil { - return nil, fmt.Errorf("failed to add readiness check: %v", err) + return nil, fmt.Errorf("failed to add readiness check: %w", err) } if err := mgr.AddHealthzCheck("kubeconfig", health.KubeconfigAvailable(options.kubeconfigProvider)); err != nil { - return nil, fmt.Errorf("failed to add health check: %v", err) + return nil, fmt.Errorf("failed to add health check: %w", err) } if err := mgr.AddHealthzCheck("apiserver-connection", health.ApiserverReachable(options.kubeClient)); err != nil { - return nil, fmt.Errorf("failed to add health check: %v", err) + return nil, fmt.Errorf("failed to add health check: %w", err) } if profiling { @@ -335,7 +335,7 @@ func createManager(syncPeriod time.Duration, options controllerRunOptions) (mana m.HandleFunc("/trace", pprof.Trace) if err := mgr.AddMetricsExtraHandler("/debug/pprof/", m); err != nil { - return nil, fmt.Errorf("failed to add pprof http handlers: %v", err) + return nil, fmt.Errorf("failed to add pprof http handlers: %w", err) } } @@ -343,7 +343,7 @@ func createManager(syncPeriod time.Duration, options controllerRunOptions) (mana mgr: mgr, opt: options, }); err != nil { - return nil, fmt.Errorf("failed to add bootstrap runnable: %v", err) + return nil, fmt.Errorf("failed to add bootstrap runnable: %w", err) } return mgr, nil @@ -354,7 +354,7 @@ type controllerBootstrap struct { opt controllerRunOptions } -// NeedLeaderElection implements manager.LeaderElectionRunnable +// NeedLeaderElection implements manager.LeaderElectionRunnable. func (bs *controllerBootstrap) NeedLeaderElection() bool { return true } @@ -371,14 +371,14 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { Client: client, } - // Migrate MachinesV1Alpha1Machine to ClusterV1Alpha1Machine + // Migrate MachinesV1Alpha1Machine to ClusterV1Alpha1Machine. if err := migrations.MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary(ctx, client, bs.opt.kubeClient, providerData); err != nil { - return fmt.Errorf("migration to clusterv1alpha1 failed: %v", err) + return fmt.Errorf("migration to clusterv1alpha1 failed: %w", err) } - // Migrate providerConfig field to providerSpec field + // Migrate providerConfig field to providerSpec field. if err := migrations.MigrateProviderConfigToProviderSpecIfNecesary(ctx, bs.opt.cfg, client); err != nil { - return fmt.Errorf("migration of providerConfig field to providerSpec field failed: %v", err) + return fmt.Errorf("migration of providerConfig field to providerSpec field failed: %w", err) } machineCollector := machinecontroller.NewMachineCollector(ctx, bs.mgr.GetClient()) @@ -400,20 +400,20 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { bs.opt.useOSM, bs.opt.nodePortRange, ); err != nil { - return fmt.Errorf("failed to add Machine controller to manager: %v", err) + return fmt.Errorf("failed to add Machine controller to manager: %w", err) } if err := machinesetcontroller.Add(bs.mgr); err != nil { - return fmt.Errorf("failed to add MachineSet controller to manager: %v", err) + return fmt.Errorf("failed to add MachineSet controller to manager: %w", err) } if err := machinedeploymentcontroller.Add(bs.mgr); err != nil { - return fmt.Errorf("failed to add MachineDeployment controller to manager: %v", err) + return fmt.Errorf("failed to add MachineDeployment controller to manager: %w", err) } if bs.opt.nodeCSRApprover { if err := nodecsrapprover.Add(bs.mgr); err != nil { - return fmt.Errorf("failed to add NodeCSRApprover controller to manager: %v", err) + return fmt.Errorf("failed to add NodeCSRApprover controller to manager: %w", err) } } diff --git a/go.mod b/go.mod index ad668bc56..77da41103 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/kubermatic/machine-controller go 1.18 require ( - cloud.google.com/go/logging v1.1.2 + cloud.google.com/go/logging v1.4.0 cloud.google.com/go/monitoring v1.4.0 github.com/Azure/azure-sdk-for-go v64.1.0+incompatible github.com/Azure/go-autorest/autorest/azure/auth v0.5.5 diff --git a/go.sum b/go.sum index b59717ee2..ae24cfa09 100644 --- a/go.sum +++ b/go.sum @@ -13,7 +13,6 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.71.0/go.mod h1:qZfY4Y7AEIQwG/fQYD3xrxLNkQZ0Xzf3HGeqCkA6LVM= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= @@ -42,8 +41,8 @@ cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6m cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/logging v1.1.2 h1:KNALX0NZn8UJhqKnqoHxhMqyoZfBZoh5wF7CQJZ5XrU= -cloud.google.com/go/logging v1.1.2/go.mod h1:KrljuAHIw631j9+QXsnq9vDwsrwmdxfGpivMR68M7DY= +cloud.google.com/go/logging v1.4.0 h1:suMj8d7qzDC9Gzm14aBQGWYZl6TGVz9SyOJDxLN3kNE= +cloud.google.com/go/logging v1.4.0/go.mod h1:FKOKd0UX2KtN01HZbMlVug72OgiX27ZE8AG4lktFnGo= cloud.google.com/go/monitoring v1.4.0 h1:05+IuNMbh40hbxcqQ4SnynbwZbLG1Wc9dysIJxnfv7U= cloud.google.com/go/monitoring v1.4.0/go.mod h1:y6xnxfwI3hTFWOdkOaD7nfJVlwuC3/mS/5kvtT131p4= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -1042,7 +1041,6 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201026091529-146b70c837a4/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -1075,6 +1073,8 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= @@ -1271,8 +1271,6 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201030143252-cf7a54d06671/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201105220310-78b158585360/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1316,12 +1314,12 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.34.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= @@ -1382,9 +1380,7 @@ google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201026171402-d4b8fe4fd877/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201030142918-24207fddd1c3/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201102152239-715cce707fb0/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201106154455-f9bfe239b0ba/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1395,6 +1391,7 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210413151531-c14fb6ef47c3/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go index c4799de7f..87259ac0b 100644 --- a/pkg/admission/admission.go +++ b/pkg/admission/admission.go @@ -118,12 +118,12 @@ func createAdmissionResponse(original, mutated runtime.Object) (*admissionv1.Adm if !apiequality.Semantic.DeepEqual(original, mutated) { patchOpts, err := newJSONPatch(original, mutated) if err != nil { - return nil, fmt.Errorf("failed to create json patch: %v", err) + return nil, fmt.Errorf("failed to create json patch: %w", err) } patchRaw, err := json.Marshal(patchOpts) if err != nil { - return nil, fmt.Errorf("failed to marshal json patch: %v", err) + return nil, fmt.Errorf("failed to marshal json patch: %w", err) } klog.V(3).Infof("Produced jsonpatch: %s", string(patchRaw)) @@ -183,7 +183,7 @@ func readReview(r *http.Request) (*admissionv1.AdmissionReview, error) { } body, err := ioutil.ReadAll(r.Body) if err != nil { - return nil, fmt.Errorf("error reading data from request body: %v", err) + return nil, fmt.Errorf("error reading data from request body: %w", err) } // verify the content type is accurate @@ -193,7 +193,7 @@ func readReview(r *http.Request) (*admissionv1.AdmissionReview, error) { admissionReview := &admissionv1.AdmissionReview{} if err := json.Unmarshal(body, admissionReview); err != nil { - return nil, fmt.Errorf("failed to unmarshal request into admissionReview: %v", err) + return nil, fmt.Errorf("failed to unmarshal request into admissionReview: %w", err) } if admissionReview.Request == nil { return nil, errors.New("invalid admission review: no request defined") diff --git a/pkg/admission/machinedeployments.go b/pkg/admission/machinedeployments.go index 541bacf0b..1e183aa80 100644 --- a/pkg/admission/machinedeployments.go +++ b/pkg/admission/machinedeployments.go @@ -31,14 +31,14 @@ import ( func (ad *admissionData) mutateMachineDeployments(ctx context.Context, ar admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error) { machineDeployment := clusterv1alpha1.MachineDeployment{} if err := json.Unmarshal(ar.Object.Raw, &machineDeployment); err != nil { - return nil, fmt.Errorf("failed to unmarshal: %v", err) + return nil, fmt.Errorf("failed to unmarshal: %w", err) } machineDeploymentOriginal := machineDeployment.DeepCopy() machineDeploymentDefaultingFunction(&machineDeployment) if err := mutationsForMachineDeployment(&machineDeployment, ad.useOSM); err != nil { - return nil, fmt.Errorf("mutation failed: %v", err) + return nil, fmt.Errorf("mutation failed: %w", err) } if errs := validateMachineDeployment(machineDeployment); len(errs) > 0 { @@ -57,7 +57,7 @@ func (ad *admissionData) mutateMachineDeployments(ctx context.Context, ar admiss if ar.Operation == admissionv1.Update { var oldMachineDeployment clusterv1alpha1.MachineDeployment if err := json.Unmarshal(ar.OldObject.Raw, &oldMachineDeployment); err != nil { - return nil, fmt.Errorf("failed to unmarshal OldObject: %v", err) + return nil, fmt.Errorf("failed to unmarshal OldObject: %w", err) } if equal := apiequality.Semantic.DeepEqual(oldMachineDeployment.Spec.Template.Spec, machineDeployment.Spec.Template.Spec); equal { machineSpecNeedsValidation = false diff --git a/pkg/admission/machinedeployments_validation.go b/pkg/admission/machinedeployments_validation.go index b7d50718c..664ba6644 100644 --- a/pkg/admission/machinedeployments_validation.go +++ b/pkg/admission/machinedeployments_validation.go @@ -120,7 +120,7 @@ func machineDeploymentDefaultingFunction(md *v1alpha1.MachineDeployment) { func mutationsForMachineDeployment(md *v1alpha1.MachineDeployment, useOSM bool) error { providerConfig, err := providerconfigtypes.GetConfig(md.Spec.Template.Spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to read MachineDeployment.Spec.Template.Spec.ProviderSpec: %v", err) + return fmt.Errorf("failed to read MachineDeployment.Spec.Template.Spec.ProviderSpec: %w", err) } if useOSM { @@ -134,14 +134,14 @@ func mutationsForMachineDeployment(md *v1alpha1.MachineDeployment, useOSM bool) if providerConfig.CloudProvider == cloudProviderPacket { err = migrateToEquinixMetal(providerConfig) if err != nil { - return fmt.Errorf("failed to migrate packet to equinix metal: %v", err) + return fmt.Errorf("failed to migrate packet to equinix metal: %w", err) } } // Update value in original object md.Spec.Template.Spec.ProviderSpec.Value.Raw, err = json.Marshal(providerConfig) if err != nil { - return fmt.Errorf("failed to json marshal machine.spec.providerSpec: %v", err) + return fmt.Errorf("failed to json marshal machine.spec.providerSpec: %w", err) } return nil diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index 2174fcb47..d317ef615 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -37,13 +37,13 @@ import ( // BypassSpecNoModificationRequirementAnnotation is used to bypass the "no machine.spec modification" allowed // restriction from the webhook in order to change the spec in some special cases, e.G. for the migration of -// the `providerConfig` field to `providerSpec` +// the `providerConfig` field to `providerSpec`. const BypassSpecNoModificationRequirementAnnotation = "kubermatic.io/bypass-no-spec-mutation-requirement" func (ad *admissionData) mutateMachines(ctx context.Context, ar admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error) { machine := clusterv1alpha1.Machine{} if err := json.Unmarshal(ar.Object.Raw, &machine); err != nil { - return nil, fmt.Errorf("failed to unmarshal: %v", err) + return nil, fmt.Errorf("failed to unmarshal: %w", err) } machineOriginal := machine.DeepCopy() @@ -53,11 +53,11 @@ func (ad *admissionData) mutateMachines(ctx context.Context, ar admissionv1.Admi // Only hidden exception: the machine-controller may set the .Spec.Name to .Metadata.Name // because otherwise it can never add the delete finalizer as it internally defaults the Name // as well, since on the CREATE request for machines, there is only Metadata.GenerateName set - // so we can't default it initially + // so we can't default it initially. if ar.Operation == admissionv1.Update { oldMachine := clusterv1alpha1.Machine{} if err := json.Unmarshal(ar.OldObject.Raw, &oldMachine); err != nil { - return nil, fmt.Errorf("failed to unmarshal OldObject: %v", err) + return nil, fmt.Errorf("failed to unmarshal OldObject: %w", err) } if oldMachine.Spec.Name != machine.Spec.Name && machine.Spec.Name == machine.Name { oldMachine.Spec.Name = machine.Spec.Name @@ -71,7 +71,7 @@ func (ad *admissionData) mutateMachines(ctx context.Context, ar admissionv1.Admi } } } - // Delete the `BypassSpecNoModificationRequirementAnnotation` annotation, it should be valid only once + // Delete the `BypassSpecNoModificationRequirementAnnotation` annotation, it should be valid only once. delete(machine.Annotations, BypassSpecNoModificationRequirementAnnotation) // Default name @@ -80,7 +80,7 @@ func (ad *admissionData) mutateMachines(ctx context.Context, ar admissionv1.Admi } // Default and verify .Spec on CREATE only, its expensive and not required to do it on UPDATE - // as we disallow .Spec changes anyways + // as we disallow .Spec changes anyways. if ar.Operation == admissionv1.Create { if err := ad.defaultAndValidateMachineSpec(ctx, &machine.Spec); err != nil { return nil, err @@ -103,26 +103,26 @@ func (ad *admissionData) mutateMachines(ctx context.Context, ar admissionv1.Admi func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec *clusterv1alpha1.MachineSpec) error { providerConfig, err := providerconfigtypes.GetConfig(spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to read machine.spec.providerSpec: %v", err) + return fmt.Errorf("failed to read machine.spec.providerSpec: %w", err) } // Packet has been renamed to Equinix Metal if providerConfig.CloudProvider == cloudProviderPacket { err = migrateToEquinixMetal(providerConfig) if err != nil { - return fmt.Errorf("failed to migrate packet to equinix metal: %v", err) + return fmt.Errorf("failed to migrate packet to equinix metal: %w", err) } } skg := providerconfig.NewConfigVarResolver(ctx, ad.workerClient) prov, err := cloudprovider.ForProvider(providerConfig.CloudProvider, skg) if err != nil { - return fmt.Errorf("failed to get cloud provider %q: %v", providerConfig.CloudProvider, err) + return fmt.Errorf("failed to get cloud provider %q: %w", providerConfig.CloudProvider, err) } // Verify operating system. if _, err := ad.userDataManager.ForOS(providerConfig.OperatingSystem); err != nil { - return fmt.Errorf("failed to get OS '%s': %v", providerConfig.OperatingSystem, err) + return fmt.Errorf("failed to get OS '%s': %w", providerConfig.OperatingSystem, err) } // Check kubelet version @@ -141,7 +141,7 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec // Validate SSH keys if err := validatePublicKeys(providerConfig.SSHPublicKeys); err != nil { - return fmt.Errorf("Invalid public keys specified: %v", err) + return fmt.Errorf("Invalid public keys specified: %w", err) } defaultedOperatingSystemSpec, err := providerconfig.DefaultOperatingSystemSpec( @@ -156,17 +156,17 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec providerConfig.OperatingSystemSpec = defaultedOperatingSystemSpec spec.ProviderSpec.Value.Raw, err = json.Marshal(providerConfig) if err != nil { - return fmt.Errorf("failed to json marshal machine.spec.providerSpec: %v", err) + return fmt.Errorf("failed to json marshal machine.spec.providerSpec: %w", err) } defaultedSpec, err := prov.AddDefaults(*spec) if err != nil { - return fmt.Errorf("failed to default machineSpec: %v", err) + return fmt.Errorf("failed to default machineSpec: %w", err) } spec = &defaultedSpec if err := prov.Validate(*spec); err != nil { - return fmt.Errorf("validation failed: %v", err) + return fmt.Errorf("validation failed: %w", err) } return nil @@ -176,7 +176,7 @@ func validatePublicKeys(keys []string) error { for _, s := range keys { _, _, _, _, err := ssh.ParseAuthorizedKey([]byte(s)) if err != nil { - return fmt.Errorf("invalid public key %q: %v", s, err) + return fmt.Errorf("invalid public key %q: %w", s, err) } } diff --git a/pkg/admission/util.go b/pkg/admission/util.go index ed9f6d77b..8e95017a4 100644 --- a/pkg/admission/util.go +++ b/pkg/admission/util.go @@ -32,7 +32,7 @@ func migrateToEquinixMetal(providerConfig *providerconfigtypes.Config) (err erro // We first need to perform in-place replacement for this field rawConfig := map[string]interface{}{} if err := json.Unmarshal(providerConfig.CloudProviderSpec.Raw, &rawConfig); err != nil { - return fmt.Errorf("failed to unmarshal providerConfig.CloudProviderSpec.Raw: %v", err) + return fmt.Errorf("failed to unmarshal providerConfig.CloudProviderSpec.Raw: %w", err) } // NB: We have to set the token only if apiKey existed, otherwise, migrated // machines will not create at all (authentication errors). @@ -45,7 +45,7 @@ func migrateToEquinixMetal(providerConfig *providerconfigtypes.Config) (err erro // Update original object providerConfig.CloudProviderSpec.Raw, err = json.Marshal(rawConfig) if err != nil { - return fmt.Errorf("failed to json marshal providerConfig.CloudProviderSpec.Raw: %v", err) + return fmt.Errorf("failed to json marshal providerConfig.CloudProviderSpec.Raw: %w", err) } return nil } diff --git a/pkg/apis/cluster/common/consts.go b/pkg/apis/cluster/common/consts.go index 89639b1f1..c1c12df27 100644 --- a/pkg/apis/cluster/common/consts.go +++ b/pkg/apis/cluster/common/consts.go @@ -25,7 +25,7 @@ import ( ) // Constants aren't automatically generated for unversioned packages. -// Instead share the same constant for all versioned packages +// Instead share the same constant for all versioned packages. type MachineStatusError string const ( @@ -33,7 +33,7 @@ const ( // is not supported by this cluster. This is not a transient error, but // indicates a state that must be fixed before progress can be made. // - // Example: the ProviderSpec specifies an instance type that doesn't exist, + // Example: the ProviderSpec specifies an instance type that doesn't exist,. InvalidConfigurationMachineError MachineStatusError = "InvalidConfiguration" // This indicates that the MachineSpec has been updated in a way that @@ -61,9 +61,9 @@ const ( // There was an error while trying to update a Node that this // Machine represents. This may indicate a transient problem that will be - // fixed automatically with time, such as a service outage, + // fixed automatically with time, such as a service outage. // - // Example: error updating load balancers + // Example: error updating load balancers. UpdateMachineError MachineStatusError = "UpdateError" // An error was encountered while trying to delete the Node that this @@ -80,7 +80,7 @@ const ( // // Example use case: A controller that deletes Machines which do // not result in a Node joining the cluster within a given timeout - // and that are managed by a MachineSet + // and that are managed by a MachineSet. JoinClusterTimeoutMachineError = "JoinClusterTimeoutError" ) @@ -143,14 +143,14 @@ const ( ) const ( - // Annotation prefixes, used on Machine objects to indicate the parameters that been used to create those Machines + // Annotation prefixes, used on Machine objects to indicate the parameters that been used to create those Machines. KubeletFeatureGatesAnnotationPrefixV1 = "v1.kubelet-featuregates.machine-controller.kubermatic.io" KubeletFlagsGroupAnnotationPrefixV1 = "v1.kubelet-flags.machine-controller.kubermatic.io" KubeletConfigAnnotationPrefixV1 = "v1.kubelet-config.machine-controller.kubermatic.io" ) // SetKubeletFeatureGates marshal and save featureGates into metaobject annotations with -// KubeletFeatureGatesAnnotationPrefixV1 prefix +// KubeletFeatureGatesAnnotationPrefixV1 prefix. func SetKubeletFeatureGates(metaobj metav1.Object, featureGates map[string]bool) { annts := metaobj.GetAnnotations() if annts == nil { @@ -162,7 +162,7 @@ func SetKubeletFeatureGates(metaobj metav1.Object, featureGates map[string]bool) metaobj.SetAnnotations(annts) } -// SetKubeletFlags marshal and save flags into metaobject annotations with KubeletFlagsGroupAnnotationPrefixV1 prefix +// SetKubeletFlags marshal and save flags into metaobject annotations with KubeletFlagsGroupAnnotationPrefixV1 prefix. func SetKubeletFlags(metaobj metav1.Object, flags map[KubeletFlags]string) { annts := metaobj.GetAnnotations() if annts == nil { diff --git a/pkg/apis/cluster/v1alpha1/cluster_types.go b/pkg/apis/cluster/v1alpha1/cluster_types.go index e5a12f350..3994e384a 100644 --- a/pkg/apis/cluster/v1alpha1/cluster_types.go +++ b/pkg/apis/cluster/v1alpha1/cluster_types.go @@ -45,7 +45,7 @@ type Cluster struct { /// [Cluster] /// [ClusterSpec] -// ClusterSpec defines the desired state of Cluster +// ClusterSpec defines the desired state of Cluster. type ClusterSpec struct { // Cluster network configuration ClusterNetwork ClusterNetworkingConfig `json:"clusterNetwork"` @@ -85,7 +85,7 @@ type NetworkRanges struct { /// [NetworkRanges] /// [ClusterStatus] -// ClusterStatus defines the observed state of Cluster +// ClusterStatus defines the observed state of Cluster. type ClusterStatus struct { // APIEndpoint represents the endpoint to communicate with the IP. // +optional @@ -154,7 +154,7 @@ func (o *Cluster) Validate() field.ErrorList { // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// ClusterList contains a list of Cluster +// ClusterList contains a list of Cluster. type ClusterList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` diff --git a/pkg/apis/cluster/v1alpha1/conversions/conversions.go b/pkg/apis/cluster/v1alpha1/conversions/conversions.go index ce50fc54f..d33350b8e 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/conversions.go +++ b/pkg/apis/cluster/v1alpha1/conversions/conversions.go @@ -30,7 +30,6 @@ import ( func Convert_MachinesV1alpha1Machine_To_ClusterV1alpha1Machine(in *machinesv1alpha1.Machine, out *clusterv1alpha1.Machine) error { out.ObjectMeta = in.ObjectMeta out.Spec.ObjectMeta = in.Spec.ObjectMeta - out.SelfLink = "" out.UID = "" out.ResourceVersion = "" out.Generation = 0 @@ -42,10 +41,10 @@ func Convert_MachinesV1alpha1Machine_To_ClusterV1alpha1Machine(in *machinesv1alp // only has one additional field, so we cast by serializing and deserializing inStatusJSON, err := json.Marshal(in.Status) if err != nil { - return fmt.Errorf("failed to marshal downstreammachine status: %v", err) + return fmt.Errorf("failed to marshal downstreammachine status: %w", err) } if err = json.Unmarshal(inStatusJSON, &out.Status); err != nil { - return fmt.Errorf("failed to unmarshal downstreammachine status: %v", err) + return fmt.Errorf("failed to unmarshal downstreammachine status: %w", err) } out.Spec.ObjectMeta = in.Spec.ObjectMeta out.Spec.Taints = in.Spec.Taints @@ -61,10 +60,10 @@ func Convert_MachinesV1alpha1Machine_To_ClusterV1alpha1Machine(in *machinesv1alp // To work around this, we put it into the providerConfig inMachineVersionJSON, err := json.Marshal(in.Spec.Versions) if err != nil { - return fmt.Errorf("failed to marshal downstreammachine version: %v", err) + return fmt.Errorf("failed to marshal downstreammachine version: %w", err) } if err = json.Unmarshal(inMachineVersionJSON, &out.Spec.Versions); err != nil { - return fmt.Errorf("failed to unmarshal downstreammachine version: %v", err) + return fmt.Errorf("failed to unmarshal downstreammachine version: %w", err) } out.Spec.ConfigSource = in.Spec.ConfigSource return nil diff --git a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec.go b/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec.go index e163e257c..d48b19462 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec.go +++ b/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec.go @@ -86,7 +86,7 @@ func Convert_MachineDeployment_ProviderConfig_To_ProviderSpec(in []byte) (*clust var wasConverted bool superMachineDeployment := &machineDeploymentWithProviderSpecAndProviderConfig{} if err := json.Unmarshal(in, superMachineDeployment); err != nil { - return nil, wasConverted, fmt.Errorf("error unmarshalling machineDeployment object: %v", err) + return nil, wasConverted, fmt.Errorf("error unmarshalling machineDeployment object: %w", err) } if superMachineDeployment.Spec.Template.Spec.ProviderConfig != nil && superMachineDeployment.Spec.Template.Spec.ProviderSpec != nil { return nil, wasConverted, fmt.Errorf("both .spec.template.spec.providerConfig and .spec.template.spec.providerSpec were non-nil for machineDeployment %s", superMachineDeployment.Name) @@ -100,10 +100,10 @@ func Convert_MachineDeployment_ProviderConfig_To_ProviderSpec(in []byte) (*clust machineDeployment := &clusterv1alpha1.MachineDeployment{} superMachineDeploymentBytes, err := json.Marshal(superMachineDeployment) if err != nil { - return nil, wasConverted, fmt.Errorf("failed to marshal superMachineDeployment object for machineDeployment %s: %v", superMachineDeployment.Name, err) + return nil, wasConverted, fmt.Errorf("failed to marshal superMachineDeployment object for machineDeployment %s: %w", superMachineDeployment.Name, err) } if err := json.Unmarshal(superMachineDeploymentBytes, machineDeployment); err != nil { - return nil, wasConverted, fmt.Errorf("failed to unmarshal superMachineDeployment object for machineDeployment %s back into machineDeployment object: %v", superMachineDeployment.Name, err) + return nil, wasConverted, fmt.Errorf("failed to unmarshal superMachineDeployment object for machineDeployment %s back into machineDeployment object: %w", superMachineDeployment.Name, err) } return machineDeployment, wasConverted, nil } @@ -112,7 +112,7 @@ func Convert_MachineSet_ProviderConfig_To_ProviderSpec(in []byte) (*clusterv1alp var wasConverted bool superMachineSet := &machineSetWithProviderSpecAndProviderConfig{} if err := json.Unmarshal(in, superMachineSet); err != nil { - return nil, wasConverted, fmt.Errorf("error unmarshalling machineSet object: %v", err) + return nil, wasConverted, fmt.Errorf("error unmarshalling machineSet object: %w", err) } if superMachineSet.Spec.Template.Spec.ProviderConfig != nil && superMachineSet.Spec.Template.Spec.ProviderSpec != nil { return nil, wasConverted, fmt.Errorf("both .spec.template.spec.providerConfig and .spec.template.spec.providerSpec were non-nil for machineSet %s", superMachineSet.Name) @@ -126,10 +126,10 @@ func Convert_MachineSet_ProviderConfig_To_ProviderSpec(in []byte) (*clusterv1alp machineSet := &clusterv1alpha1.MachineSet{} superMachineSetBytes, err := json.Marshal(superMachineSet) if err != nil { - return nil, wasConverted, fmt.Errorf("failed to marshal superMachineSet object for machineSet %s: %v", superMachineSet.Name, err) + return nil, wasConverted, fmt.Errorf("failed to marshal superMachineSet object for machineSet %s: %w", superMachineSet.Name, err) } if err := json.Unmarshal(superMachineSetBytes, machineSet); err != nil { - return nil, wasConverted, fmt.Errorf("failed to unmarshal superMachineSet object for machineSet %s back into machineSet object: %v", superMachineSet.Name, err) + return nil, wasConverted, fmt.Errorf("failed to unmarshal superMachineSet object for machineSet %s back into machineSet object: %w", superMachineSet.Name, err) } return machineSet, wasConverted, nil } @@ -139,7 +139,7 @@ func Convert_Machine_ProviderConfig_To_ProviderSpec(in []byte) (*clusterv1alpha1 superMachine := &machineWithProviderSpecAndProviderConfig{} if err := json.Unmarshal(in, superMachine); err != nil { - return nil, wasConverted, fmt.Errorf("error unmarshalling machine object: %v", err) + return nil, wasConverted, fmt.Errorf("error unmarshalling machine object: %w", err) } if superMachine.Spec.ProviderConfig != nil && superMachine.Spec.ProviderSpec != nil { return nil, wasConverted, fmt.Errorf("both .spec.providerConfig and .spec.ProviderSpec were non-nil for machine %s", superMachine.Name) @@ -153,10 +153,10 @@ func Convert_Machine_ProviderConfig_To_ProviderSpec(in []byte) (*clusterv1alpha1 machine := &clusterv1alpha1.Machine{} superMachineBytes, err := json.Marshal(superMachine) if err != nil { - return nil, wasConverted, fmt.Errorf("failed to marshal superMachine object for machine %s: %v", superMachine.Name, err) + return nil, wasConverted, fmt.Errorf("failed to marshal superMachine object for machine %s: %w", superMachine.Name, err) } if err := json.Unmarshal(superMachineBytes, machine); err != nil { - return nil, wasConverted, fmt.Errorf("failed to unmarshal superMachine object for machine %s back into machine object: %v", superMachine.Name, err) + return nil, wasConverted, fmt.Errorf("failed to unmarshal superMachine object for machine %s back into machine object: %w", superMachine.Name, err) } return machine, wasConverted, nil } diff --git a/pkg/apis/cluster/v1alpha1/defaults.go b/pkg/apis/cluster/v1alpha1/defaults.go index 743265cd9..33be49315 100644 --- a/pkg/apis/cluster/v1alpha1/defaults.go +++ b/pkg/apis/cluster/v1alpha1/defaults.go @@ -24,7 +24,7 @@ import ( ) // PopulateDefaultsMachineDeployment fills in default field values -// Currently it is called after reading objects, but it could be called in an admission webhook also +// Currently it is called after reading objects, but it could be called in an admission webhook also. func PopulateDefaultsMachineDeployment(d *MachineDeployment) { if d.Spec.Replicas == nil { d.Spec.Replicas = new(int32) diff --git a/pkg/apis/cluster/v1alpha1/machine_types.go b/pkg/apis/cluster/v1alpha1/machine_types.go index c7ca5f841..c6c9636f1 100644 --- a/pkg/apis/cluster/v1alpha1/machine_types.go +++ b/pkg/apis/cluster/v1alpha1/machine_types.go @@ -54,7 +54,7 @@ type Machine struct { /// [Machine] /// [MachineSpec] -// MachineSpec defines the desired state of Machine +// MachineSpec defines the desired state of Machine. type MachineSpec struct { // ObjectMeta will autopopulate the Node created. Use this to // indicate what labels, annotations, name prefix, etc., should be used @@ -109,7 +109,7 @@ type MachineSpec struct { /// [MachineSpec] /// [MachineStatus] -// MachineStatus defines the observed state of Machine +// MachineStatus defines the observed state of Machine. type MachineStatus struct { // NodeRef will point to the corresponding Node if it exists. // +optional @@ -225,6 +225,7 @@ type LastOperation struct { /// [MachineStatus] /// [MachineVersionInfo] +// Holds information regarding kubelet and controlplane versions for machine. type MachineVersionInfo struct { // Kubelet is the semantic version of kubelet to run Kubelet string `json:"kubelet"` @@ -240,7 +241,7 @@ type MachineVersionInfo struct { // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// MachineList contains a list of Machine +// MachineList contains a list of Machine. type MachineList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` diff --git a/pkg/apis/cluster/v1alpha1/machineclass_types.go b/pkg/apis/cluster/v1alpha1/machineclass_types.go index 9402ef3ca..a63b62e15 100644 --- a/pkg/apis/cluster/v1alpha1/machineclass_types.go +++ b/pkg/apis/cluster/v1alpha1/machineclass_types.go @@ -67,7 +67,7 @@ type MachineClass struct { // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// MachineClassList contains a list of MachineClasses +// MachineClassList contains a list of MachineClasses. type MachineClassList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` diff --git a/pkg/apis/cluster/v1alpha1/machinedeployment_types.go b/pkg/apis/cluster/v1alpha1/machinedeployment_types.go index bdf89d27e..1455d7129 100644 --- a/pkg/apis/cluster/v1alpha1/machinedeployment_types.go +++ b/pkg/apis/cluster/v1alpha1/machinedeployment_types.go @@ -24,7 +24,7 @@ import ( ) /// [MachineDeploymentSpec] -// MachineDeploymentSpec defines the desired state of MachineDeployment +// MachineDeploymentSpec defines the desired state of MachineDeployment. type MachineDeploymentSpec struct { // Number of desired machines. Defaults to 1. // This is a pointer to distinguish between explicit zero and not specified. @@ -126,7 +126,7 @@ type MachineRollingUpdateDeployment struct { /// [MachineRollingUpdateDeployment] /// [MachineDeploymentStatus] -// MachineDeploymentStatus defines the observed state of MachineDeployment +// MachineDeploymentStatus defines the observed state of MachineDeployment. type MachineDeploymentStatus struct { // The generation observed by the deployment controller. // +optional @@ -183,7 +183,7 @@ type MachineDeployment struct { // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// MachineDeploymentList contains a list of MachineDeployment +// MachineDeploymentList contains a list of MachineDeployment. type MachineDeploymentList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` diff --git a/pkg/apis/cluster/v1alpha1/machineset_types.go b/pkg/apis/cluster/v1alpha1/machineset_types.go index e30b11205..3d0e8adfb 100644 --- a/pkg/apis/cluster/v1alpha1/machineset_types.go +++ b/pkg/apis/cluster/v1alpha1/machineset_types.go @@ -47,7 +47,7 @@ type MachineSet struct { /// [MachineSet] /// [MachineSetSpec] -// MachineSetSpec defines the desired state of MachineSet +// MachineSetSpec defines the desired state of MachineSet. type MachineSetSpec struct { // Replicas is the number of desired replicas. // This is a pointer to distinguish between explicit zero and unspecified. @@ -104,7 +104,7 @@ const ( /// [MachineSetSpec] // doxygen marker /// [MachineTemplateSpec] // doxygen marker -// MachineTemplateSpec describes the data needed to create a Machine from a template +// MachineTemplateSpec describes the data needed to create a Machine from a template. type MachineTemplateSpec struct { // Standard object's metadata. // More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata @@ -120,7 +120,7 @@ type MachineTemplateSpec struct { /// [MachineTemplateSpec] /// [MachineSetStatus] -// MachineSetStatus defines the observed state of MachineSet +// MachineSetStatus defines the observed state of MachineSet. type MachineSetStatus struct { // Replicas is the most recently observed number of replicas. Replicas int32 `json:"replicas"` @@ -189,7 +189,7 @@ func (m *MachineSet) Validate() field.ErrorList { return errors } -// DefaultingFunction sets default MachineSet field values +// DefaultingFunction sets default MachineSet field values... func (m *MachineSet) Default() { log.Printf("Defaulting fields for MachineSet %s\n", m.Name) @@ -211,7 +211,7 @@ func (m *MachineSet) Default() { // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// MachineSetList contains a list of MachineSet +// MachineSetList contains a list of MachineSet. type MachineSetList struct { metav1.TypeMeta `json:",inline"` metav1.ListMeta `json:"metadata,omitempty"` diff --git a/pkg/apis/cluster/v1alpha1/migrations/migrations.go b/pkg/apis/cluster/v1alpha1/migrations/migrations.go index 5d2b39f03..b2696588e 100644 --- a/pkg/apis/cluster/v1alpha1/migrations/migrations.go +++ b/pkg/apis/cluster/v1alpha1/migrations/migrations.go @@ -18,6 +18,7 @@ package migrations import ( "context" + "errors" "fmt" "time" @@ -54,7 +55,7 @@ func MigrateProviderConfigToProviderSpecIfNecesary(ctx context.Context, config * klog.Infof("Starting to migrate providerConfigs to providerSpecs") dynamicClient, err := dynamicclient.NewForConfig(config) if err != nil { - return fmt.Errorf("failed to construct dynamic client: %v", err) + return fmt.Errorf("failed to construct dynamic client: %w", err) } machineGVR := schema.GroupVersionResource{Group: "cluster.k8s.io", Version: "v1alpha1", Resource: "machines"} @@ -63,16 +64,16 @@ func MigrateProviderConfigToProviderSpecIfNecesary(ctx context.Context, config * machines, err := dynamicClient.Resource(machineGVR).List(ctx, metav1.ListOptions{}) if err != nil { - return fmt.Errorf("failed to list machine objects: %v", err) + return fmt.Errorf("failed to list machine objects: %w", err) } for _, machine := range machines.Items { marshalledObject, err := machine.MarshalJSON() if err != nil { - return fmt.Errorf("failed to marshal unstructured machine %s: %v", machine.GetName(), err) + return fmt.Errorf("failed to marshal unstructured machine %s: %w", machine.GetName(), err) } convertedMachine, wasConverted, err := conversions.Convert_Machine_ProviderConfig_To_ProviderSpec(marshalledObject) if err != nil { - return fmt.Errorf("failed to convert machine: %v", err) + return fmt.Errorf("failed to convert machine: %w", err) } if wasConverted { klog.Infof("Converted providerConfig -> providerSpec for machine %s/%s, attempting to update", convertedMachine.Namespace, convertedMachine.Name) @@ -83,7 +84,7 @@ func MigrateProviderConfigToProviderSpecIfNecesary(ctx context.Context, config * // spec are not allowed convertedMachine.Annotations[machinecontrolleradmission.BypassSpecNoModificationRequirementAnnotation] = "true" if err := client.Update(ctx, convertedMachine); err != nil { - return fmt.Errorf("failed to update converted machine %s: %v", convertedMachine.Name, err) + return fmt.Errorf("failed to update converted machine %s: %w", convertedMachine.Name, err) } klog.Infof("Successfully updated machine %s/%s after converting providerConfig -> providerSpec", convertedMachine.Namespace, convertedMachine.Name) } @@ -91,21 +92,21 @@ func MigrateProviderConfigToProviderSpecIfNecesary(ctx context.Context, config * machineSets, err := dynamicClient.Resource(machineSetGVR).List(ctx, metav1.ListOptions{}) if err != nil { - return fmt.Errorf("failed to list MachineSets: %v", err) + return fmt.Errorf("failed to list MachineSets: %w", err) } for _, machineSet := range machineSets.Items { marshalledObject, err := machineSet.MarshalJSON() if err != nil { - return fmt.Errorf("failed to marshal unstructured MachineSet %s: %v", machineSet.GetName(), err) + return fmt.Errorf("failed to marshal unstructured MachineSet %s: %w", machineSet.GetName(), err) } convertedMachineSet, machineSetWasConverted, err := conversions.Convert_MachineSet_ProviderConfig_To_ProviderSpec(marshalledObject) if err != nil { - return fmt.Errorf("failed to convert MachineSet %s/%s: %v", machineSet.GetNamespace(), machineSet.GetName(), err) + return fmt.Errorf("failed to convert MachineSet %s/%s: %w", machineSet.GetNamespace(), machineSet.GetName(), err) } if machineSetWasConverted { klog.Infof("Converted providerConfig -> providerSpec for MachineSet %s/%s, attempting to update", convertedMachineSet.Namespace, convertedMachineSet.Name) if err := client.Update(ctx, convertedMachineSet); err != nil { - return fmt.Errorf("failed to update MachineSet %s/%s after converting providerConfig -> providerSpec: %v", convertedMachineSet.Namespace, convertedMachineSet.Name, err) + return fmt.Errorf("failed to update MachineSet %s/%s after converting providerConfig -> providerSpec: %w", convertedMachineSet.Namespace, convertedMachineSet.Name, err) } klog.Infof("Successfully updated MachineSet %s/%s after converting providerConfig -> providerSpec", convertedMachineSet.Namespace, convertedMachineSet.Name) } @@ -113,21 +114,21 @@ func MigrateProviderConfigToProviderSpecIfNecesary(ctx context.Context, config * machineDeployments, err := dynamicClient.Resource(machineDeploymentsGVR).List(ctx, metav1.ListOptions{}) if err != nil { - return fmt.Errorf("failed to list MachineDeplyoments: %v", err) + return fmt.Errorf("failed to list MachineDeplyoments: %w", err) } for _, machineDeployment := range machineDeployments.Items { marshalledObject, err := machineDeployment.MarshalJSON() if err != nil { - return fmt.Errorf("failed to marshal unstructured MachineDeployment %s: %v", machineDeployment.GetName(), err) + return fmt.Errorf("failed to marshal unstructured MachineDeployment %s: %w", machineDeployment.GetName(), err) } convertedMachineDeployment, machineDeploymentWasConverted, err := conversions.Convert_MachineDeployment_ProviderConfig_To_ProviderSpec(marshalledObject) if err != nil { - return fmt.Errorf("failed to convert MachineDeployment %s/%s: %v", machineDeployment.GetNamespace(), machineDeployment.GetName(), err) + return fmt.Errorf("failed to convert MachineDeployment %s/%s: %w", machineDeployment.GetNamespace(), machineDeployment.GetName(), err) } if machineDeploymentWasConverted { klog.Infof("Converted providerConfig -> providerSpec for MachineDeployment %s/%s, attempting to update", convertedMachineDeployment.Namespace, convertedMachineDeployment.Name) if err := client.Update(ctx, convertedMachineDeployment); err != nil { - return fmt.Errorf("failed to update MachineDeployment %s/%s after converting providerConfig -> providerSpec: %v", convertedMachineDeployment.Namespace, convertedMachineDeployment.Name, err) + return fmt.Errorf("failed to update MachineDeployment %s/%s after converting providerConfig -> providerSpec: %w", convertedMachineDeployment.Namespace, convertedMachineDeployment.Name, err) } klog.Infof("Successfully updated MachineDeployment %s/%s after converting providerConfig -> providerSpec", convertedMachineDeployment.Namespace, convertedMachineDeployment.Name) } @@ -141,7 +142,6 @@ func MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary( ctx context.Context, client ctrlruntimeclient.Client, kubeClient kubernetes.Interface, providerData *cloudprovidertypes.ProviderData) error { - var ( cachePopulatingInterval = 15 * time.Second cachePopulatingTimeout = 10 * time.Minute @@ -156,12 +156,13 @@ func MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary( return true, nil } - if _, ok := err.(*cache.ErrCacheNotStarted); ok { + var cerr *cache.ErrCacheNotStarted + if errors.As(err, &cerr) { klog.Info("Cache hasn't started yet, trying in 5 seconds") return false, nil } - return false, fmt.Errorf("failed to get crds: %v", err) + return false, fmt.Errorf("failed to get crds: %w", err) } return true, nil }) @@ -178,15 +179,15 @@ func MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary( err = client.Get(ctx, types.NamespacedName{Name: "machines.cluster.k8s.io"}, &apiextensionsv1.CustomResourceDefinition{}) if err != nil { - return fmt.Errorf("error when checking for existence of 'machines.cluster.k8s.io' crd: %v", err) + return fmt.Errorf("error when checking for existence of 'machines.cluster.k8s.io' crd: %w", err) } if err := migrateMachines(ctx, client, kubeClient, providerData); err != nil { - return fmt.Errorf("failed to migrate machines: %v", err) + return fmt.Errorf("failed to migrate machines: %w", err) } klog.Infof("Attempting to delete CRD %s", machines.CRDName) if err := client.Delete(ctx, &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: machines.CRDName}}); err != nil { - return fmt.Errorf("failed to delete machinesv1alpha1.machine crd: %v", err) + return fmt.Errorf("failed to delete machinesv1alpha1.machine crd: %w", err) } klog.Infof("Successfully deleted CRD %s", machines.CRDName) return nil @@ -200,7 +201,7 @@ func migrateMachines(ctx context.Context, client ctrlruntimeclient.Client, kubeC machinesv1Alpha1Machines := &machinesv1alpha1.MachineList{} if err := client.List(ctx, machinesv1Alpha1Machines); err != nil { - return fmt.Errorf("failed to list machinesV1Alpha1 machines: %v", err) + return fmt.Errorf("failed to list machinesV1Alpha1 machines: %w", err) } klog.Infof("Found %v machine.machines.k8s.io/v1alpha1", len(machinesv1Alpha1Machines.Items)) @@ -213,7 +214,7 @@ func migrateMachines(ctx context.Context, client ctrlruntimeclient.Client, kubeC err := conversions.Convert_MachinesV1alpha1Machine_To_ClusterV1alpha1Machine(&machinesV1Alpha1Machine, convertedClusterv1alpha1Machine) if err != nil { - return fmt.Errorf("failed to convert machinesV1alpha1.machine to clusterV1alpha1.machine name=%s err=%v", + return fmt.Errorf("failed to convert machinesV1alpha1.machine to clusterV1alpha1.machine name=%s err=%w", machinesV1Alpha1Machine.Name, err) } convertedClusterv1alpha1Machine.Finalizers = append(convertedClusterv1alpha1Machine.Finalizers, machinecontroller.FinalizerDeleteNode) @@ -222,12 +223,12 @@ func migrateMachines(ctx context.Context, client ctrlruntimeclient.Client, kubeC // to not fail in a half-migrated state when the providerconfig is invalid providerConfig, err := providerconfigtypes.GetConfig(convertedClusterv1alpha1Machine.Spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to get provider config: %v", err) + return fmt.Errorf("failed to get provider config: %w", err) } skg := providerconfig.NewConfigVarResolver(ctx, client) prov, err := cloudprovider.ForProvider(providerConfig.CloudProvider, skg) if err != nil { - return fmt.Errorf("failed to get cloud provider %q: %v", providerConfig.CloudProvider, err) + return fmt.Errorf("failed to get cloud provider %q: %w", providerConfig.CloudProvider, err) } // We will set that to what's finally in the apisever, be that a created a clusterv1alpha1machine @@ -248,7 +249,7 @@ func migrateMachines(ctx context.Context, client ctrlruntimeclient.Client, kubeC if err != nil { // Some random error occurred if !kerrors.IsNotFound(err) { - return fmt.Errorf("failed to check if converted machine %s already exists: %v", convertedClusterv1alpha1Machine.Name, err) + return fmt.Errorf("failed to check if converted machine %s already exists: %w", convertedClusterv1alpha1Machine.Name, err) } // ClusterV1alpha1Machine does not exist yet @@ -256,7 +257,7 @@ func migrateMachines(ctx context.Context, client ctrlruntimeclient.Client, kubeC convertedClusterv1alpha1Machine.Namespace, convertedClusterv1alpha1Machine.Name) if err := client.Create(ctx, convertedClusterv1alpha1Machine); err != nil { - return fmt.Errorf("failed to create clusterv1alpha1.machine %s: %v", convertedClusterv1alpha1Machine.Name, err) + return fmt.Errorf("failed to create clusterv1alpha1.machine %s: %w", convertedClusterv1alpha1Machine.Name, err) } klog.Infof("Successfully created machine.cluster.k8s.io/v1alpha1 %s/%s", convertedClusterv1alpha1Machine.Namespace, convertedClusterv1alpha1Machine.Name) @@ -275,7 +276,7 @@ func migrateMachines(ctx context.Context, client ctrlruntimeclient.Client, kubeC existingClusterV1alpha1Machine.Namespace, existingClusterV1alpha1Machine.Name) if err := client.Update(ctx, existingClusterV1alpha1Machine); err != nil { - return fmt.Errorf("failed to update metadata of existing clusterV1Alpha1 machine: %v", err) + return fmt.Errorf("failed to update metadata of existing clusterV1Alpha1 machine: %w", err) } klog.Infof("Successfully updated existing machine.cluster.k8s.io/v1alpha1 %s/%s", existingClusterV1alpha1Machine.Namespace, existingClusterV1alpha1Machine.Name) @@ -293,7 +294,7 @@ func migrateMachines(ctx context.Context, client ctrlruntimeclient.Client, kubeC newMachineWithOldUID := finalClusterV1Alpha1Machine.DeepCopy() newMachineWithOldUID.UID = machinesV1Alpha1Machine.UID if err := prov.MigrateUID(newMachineWithOldUID, finalClusterV1Alpha1Machine.UID); err != nil { - return fmt.Errorf("running the provider migration for the UID failed: %v", err) + return fmt.Errorf("running the provider migration for the UID failed: %w", err) } // Block until we can actually GET the instance with the new UID var isMigrated bool @@ -344,7 +345,7 @@ func ensureClusterV1Alpha1NodeOwnership(ctx context.Context, machine *clusterv1a machine.Namespace, machine.Name) continue } - return fmt.Errorf("Failed to get node %s for machine %s: %v", + return fmt.Errorf("Failed to get node %s for machine %s: %w", machine.Spec.Name, machine.Name, err) } @@ -362,7 +363,7 @@ func ensureClusterV1Alpha1NodeOwnership(ctx context.Context, machine *clusterv1a node.Labels = nodeLabels return client.Update(ctx, node) }); err != nil { - return fmt.Errorf("failed to update OwnerLabel on node %s: %v", node.Name, err) + return fmt.Errorf("failed to update OwnerLabel on node %s: %w", node.Name, err) } klog.Infof("Successfully removed ownerRef and added NodeOwnerLabelName to node %s for machines.cluster.k8s.io/v1alpha1 %s/%s", node.Name, machine.Namespace, machine.Name) @@ -373,19 +374,18 @@ func ensureClusterV1Alpha1NodeOwnership(ctx context.Context, machine *clusterv1a func deleteMachinesV1Alpha1Machine(ctx context.Context, machine *machinesv1alpha1.Machine, client ctrlruntimeclient.Client) error { - machine.Finalizers = []string{} if err := client.Update(ctx, machine); err != nil { - return fmt.Errorf("failed to update machinesv1alpha1.machine %s after removing finalizer: %v", machine.Name, err) + return fmt.Errorf("failed to update machinesv1alpha1.machine %s after removing finalizer: %w", machine.Name, err) } if err := client.Delete(ctx, machine); err != nil { - return fmt.Errorf("failed to delete machine %s: %v", machine.Name, err) + return fmt.Errorf("failed to delete machine %s: %w", machine.Name, err) } if err := wait.Poll(500*time.Millisecond, 60*time.Second, func() (bool, error) { return isMachinesV1Alpha1MachineDeleted(ctx, machine.Name, client) }); err != nil { - return fmt.Errorf("failed to wait for machine %s to be deleted: %v", machine.Name, err) + return fmt.Errorf("failed to wait for machine %s to be deleted: %w", machine.Name, err) } return nil diff --git a/pkg/cloudprovider/cache/cloudprovidercache.go b/pkg/cloudprovider/cache/cloudprovidercache.go index f3c93953c..7b2c576fe 100644 --- a/pkg/cloudprovider/cache/cloudprovidercache.go +++ b/pkg/cloudprovider/cache/cloudprovidercache.go @@ -31,13 +31,13 @@ type CloudproviderCache struct { cache *gocache.Cache } -// New returns a new cloudproviderCache +// New returns a new cloudproviderCache. func New() *CloudproviderCache { return &CloudproviderCache{cache: gocache.New(5*time.Minute, 5*time.Minute)} } // Get returns an error indicating the result of the validation and a boolean indicating if -// it got a cache hit or miss +// it got a cache hit or miss. func (c *CloudproviderCache) Get(machineSpec clusterv1alpha1.MachineSpec) (error, bool, error) { id, err := getID(machineSpec) if err != nil { @@ -55,12 +55,12 @@ func (c *CloudproviderCache) Get(machineSpec clusterv1alpha1.MachineSpec) (error errVal, castable := val.(error) if !castable { - return nil, false, fmt.Errorf("failed to cast val to err: %v", err) + return nil, false, fmt.Errorf("failed to cast val to err: %w", err) } return errVal, true, nil } -// Set sets the passed value for the given machineSpec +// Set sets the passed value for the given machineSpec. func (c *CloudproviderCache) Set(machineSpec clusterv1alpha1.MachineSpec, val error) error { id, err := getID(machineSpec) if err != nil { @@ -74,7 +74,7 @@ func (c *CloudproviderCache) Set(machineSpec clusterv1alpha1.MachineSpec, val er func getID(machineSpec clusterv1alpha1.MachineSpec) (string, error) { b, err := json.Marshal(machineSpec.ProviderSpec) if err != nil { - return "", fmt.Errorf("failed to marshal MachineSpec: %v", err) + return "", fmt.Errorf("failed to marshal MachineSpec: %w", err) } sum := sha256.Sum256(b) diff --git a/pkg/cloudprovider/common/ssh/ssh.go b/pkg/cloudprovider/common/ssh/ssh.go index 4ab682e61..fce39a7f3 100644 --- a/pkg/cloudprovider/common/ssh/ssh.go +++ b/pkg/cloudprovider/common/ssh/ssh.go @@ -30,7 +30,7 @@ const privateRSAKeyBitSize = 4096 // Pubkey is only used to create temporary key pairs, thus we // do not need the Private key // The reason for not hardcoding a random public key is that -// it would look like a backdoor +// it would look like a backdoor. type Pubkey struct { Name string PublicKey string @@ -40,16 +40,16 @@ type Pubkey struct { func NewKey() (*Pubkey, error) { tmpRSAKeyPair, err := rsa.GenerateKey(rand.Reader, privateRSAKeyBitSize) if err != nil { - return nil, fmt.Errorf("failed to create private RSA key: %v", err) + return nil, fmt.Errorf("failed to create private RSA key: %w", err) } if err := tmpRSAKeyPair.Validate(); err != nil { - return nil, fmt.Errorf("failed to validate private RSA key: %v", err) + return nil, fmt.Errorf("failed to validate private RSA key: %w", err) } pubKey, err := ssh.NewPublicKey(&tmpRSAKeyPair.PublicKey) if err != nil { - return nil, fmt.Errorf("failed to generate ssh public key: %v", err) + return nil, fmt.Errorf("failed to generate ssh public key: %w", err) } return &Pubkey{ diff --git a/pkg/cloudprovider/errors/errors.go b/pkg/cloudprovider/errors/errors.go index 25681998c..d0df77409 100644 --- a/pkg/cloudprovider/errors/errors.go +++ b/pkg/cloudprovider/errors/errors.go @@ -24,15 +24,15 @@ import ( ) var ( - // ErrInstanceNotFound tells that the requested instance was not found on the cloud provider + // ErrInstanceNotFound tells that the requested instance was not found on the cloud provider. ErrInstanceNotFound = errors.New("instance not found") ) func IsNotFound(err error) bool { - return err == ErrInstanceNotFound + return errors.Is(err, ErrInstanceNotFound) } -// TerminalError is a helper struct that holds errors of type "terminal" +// TerminalError is a helper struct that holds errors of type "terminal". type TerminalError struct { Reason common.MachineStatusError Message string @@ -42,10 +42,10 @@ func (te TerminalError) Error() string { return fmt.Sprintf("An error of type = %v, with message = %v occurred", te.Reason, te.Message) } -// IsTerminalError is a helper function that helps to determine if a given error is terminal +// IsTerminalError is a helper function that helps to determine if a given error is terminal. func IsTerminalError(err error) (bool, common.MachineStatusError, string) { - tError, ok := err.(TerminalError) - if !ok { + var tError TerminalError + if !errors.As(err, &tError) { return false, "", "" } return true, tError.Reason, tError.Message diff --git a/pkg/cloudprovider/instance/instance.go b/pkg/cloudprovider/instance/instance.go index edadc9918..64299ada4 100644 --- a/pkg/cloudprovider/instance/instance.go +++ b/pkg/cloudprovider/instance/instance.go @@ -18,7 +18,7 @@ package instance import v1 "k8s.io/api/core/v1" -// Instance represents a instance on the cloud provider +// Instance represents a instance on the cloud provider. type Instance interface { // Name returns the instance name. Name() string diff --git a/pkg/cloudprovider/provider.go b/pkg/cloudprovider/provider.go index 3e6756465..c238052dc 100644 --- a/pkg/cloudprovider/provider.go +++ b/pkg/cloudprovider/provider.go @@ -44,7 +44,7 @@ import ( var ( cache = cloudprovidercache.New() - // ErrProviderNotFound tells that the requested cloud provider was not found + // ErrProviderNotFound tells that the requested cloud provider was not found. ErrProviderNotFound = errors.New("cloudprovider not found") providers = map[providerconfigtypes.CloudProvider]func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider{ @@ -107,7 +107,7 @@ var ( } ) -// ForProvider returns a CloudProvider actuator for the requested provider +// ForProvider returns a CloudProvider actuator for the requested provider. func ForProvider(p providerconfigtypes.CloudProvider, cvr *providerconfig.ConfigVarResolver) (cloudprovidertypes.Provider, error) { if p, found := providers[p]; found { return NewValidationCacheWrappingCloudProvider(p(cvr)), nil diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index 0687b8a28..33201c3b4 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -98,7 +98,7 @@ func (a *alibabaInstance) Status() instance.Status { return instance.Status(a.instance.Status) } -// New returns an Alibaba cloud provider +// New returns an Alibaba cloud provider. func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -110,7 +110,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha func (p *provider) Validate(machineSpec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(machineSpec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to parse config: %v", err) + return fmt.Errorf("failed to parse config: %w", err) } if c.AccessKeyID == "" { @@ -136,7 +136,7 @@ func (p *provider) Validate(machineSpec clusterv1alpha1.MachineSpec) error { } _, err = p.getImageIDForOS(machineSpec, pc.OperatingSystem) if err != nil { - return fmt.Errorf("invalid/not supported operating system specified %q: %v", pc.OperatingSystem, err) + return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, err) } if c.DiskType == "" { return errors.New("DiskType is missing") @@ -159,7 +159,7 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertype client, err := getClient(c.RegionID, c.AccessKeyID, c.AccessKeySecret) if err != nil { - return nil, fmt.Errorf("failed to get alibaba client: %v", err) + return nil, fmt.Errorf("failed to get alibaba client: %w", err) } foundInstance, err := getInstance(client, machine.Name, string(machine.UID)) @@ -174,7 +174,7 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertype _, err = client.StartInstance(startRequest) if err != nil { - return nil, fmt.Errorf("failed to start instance %v: %v", foundInstance.InstanceId, err) + return nil, fmt.Errorf("failed to start instance %v: %w", foundInstance.InstanceId, err) } return nil, fmt.Errorf("instance %v is in a stopped state", foundInstance.InstanceId) case runningStatus: @@ -184,9 +184,8 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertype _, err = client.AllocatePublicIpAddress(ipAddress) if err != nil { - return nil, fmt.Errorf("failed to allocate ip address for instance %v: %v", foundInstance.InstanceId, err) + return nil, fmt.Errorf("failed to allocate ip address for instance %v: %w", foundInstance.InstanceId, err) } - } return &alibabaInstance{instance: foundInstance}, nil } @@ -209,13 +208,13 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert client, err := getClient(c.RegionID, c.AccessKeyID, c.AccessKeySecret) if err != nil { - return nil, fmt.Errorf("failed to get alibaba client: %v", err) + return nil, fmt.Errorf("failed to get alibaba client: %w", err) } createInstanceRequest := ecs.CreateCreateInstanceRequest() createInstanceRequest.ImageId, err = p.getImageIDForOS(machine.Spec, pc.OperatingSystem) if err != nil { - return nil, fmt.Errorf("failed to get a valid image for machine : %v", err) + return nil, fmt.Errorf("failed to get a valid image for machine : %w", err) } createInstanceRequest.InstanceName = machine.Name createInstanceRequest.InstanceType = c.InstanceType @@ -239,7 +238,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert _, err = client.CreateInstance(createInstanceRequest) if err != nil { - return nil, fmt.Errorf("failed to create instance at Alibaba cloud: %v", err) + return nil, fmt.Errorf("failed to create instance at Alibaba cloud: %w", err) } if err = data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { @@ -247,12 +246,12 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerInstance) } }); err != nil { - return nil, fmt.Errorf("failed updating machine %v finzaliers: %v", machine.Name, err) + return nil, fmt.Errorf("failed updating machine %v finzaliers: %w", machine.Name, err) } foundInstance, err := getInstance(client, machine.Name, string(machine.UID)) if err != nil { - return nil, fmt.Errorf("failed to get alibaba instance %v due to %v", machine.Name, err) + return nil, fmt.Errorf("failed to get alibaba instance %v due to %w", machine.Name, err) } return &alibabaInstance{instance: foundInstance}, nil @@ -261,7 +260,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { foundInstance, err := p.Get(machine, data) if err != nil { - if err == cloudprovidererrors.ErrInstanceNotFound { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return util.RemoveFinalizerOnInstanceNotFound(finalizerInstance, machine, data) } return false, err @@ -277,7 +276,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider client, err := getClient(c.RegionID, c.AccessKeyID, c.AccessKeySecret) if err != nil { - return false, fmt.Errorf("failed to get alibaba client: %v", err) + return false, fmt.Errorf("failed to get alibaba client: %w", err) } deleteInstancesRequest := ecs.CreateDeleteInstancesRequest() @@ -285,7 +284,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider deleteInstancesRequest.Force = requests.Boolean("True") if _, err = client.DeleteInstances(deleteInstancesRequest); err != nil { - return false, fmt.Errorf("failed to delete instance with instanceID %s, due to %v", foundInstance.ID(), err) + return false, fmt.Errorf("failed to delete instance with instanceID %s, due to %w", foundInstance.ID(), err) } return false, nil @@ -303,24 +302,24 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to decode providerconfig: %v", err) + return fmt.Errorf("failed to decode providerconfig: %w", err) } client, err := getClient(c.RegionID, c.AccessKeyID, c.AccessKeySecret) if err != nil { - return fmt.Errorf("failed to get alibaba client: %v", err) + return fmt.Errorf("failed to get alibaba client: %w", err) } foundInstance, err := getInstance(client, machine.Name, string(machine.UID)) if err != nil { - return fmt.Errorf("failed to get alibaba instance %v due to %v", machine.Name, err) + return fmt.Errorf("failed to get alibaba instance %v due to %w", machine.Name, err) } tag := ecs.AddTagsTag{ - Value: string(new), + Value: string(newUID), Key: machineUIDTag, } request := ecs.CreateAddTagsRequest() @@ -330,7 +329,7 @@ func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) e request.Tag = &tags if _, err := client.AddTags(request); err != nil { - return fmt.Errorf("failed to create new UID tag: %v", err) + return fmt.Errorf("failed to create new UID tag: %w", err) } return nil @@ -347,7 +346,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { - return nil, nil, fmt.Errorf("failed to decode providers config: %v", err) + return nil, nil, fmt.Errorf("failed to decode providers config: %w", err) } if pconfig.OperatingSystemSpec.Raw == nil { @@ -356,46 +355,46 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p rawConfig, err := alibabatypes.GetConfig(*pconfig) if err != nil { - return nil, nil, fmt.Errorf("failed to decode alibaba providers config: %v", err) + return nil, nil, fmt.Errorf("failed to decode alibaba providers config: %w", err) } c := Config{} c.AccessKeyID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AccessKeyID, "ALIBABA_ACCESS_KEY_ID") if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"AccessKeyID\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"AccessKeyID\" field, error = %w", err) } c.AccessKeySecret, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AccessKeySecret, "ALIBABA_ACCESS_KEY_SECRET") if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"AccessKeySecret\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"AccessKeySecret\" field, error = %w", err) } c.InstanceType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.InstanceType) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"instanceType\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"instanceType\" field, error = %w", err) } c.RegionID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.RegionID) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"regionID\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"regionID\" field, error = %w", err) } c.VSwitchID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VSwitchID) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"vSwitchID\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"vSwitchID\" field, error = %w", err) } c.ZoneID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ZoneID) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"zoneID\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"zoneID\" field, error = %w", err) } c.InternetMaxBandwidthOut, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.InternetMaxBandwidthOut) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"internetMaxBandwidthOut\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"internetMaxBandwidthOut\" field, error = %w", err) } c.Labels = rawConfig.Labels c.DiskType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.DiskType) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"diskType\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"diskType\" field, error = %w", err) } c.DiskSize, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.DiskSize) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"diskSize\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"diskSize\" field, error = %w", err) } return &c, pconfig, err @@ -404,7 +403,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p func getClient(regionID, accessKeyID, accessKeySecret string) (*ecs.Client, error) { client, err := ecs.NewClientWithAccessKey(regionID, accessKeyID, accessKeySecret) if err != nil { - return nil, fmt.Errorf("failed to get Alibaba cloud client: %v", err) + return nil, fmt.Errorf("failed to get Alibaba cloud client: %w", err) } return client, nil } @@ -422,7 +421,7 @@ func getInstance(client *ecs.Client, instanceName string, uid string) (*ecs.Inst response, err := client.DescribeInstances(describeInstanceRequest) if err != nil { - return nil, fmt.Errorf("failed to describe instance with instanceName: %s: %v", instanceName, err) + return nil, fmt.Errorf("failed to describe instance with instanceName: %s: %w", instanceName, err) } if response.Instances.Instance == nil || @@ -437,12 +436,12 @@ func getInstance(client *ecs.Client, instanceName string, uid string) (*ecs.Inst func (p *provider) getImageIDForOS(machineSpec clusterv1alpha1.MachineSpec, os providerconfigtypes.OperatingSystem) (string, error) { c, _, err := p.getConfig(machineSpec.ProviderSpec) if err != nil { - return "", fmt.Errorf("failed to get alibaba client: %v", err) + return "", fmt.Errorf("failed to get alibaba client: %w", err) } client, err := getClient(c.RegionID, c.AccessKeyID, c.AccessKeySecret) if err != nil { - return "", fmt.Errorf("failed to get alibaba client: %v", err) + return "", fmt.Errorf("failed to get alibaba client: %w", err) } request := ecs.CreateDescribeImagesRequest() @@ -452,7 +451,7 @@ func (p *provider) getImageIDForOS(machineSpec clusterv1alpha1.MachineSpec, os p response, err := client.DescribeImages(request) if err != nil { - return "", fmt.Errorf("failed to describe alibaba images: %v", err) + return "", fmt.Errorf("failed to describe alibaba images: %w", err) } var availableImage = map[providerconfigtypes.OperatingSystem]string{} diff --git a/pkg/cloudprovider/provider/anexia/helper_test.go b/pkg/cloudprovider/provider/anexia/helper_test.go index adc1b514e..f15ecaea2 100644 --- a/pkg/cloudprovider/provider/anexia/helper_test.go +++ b/pkg/cloudprovider/provider/anexia/helper_test.go @@ -42,7 +42,6 @@ type ValidateCallTestCase struct { } func getSpecsForValidationTest(t *testing.T, configCases []ConfigTestCase) []ValidateCallTestCase { - var testCases []ValidateCallTestCase for _, configCase := range configCases { diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 22b8104b1..fca8ea5e4 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -299,7 +299,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*anxtypes.C c := anxtypes.Config{} c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, anxtypes.AnxTokenEnv) if err != nil { - return nil, nil, fmt.Errorf("failed to get 'token': %v", err) + return nil, nil, fmt.Errorf("failed to get 'token': %w", err) } c.CPUs = rawConfig.CPUs @@ -308,33 +308,33 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*anxtypes.C c.LocationID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.LocationID) if err != nil { - return nil, nil, fmt.Errorf("failed to get 'locationID': %v", err) + return nil, nil, fmt.Errorf("failed to get 'locationID': %w", err) } c.TemplateID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.TemplateID) if err != nil { - return nil, nil, fmt.Errorf("failed to get 'templateID': %v", err) + return nil, nil, fmt.Errorf("failed to get 'templateID': %w", err) } c.VlanID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VlanID) if err != nil { - return nil, nil, fmt.Errorf("failed to get 'vlanID': %v", err) + return nil, nil, fmt.Errorf("failed to get 'vlanID': %w", err) } return &c, pconfig, nil } -// New returns an Anexia provider +// New returns an Anexia provider. func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } -// AddDefaults adds omitted optional values to the given MachineSpec +// AddDefaults adds omitted optional values to the given MachineSpec. func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } -// Validate returns success or failure based according to its ProviderSpec +// Validate returns success or failure based according to its ProviderSpec. func (p *provider) Validate(machinespec clusterv1alpha1.MachineSpec) error { config, _, err := p.getConfig(machinespec.ProviderSpec) if err != nil { @@ -509,7 +509,7 @@ func newError(reason common.MachineStatusError, msg string, args ...interface{}) } // updateMachineStatus tries to update the machine status by any means -// an error will lead to a panic +// an error will lead to a panic. func updateMachineStatus(machine *clusterv1alpha1.Machine, status anxtypes.ProviderStatus, updater cloudprovidertypes.MachineUpdater) error { rawStatus, err := json.Marshal(status) if err != nil { diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index bb23fb5ce..5daaaf4af 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -75,7 +75,6 @@ func TestAnexiaProvider(t *testing.T) { err := waitForVM(ctx, client) if err != nil { t.Fatal("No error was expected", err) - } if providerStatus.InstanceID != TestIdentifier { @@ -308,7 +307,6 @@ func TestGetProviderStatus(t *testing.T) { returnedStatus := getProviderStatus(machine) testhelper.AssertEquals(t, "InstanceID", returnedStatus.InstanceID) - } func TestUpdateStatus(t *testing.T) { diff --git a/pkg/cloudprovider/provider/anexia/types/errors.go b/pkg/cloudprovider/provider/anexia/types/errors.go index 6e73d232f..28d3dd7bb 100644 --- a/pkg/cloudprovider/provider/anexia/types/errors.go +++ b/pkg/cloudprovider/provider/anexia/types/errors.go @@ -22,9 +22,9 @@ import ( ) // MultiError represent multiple errors at the same time. -type MultiError []error +type MultiErrors []error -func (r MultiError) Error() string { +func (r MultiErrors) Error() string { errString := make([]string, len(r)) for i, err := range r { errString[i] = fmt.Sprintf("Error %d: %s", i, err) @@ -42,7 +42,7 @@ func NewMultiError(errs ...error) error { } if len(combinedErr) > 0 { - return MultiError(combinedErr) + return MultiErrors(combinedErr) } return nil diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index b0be676ff..fc2592c13 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -76,7 +76,7 @@ type provider struct { configVarResolver *providerconfig.ConfigVarResolver } -// New returns a aws provider +// New returns a aws provider. func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -190,7 +190,7 @@ var ( } // cacheLock protects concurrent cache misses against a single key. This usually happens when multiple machines get created simultaneously - // We lock so the first access updates/writes the data to the cache and afterwards everyone reads the cached data + // We lock so the first access updates/writes the data to the cache and afterwards everyone reads the cached data. cacheLock = &sync.Mutex{} cache = gocache.New(5*time.Minute, 5*time.Minute) ) @@ -375,17 +375,17 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p rawConfig, err := awstypes.GetConfig(*pconfig) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to unmarshal: %v", err) + return nil, nil, nil, fmt.Errorf("failed to unmarshal: %w", err) } c := Config{} c.AccessKeyID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AccessKeyID, "AWS_ACCESS_KEY_ID") if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"accessKeyId\" field, error = %v", err) + return nil, nil, nil, fmt.Errorf("failed to get the value of \"accessKeyId\" field, error = %w", err) } c.SecretAccessKey, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.SecretAccessKey, "AWS_SECRET_ACCESS_KEY") if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"secretAccessKey\" field, error = %v", err) + return nil, nil, nil, fmt.Errorf("failed to get the value of \"secretAccessKey\" field, error = %w", err) } c.Region, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Region) if err != nil { @@ -451,7 +451,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c.EBSVolumeEncrypted, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.EBSVolumeEncrypted) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get ebsVolumeEncrypted value: %v", err) + return nil, nil, nil, fmt.Errorf("failed to get ebsVolumeEncrypted value: %w", err) } c.Tags = rawConfig.Tags c.AssignPublicIP = rawConfig.AssignPublicIP @@ -496,14 +496,14 @@ func getSession(id, secret, token, region, assumeRoleARN, assumeRoleExternalID s config = config.WithMaxRetries(maxRetries) awsSession, err := session.NewSession(config) if err != nil { - return nil, fmt.Errorf("failed to create AWS session: %v", err) + return nil, fmt.Errorf("failed to create AWS session: %w", err) } // Assume IAM role of e.g. external AWS account if configured if assumeRoleARN != "" { awsSession, err = getAssumeRoleSession(awsSession, assumeRoleARN, assumeRoleExternalID, region) if err != nil { - return nil, fmt.Errorf("failed to create temporary AWS session for assumed role: %v", err) + return nil, fmt.Errorf("failed to create temporary AWS session for assumed role: %w", err) } } @@ -575,7 +575,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { config, pc, _, err := p.getConfig(spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to parse config: %v", err) + return fmt.Errorf("failed to parse config: %w", err) } if _, osSupported := amiFilters[pc.OperatingSystem]; !osSupported { @@ -598,20 +598,20 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { - return fmt.Errorf("failed to create ec2 client: %v", err) + return fmt.Errorf("failed to create ec2 client: %w", err) } if config.AMI != "" { _, err := ec2Client.DescribeImages(&ec2.DescribeImagesInput{ ImageIds: aws.StringSlice([]string{config.AMI}), }) if err != nil { - return fmt.Errorf("failed to validate ami: %v", err) + return fmt.Errorf("failed to validate ami: %w", err) } } vpc, err := getVpc(ec2Client, config.VpcID) if err != nil { - return fmt.Errorf("invalid vpc %q specified: %v", config.VpcID, err) + return fmt.Errorf("invalid vpc %q specified: %w", config.VpcID, err) } switch f := pc.Network.GetIPFamily(); f { @@ -627,12 +627,12 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { _, err = ec2Client.DescribeAvailabilityZones(&ec2.DescribeAvailabilityZonesInput{ZoneNames: aws.StringSlice([]string{config.AvailabilityZone})}) if err != nil { - return fmt.Errorf("invalid zone %q specified: %v", config.AvailabilityZone, err) + return fmt.Errorf("invalid zone %q specified: %w", config.AvailabilityZone, err) } _, err = ec2Client.DescribeRegions(&ec2.DescribeRegionsInput{RegionNames: aws.StringSlice([]string{config.Region})}) if err != nil { - return fmt.Errorf("invalid region %q specified: %v", config.Region, err) + return fmt.Errorf("invalid region %q specified: %w", config.Region, err) } if len(config.SecurityGroupIDs) == 0 { @@ -642,19 +642,19 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { GroupIds: aws.StringSlice(config.SecurityGroupIDs), }) if err != nil { - return fmt.Errorf("failed to validate security group id's: %v", err) + return fmt.Errorf("failed to validate security group id's: %w", err) } iamClient, err := getIAMclient(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { - return fmt.Errorf("failed to create iam client: %v", err) + return fmt.Errorf("failed to create iam client: %w", err) } if config.InstanceProfile == "" { - return fmt.Errorf("invalid instance profile specified %q: %v", config.InstanceProfile, err) + return fmt.Errorf("invalid instance profile specified %q: %w", config.InstanceProfile, err) } if _, err := iamClient.GetInstanceProfile(&iam.GetInstanceProfileInput{InstanceProfileName: aws.String(config.InstanceProfile)}); err != nil { - return fmt.Errorf("failed to validate instance profile: %v", err) + return fmt.Errorf("failed to validate instance profile: %w", err) } if config.IsSpotInstance != nil && *config.IsSpotInstance { @@ -839,7 +839,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { ec2instance, err := p.get(machine) if err != nil { - if err == cloudprovidererrors.ErrInstanceNotFound { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return true, nil } return false, err @@ -862,7 +862,6 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertyp if config.IsSpotInstance != nil && *config.IsSpotInstance && config.SpotPersistentRequest != nil && *config.SpotPersistentRequest { - cOut, err := ec2Client.CancelSpotInstanceRequests(&ec2.CancelSpotInstanceRequestsInput{ SpotInstanceRequestIds: aws.StringSlice([]string{*ec2instance.instance.SpotInstanceRequestId}), }) @@ -944,7 +943,7 @@ func (p *provider) get(machine *clusterv1alpha1.Machine) (*awsInstance, error) { func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { c, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { - return "", "", fmt.Errorf("failed to parse config: %v", err) + return "", "", fmt.Errorf("failed to parse config: %w", err) } cc := &awstypes.CloudConfig{ @@ -957,11 +956,10 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri s, err := awstypes.CloudConfigToString(cc) if err != nil { - return "", "", fmt.Errorf("failed to convert cloud-config to string: %v", err) + return "", "", fmt.Errorf("failed to convert cloud-config to string: %w", err) } return s, "aws", nil - } func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { @@ -978,13 +976,13 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { machineInstance, err := p.get(machine) if err != nil { - if err == cloudprovidererrors.ErrInstanceNotFound { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return nil } - return fmt.Errorf("failed to get instance: %v", err) + return fmt.Errorf("failed to get instance: %w", err) } config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) @@ -997,14 +995,14 @@ func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) e ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { - return fmt.Errorf("failed to get EC2 client: %v", err) + return fmt.Errorf("failed to get EC2 client: %w", err) } _, err = ec2Client.CreateTags(&ec2.CreateTagsInput{ Resources: aws.StringSlice([]string{machineInstance.ID()}), - Tags: []*ec2.Tag{{Key: aws.String(machineUIDTag), Value: aws.String(string(new))}}}) + Tags: []*ec2.Tag{{Key: aws.String(machineUIDTag), Value: aws.String(string(newUID))}}}) if err != nil { - return fmt.Errorf("failed to update instance with new machineUIDTag: %v", err) + return fmt.Errorf("failed to update instance with new machineUIDTag: %w", err) } return nil @@ -1075,15 +1073,15 @@ func getTagValue(name string, tags []*ec2.Tag) string { // can be qualified as a "terminal" error, for more info see v1alpha1.MachineStatus // // if the given error doesn't qualify the error passed as -// an argument will be formatted according to msg and returned +// an argument will be formatted according to msg and returned. func awsErrorToTerminalError(err error, msg string) error { prepareAndReturnError := func() error { - return fmt.Errorf("%s, due to %s", msg, err) + return fmt.Errorf("%s, due to %w", msg, err) } if err != nil { - aerr, ok := err.(awserr.Error) - if !ok { + var aerr awserr.Error + if !errors.As(err, &aerr) { return prepareAndReturnError() } switch aerr.Code() { @@ -1156,7 +1154,7 @@ func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) e for _, machine := range machines.Items { config, _, _, err := p.getConfig(machines.Items[0].Spec.ProviderSpec) if err != nil { - machineErrors = append(machineErrors, fmt.Errorf("failed to parse MachineSpec of machine %s/%s, due to %v", machine.Namespace, machine.Name, err)) + machineErrors = append(machineErrors, fmt.Errorf("failed to parse MachineSpec of machine %s/%s, due to %w", machine.Namespace, machine.Name, err)) continue } @@ -1174,12 +1172,12 @@ func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) e for _, cred := range machineEc2Credentials { ec2Client, err := getEC2client(cred.acccessKeyID, cred.secretAccessKey, cred.region, cred.assumeRoleARN, cred.assumeRoleExternalID) if err != nil { - machineErrors = append(machineErrors, fmt.Errorf("failed to get EC2 client: %v", err)) + machineErrors = append(machineErrors, fmt.Errorf("failed to get EC2 client: %w", err)) continue } inOut, err := ec2Client.DescribeInstances(&ec2.DescribeInstancesInput{}) if err != nil { - machineErrors = append(machineErrors, fmt.Errorf("failed to get EC2 instances: %v", err)) + machineErrors = append(machineErrors, fmt.Errorf("failed to get EC2 instances: %w", err)) continue } allReservations = append(allReservations, inOut.Reservations...) @@ -1247,7 +1245,7 @@ func filterSupportedRHELImages(images []*ec2.Image) ([]*ec2.Image, error) { func (p *provider) waitForInstance(machine *clusterv1alpha1.Machine) error { return wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) { _, err := p.get(machine) - if err == cloudprovidererrors.ErrInstanceNotFound { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { // Retry if instance is not found return false, nil } else if err != nil { diff --git a/pkg/cloudprovider/provider/aws/types/cloudconfig.go b/pkg/cloudprovider/provider/aws/types/cloudconfig.go index 7343fa7a5..f70761532 100644 --- a/pkg/cloudprovider/provider/aws/types/cloudconfig.go +++ b/pkg/cloudprovider/provider/aws/types/cloudconfig.go @@ -63,12 +63,12 @@ func CloudConfigToString(c *CloudConfig) (string, error) { tpl, err := template.New("cloud-config").Funcs(funcMap).Parse(cloudConfigTpl) if err != nil { - return "", fmt.Errorf("failed to parse the cloud config template: %v", err) + return "", fmt.Errorf("failed to parse the cloud config template: %w", err) } buf := &bytes.Buffer{} if err := tpl.Execute(buf, c); err != nil { - return "", fmt.Errorf("failed to execute cloud config template: %v", err) + return "", fmt.Errorf("failed to execute cloud config template: %w", err) } return buf.String(), nil diff --git a/pkg/cloudprovider/provider/aws/types/types.go b/pkg/cloudprovider/provider/aws/types/types.go index 11974b9e3..ac03734da 100644 --- a/pkg/cloudprovider/provider/aws/types/types.go +++ b/pkg/cloudprovider/provider/aws/types/types.go @@ -53,7 +53,7 @@ type SpotInstanceConfig struct { InterruptionBehavior providerconfigtypes.ConfigVarString `json:"interruptionBehavior,omitempty"` } -// CPUArchitecture defines processor architectures returned by the AWS API +// CPUArchitecture defines processor architectures returned by the AWS API. type CPUArchitecture string const ( diff --git a/pkg/cloudprovider/provider/azure/create_delete_resources.go b/pkg/cloudprovider/provider/azure/create_delete_resources.go index 4bf719a17..90ddce7ea 100644 --- a/pkg/cloudprovider/provider/azure/create_delete_resources.go +++ b/pkg/cloudprovider/provider/azure/create_delete_resources.go @@ -37,7 +37,7 @@ import ( func deleteInterfacesByMachineUID(ctx context.Context, c *config, machineUID types.UID) error { ifClient, err := getInterfacesClient(c) if err != nil { - return fmt.Errorf("failed to create interfaces client: %v", err) + return fmt.Errorf("failed to create interfaces client: %w", err) } list, err := ifClient.List(ctx, c.ResourceGroup) @@ -50,7 +50,7 @@ func deleteInterfacesByMachineUID(ctx context.Context, c *config, machineUID typ for list.NotDone() { allInterfaces = append(allInterfaces, list.Values()...) if err = list.NextWithContext(ctx); err != nil { - return fmt.Errorf("failed to iterate the result list: %s", err) + return fmt.Errorf("failed to iterate the result list: %w", err) } } @@ -76,7 +76,7 @@ func deleteInterfacesByMachineUID(ctx context.Context, c *config, machineUID typ func deleteIPAddressesByMachineUID(ctx context.Context, c *config, machineUID types.UID) error { ipClient, err := getIPClient(c) if err != nil { - return fmt.Errorf("failed to create IP addresses client: %v", err) + return fmt.Errorf("failed to create IP addresses client: %w", err) } list, err := ipClient.List(ctx, c.ResourceGroup) @@ -89,7 +89,7 @@ func deleteIPAddressesByMachineUID(ctx context.Context, c *config, machineUID ty for list.NotDone() { allIPs = append(allIPs, list.Values()...) if err = list.Next(); err != nil { - return fmt.Errorf("failed to iterate the result list: %s", err) + return fmt.Errorf("failed to iterate the result list: %w", err) } } @@ -126,7 +126,7 @@ func deleteVMsByMachineUID(ctx context.Context, c *config, machineUID types.UID) for list.NotDone() { allServers = append(allServers, list.Values()...) if err = list.Next(); err != nil { - return fmt.Errorf("failed to iterate the result list: %s", err) + return fmt.Errorf("failed to iterate the result list: %w", err) } } @@ -149,7 +149,7 @@ func deleteVMsByMachineUID(ctx context.Context, c *config, machineUID types.UID) func deleteDisksByMachineUID(ctx context.Context, c *config, machineUID types.UID) error { disksClient, err := getDisksClient(c) if err != nil { - return fmt.Errorf("failed to get disks client: %v", err) + return fmt.Errorf("failed to get disks client: %w", err) } matchingDisks, err := getDisksByMachineUID(ctx, disksClient, c, machineUID) @@ -160,11 +160,11 @@ func deleteDisksByMachineUID(ctx context.Context, c *config, machineUID types.UI for _, disk := range matchingDisks { future, err := disksClient.Delete(ctx, c.ResourceGroup, *disk.Name) if err != nil { - return fmt.Errorf("failed to delete disk %s: %v", *disk.Name, err) + return fmt.Errorf("failed to delete disk %s: %w", *disk.Name, err) } if err = future.WaitForCompletionRef(ctx, disksClient.Client); err != nil { - return fmt.Errorf("failed to wait for deletion of disk %s: %v", *disk.Name, err) + return fmt.Errorf("failed to wait for deletion of disk %s: %w", *disk.Name, err) } } @@ -172,17 +172,16 @@ func deleteDisksByMachineUID(ctx context.Context, c *config, machineUID types.UI } func getDisksByMachineUID(ctx context.Context, disksClient *compute.DisksClient, c *config, UID types.UID) ([]compute.Disk, error) { - list, err := disksClient.List(ctx) if err != nil { - return nil, fmt.Errorf("failed to list disks: %v", err) + return nil, fmt.Errorf("failed to list disks: %w", err) } var allDisks, matchingDisks []compute.Disk for list.NotDone() { allDisks = append(allDisks, list.Values()...) if err = list.Next(); err != nil { - return nil, fmt.Errorf("failed to iterate the result list: %s", err) + return nil, fmt.Errorf("failed to iterate the result list: %w", err) } } @@ -218,22 +217,22 @@ func createOrUpdatePublicIPAddress(ctx context.Context, ipName string, ipVersion future, err := ipClient.CreateOrUpdate(ctx, c.ResourceGroup, ipName, ipParams) if err != nil { - return nil, fmt.Errorf("failed to create public IP address: %v", err) + return nil, fmt.Errorf("failed to create public IP address: %w", err) } err = future.WaitForCompletionRef(ctx, ipClient.Client) if err != nil { - return nil, fmt.Errorf("failed to retrieve public IP address creation result: %v", err) + return nil, fmt.Errorf("failed to retrieve public IP address creation result: %w", err) } if _, err = future.Result(*ipClient); err != nil { - return nil, fmt.Errorf("failed to create public IP address: %v", err) + return nil, fmt.Errorf("failed to create public IP address: %w", err) } klog.Infof("Fetching info for IP address %q", ipName) ip, err := getPublicIPAddress(ctx, ipName, c.ResourceGroup, ipClient) if err != nil { - return nil, fmt.Errorf("failed to fetch info about public IP %q: %v", ipName, err) + return nil, fmt.Errorf("failed to fetch info about public IP %q: %w", ipName, err) } return ip, nil @@ -251,7 +250,7 @@ func getPublicIPAddress(ctx context.Context, ipName string, resourceGroup string func getSubnet(ctx context.Context, c *config) (network.Subnet, error) { subnetsClient, err := getSubnetsClient(c) if err != nil { - return network.Subnet{}, fmt.Errorf("failed to create subnets client: %v", err) + return network.Subnet{}, fmt.Errorf("failed to create subnets client: %w", err) } return subnetsClient.Get(ctx, c.VNetResourceGroup, c.VNetName, c.SubnetName, "") @@ -282,14 +281,14 @@ func getSKU(ctx context.Context, c *config) (compute.ResourceSku, error) { for skuPages.NotDone() && sku == nil { skus := skuPages.Values() - for _, skuResult := range skus { + for i, skuResult := range skus { // skip invalid SKU results so we don't trigger a nil pointer exception if skuResult.ResourceType == nil || skuResult.Name == nil { continue } if *skuResult.ResourceType == "virtualMachines" && *skuResult.Name == c.VMSize { - sku = &skuResult + sku = &skus[i] break } } @@ -323,12 +322,12 @@ func getVirtualNetwork(ctx context.Context, c *config) (network.VirtualNetwork, func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineUID types.UID, config *config, publicIP, publicIPv6 *network.PublicIPAddress, ipFamily util.IPFamily) (*network.Interface, error) { ifClient, err := getInterfacesClient(config) if err != nil { - return nil, fmt.Errorf("failed to create interfaces client: %v", err) + return nil, fmt.Errorf("failed to create interfaces client: %w", err) } subnet, err := getSubnet(ctx, config) if err != nil { - return nil, fmt.Errorf("failed to fetch subnet: %v", err) + return nil, fmt.Errorf("failed to fetch subnet: %w", err) } ifSpec := network.Interface{ @@ -366,36 +365,36 @@ func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineU if config.SecurityGroupName != "" { authorizer, err := auth.NewClientCredentialsConfig(config.ClientID, config.ClientSecret, config.TenantID).Authorizer() if err != nil { - return nil, fmt.Errorf("failed to create authorizer for security groups: %v", err) + return nil, fmt.Errorf("failed to create authorizer for security groups: %w", err) } secGroupClient := network.NewSecurityGroupsClient(config.SubscriptionID) secGroupClient.Authorizer = authorizer secGroup, err := secGroupClient.Get(ctx, config.ResourceGroup, config.SecurityGroupName, "") if err != nil { - return nil, fmt.Errorf("failed to get securityGroup %q: %v", config.SecurityGroupName, err) + return nil, fmt.Errorf("failed to get securityGroup %q: %w", config.SecurityGroupName, err) } ifSpec.NetworkSecurityGroup = &secGroup } klog.Infof("Creating/Updating public network interface %q", ifName) future, err := ifClient.CreateOrUpdate(ctx, config.ResourceGroup, ifName, ifSpec) if err != nil { - return nil, fmt.Errorf("failed to create interface: %v", err) + return nil, fmt.Errorf("failed to create interface: %w", err) } err = future.WaitForCompletionRef(ctx, ifClient.Client) if err != nil { - return nil, fmt.Errorf("failed to get interface creation response: %v", err) + return nil, fmt.Errorf("failed to get interface creation response: %w", err) } _, err = future.Result(*ifClient) if err != nil { - return nil, fmt.Errorf("failed to get interface creation result: %v", err) + return nil, fmt.Errorf("failed to get interface creation result: %w", err) } klog.Infof("Fetching info about network interface %q", ifName) iface, err := ifClient.Get(ctx, config.ResourceGroup, ifName, "") if err != nil { - return nil, fmt.Errorf("failed to fetch info about interface %q: %v", ifName, err) + return nil, fmt.Errorf("failed to fetch info about interface %q: %w", ifName, err) } return &iface, nil diff --git a/pkg/cloudprovider/provider/azure/get_client.go b/pkg/cloudprovider/provider/azure/get_client.go index 80de3b35f..a4ee34021 100644 --- a/pkg/cloudprovider/provider/azure/get_client.go +++ b/pkg/cloudprovider/provider/azure/get_client.go @@ -29,7 +29,7 @@ func getIPClient(c *config) (*network.PublicIPAddressesClient, error) { ipClient := network.NewPublicIPAddressesClient(c.SubscriptionID) ipClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() if err != nil { - return nil, fmt.Errorf("failed to create authorizer: %v", err) + return nil, fmt.Errorf("failed to create authorizer: %w", err) } return &ipClient, nil @@ -40,7 +40,7 @@ func getIPConfigClient(c *config) (*network.InterfaceIPConfigurationsClient, err ipConfigClient := network.NewInterfaceIPConfigurationsClient(c.SubscriptionID) ipConfigClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() if err != nil { - return nil, fmt.Errorf("failed to create authorizer: %s", err.Error()) + return nil, fmt.Errorf("failed to create authorizer: %w", err) } return &ipConfigClient, nil @@ -51,7 +51,7 @@ func getSubnetsClient(c *config) (*network.SubnetsClient, error) { subnetClient := network.NewSubnetsClient(c.SubscriptionID) subnetClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() if err != nil { - return nil, fmt.Errorf("failed to create authorizer: %s", err.Error()) + return nil, fmt.Errorf("failed to create authorizer: %w", err) } return &subnetClient, nil @@ -62,7 +62,7 @@ func getVirtualNetworksClient(c *config) (*network.VirtualNetworksClient, error) virtualNetworksClient := network.NewVirtualNetworksClient(c.SubscriptionID) virtualNetworksClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() if err != nil { - return nil, fmt.Errorf("failed to create authorizer: %v", err) + return nil, fmt.Errorf("failed to create authorizer: %w", err) } return &virtualNetworksClient, nil } @@ -72,7 +72,7 @@ func getVMClient(c *config) (*compute.VirtualMachinesClient, error) { vmClient := compute.NewVirtualMachinesClient(c.SubscriptionID) vmClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() if err != nil { - return nil, fmt.Errorf("failed to create authorizer: %s", err.Error()) + return nil, fmt.Errorf("failed to create authorizer: %w", err) } return &vmClient, nil @@ -94,7 +94,7 @@ func getInterfacesClient(c *config) (*network.InterfacesClient, error) { ifClient := network.NewInterfacesClient(c.SubscriptionID) ifClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() if err != nil { - return nil, fmt.Errorf("failed to create authorizer: %s", err.Error()) + return nil, fmt.Errorf("failed to create authorizer: %w", err) } return &ifClient, err @@ -105,7 +105,7 @@ func getDisksClient(c *config) (*compute.DisksClient, error) { disksClient := compute.NewDisksClient(c.SubscriptionID) disksClient.Authorizer, err = auth.NewClientCredentialsConfig(c.ClientID, c.ClientSecret, c.TenantID).Authorizer() if err != nil { - return nil, fmt.Errorf("failed to create authorizer: %s", err.Error()) + return nil, fmt.Errorf("failed to create authorizer: %w", err) } return &disksClient, err diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index c002224b7..beaf5a9dc 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -192,7 +192,7 @@ var dataDiskSKUs = map[compute.StorageAccountTypes]string{ var ( // cacheLock protects concurrent cache misses against a single key. This usually happens when multiple machines get created simultaneously - // We lock so the first access updates/writes the data to the cache and afterwards everyone reads the cached data + // We lock so the first access updates/writes the data to the cache and afterwards everyone reads the cached data. cacheLock = &sync.Mutex{} cache = gocache.New(10*time.Minute, 10*time.Minute) ) @@ -221,7 +221,7 @@ func getOSImageReference(c *config, os providerconfigtypes.OperatingSystem) (*co return &ref, nil } -// New returns a new azure provider +// New returns a new azure provider. func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -248,32 +248,32 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p c := config{} c.SubscriptionID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawCfg.SubscriptionID, envSubscriptionID) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"subscriptionID\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"subscriptionID\" field, error = %w", err) } c.TenantID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawCfg.TenantID, envTenantID) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"tenantID\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"tenantID\" field, error = %w", err) } c.ClientID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawCfg.ClientID, envClientID) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"clientID\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"clientID\" field, error = %w", err) } c.ClientSecret, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawCfg.ClientSecret, envClientSecret) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"clientSecret\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"clientSecret\" field, error = %w", err) } c.ResourceGroup, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.ResourceGroup) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"resourceGroup\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"resourceGroup\" field, error = %w", err) } c.VNetResourceGroup, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.VNetResourceGroup) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"vnetResourceGroup\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"vnetResourceGroup\" field, error = %w", err) } if c.VNetResourceGroup == "" { @@ -282,49 +282,49 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p c.Location, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.Location) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"location\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"location\" field, error = %w", err) } c.VMSize, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.VMSize) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"vmSize\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"vmSize\" field, error = %w", err) } c.VNetName, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.VNetName) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"vnetName\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"vnetName\" field, error = %w", err) } c.SubnetName, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.SubnetName) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"subnetName\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"subnetName\" field, error = %w", err) } c.LoadBalancerSku, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.LoadBalancerSku) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"loadBalancerSku\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"loadBalancerSku\" field, error = %w", err) } c.RouteTableName, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.RouteTableName) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"routeTableName\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"routeTableName\" field, error = %w", err) } c.AssignPublicIP, _, err = p.configVarResolver.GetConfigVarBoolValue(rawCfg.AssignPublicIP) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"assignPublicIP\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"assignPublicIP\" field, error = %w", err) } c.AssignAvailabilitySet = rawCfg.AssignAvailabilitySet c.AvailabilitySet, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.AvailabilitySet) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"availabilitySet\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"availabilitySet\" field, error = %w", err) } c.SecurityGroupName, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.SecurityGroupName) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"securityGroupName\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"securityGroupName\" field, error = %w", err) } c.Zones = rawCfg.Zones @@ -359,7 +359,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p c.ImageID, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.ImageID) if err != nil { - return nil, nil, fmt.Errorf("failed to get image id: %v", err) + return nil, nil, fmt.Errorf("failed to get image id: %w", err) } return &c, pconfig, nil @@ -392,7 +392,7 @@ func getVMIPAddresses(ctx context.Context, c *config, vm *compute.VirtualMachine ifaceName := splitIfaceID[len(splitIfaceID)-1] ipAddresses, err = getNICIPAddresses(ctx, c, ifaceName) if vm.NetworkProfile.NetworkInterfaces == nil { - return nil, fmt.Errorf("failed to get addresses for interface %q: %v", ifaceName, err) + return nil, fmt.Errorf("failed to get addresses for interface %q: %w", ifaceName, err) } } @@ -402,12 +402,12 @@ func getVMIPAddresses(ctx context.Context, c *config, vm *compute.VirtualMachine func getNICIPAddresses(ctx context.Context, c *config, ifaceName string) (map[string]v1.NodeAddressType, error) { ifClient, err := getInterfacesClient(c) if err != nil { - return nil, fmt.Errorf("failed to create interfaces client: %v", err) + return nil, fmt.Errorf("failed to create interfaces client: %w", err) } netIf, err := ifClient.Get(ctx, c.ResourceGroup, ifaceName, "") if err != nil { - return nil, fmt.Errorf("failed to get interface %q: %v", ifaceName, err.Error()) + return nil, fmt.Errorf("failed to get interface %q: %w", ifaceName, err) } ipAddresses := map[string]v1.NodeAddressType{} @@ -432,7 +432,7 @@ func getNICIPAddresses(ctx context.Context, c *config, ifaceName string) (map[st if c.AssignPublicIP { publicIPs, err := getIPAddressStrings(ctx, c, publicIPName(ifaceName)) if err != nil { - return nil, fmt.Errorf("failed to retrieve IP string for IP %q: %v", name, err) + return nil, fmt.Errorf("failed to retrieve IP string for IP %q: %w", name, err) } for _, ip := range publicIPs { ipAddresses[ip] = v1.NodeExternalIP @@ -440,36 +440,33 @@ func getNICIPAddresses(ctx context.Context, c *config, ifaceName string) (map[st publicIP6s, err := getIPAddressStrings(ctx, c, publicIPv6Name(ifaceName)) if err != nil { - return nil, fmt.Errorf("failed to retrieve IP string for IP %q: %v", name, err) + return nil, fmt.Errorf("failed to retrieve IP string for IP %q: %w", name, err) } for _, ip := range publicIP6s { ipAddresses[ip] = v1.NodeExternalIP } - } internalIPs, err := getInternalIPAddresses(ctx, c, ifaceName, name) if err != nil { - return nil, fmt.Errorf("failed to retrieve internal IP string for IP %q: %v", name, err) + return nil, fmt.Errorf("failed to retrieve internal IP string for IP %q: %w", name, err) } for _, ip := range internalIPs { ipAddresses[ip] = v1.NodeInternalIP } - } - return ipAddresses, nil } func getIPAddressStrings(ctx context.Context, c *config, addrName string) ([]string, error) { ipClient, err := getIPClient(c) if err != nil { - return nil, fmt.Errorf("failed to create IP address client: %v", err) + return nil, fmt.Errorf("failed to create IP address client: %w", err) } ip, err := ipClient.Get(ctx, c.ResourceGroup, addrName, "") if err != nil { - return nil, fmt.Errorf("failed to get IP %q: %v", addrName, err) + return nil, fmt.Errorf("failed to get IP %q: %w", addrName, err) } if ip.IPConfiguration == nil { @@ -488,12 +485,12 @@ func getInternalIPAddresses(ctx context.Context, c *config, inetface, ipconfigNa var ipAddresses []string ipConfigClient, err := getIPConfigClient(c) if err != nil { - return nil, fmt.Errorf("failed to create IP config client: %v", err) + return nil, fmt.Errorf("failed to create IP config client: %w", err) } internalIP, err := ipConfigClient.Get(ctx, c.ResourceGroup, inetface, ipconfigName) if err != nil { - return nil, fmt.Errorf("failed to get IP config %q: %v", inetface, err) + return nil, fmt.Errorf("failed to get IP config %q: %w", inetface, err) } if internalIP.ID == nil { @@ -513,7 +510,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) (*compute.StorageProfile, error) { osRef, err := getOSImageReference(config, providerCfg.OperatingSystem) if err != nil { - return nil, fmt.Errorf("failed to get OSImageReference: %v", err) + return nil, fmt.Errorf("failed to get OSImageReference: %w", err) } // initial default storage profile, this will use the VMSize default storage profile sp := &compute.StorageProfile{ @@ -547,7 +544,6 @@ func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) StorageAccountType: *config.DataDiskSKU, } } - } return sp, nil } @@ -563,13 +559,13 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert vmClient, err := getVMClient(config) if err != nil { - return nil, fmt.Errorf("failed to create VM client: %v", err) + return nil, fmt.Errorf("failed to create VM client: %w", err) } // We genete a random SSH key, since Azure won't let us create a VM without an SSH key or a password key, err := ssh.NewKey() if err != nil { - return nil, fmt.Errorf("failed to generate ssh key: %v", err) + return nil, fmt.Errorf("failed to generate ssh key: %w", err) } ipFamily := providerCfg.Network.GetIPFamily() @@ -595,13 +591,13 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert } publicIP, err = createOrUpdatePublicIPAddress(context.TODO(), publicIPName(ifaceName(machine)), network.IPVersionIPv4, sku, network.IPAllocationMethodStatic, machine.UID, config) if err != nil { - return nil, fmt.Errorf("failed to create public IP: %v", err) + return nil, fmt.Errorf("failed to create public IP: %w", err) } if ipFamily == util.DualStack { publicIPv6, err = createOrUpdatePublicIPAddress(context.TODO(), publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodStatic, machine.UID, config) if err != nil { - return nil, fmt.Errorf("failed to create public IP: %v", err) + return nil, fmt.Errorf("failed to create public IP: %w", err) } } } @@ -616,7 +612,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert iface, err := createOrUpdateNetworkInterface(context.TODO(), ifaceName(machine), machine.UID, config, publicIP, publicIPv6, ipFamily) if err != nil { - return nil, fmt.Errorf("failed to generate main network interface: %v", err) + return nil, fmt.Errorf("failed to generate main network interface: %w", err) } tags := make(map[string]*string, len(config.Tags)+1) @@ -633,7 +629,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert adminUserName := getOSUsername(providerCfg.OperatingSystem) storageProfile, err := getStorageProfile(config, providerCfg) if err != nil { - return nil, fmt.Errorf("failed to get StorageProfile: %v", err) + return nil, fmt.Errorf("failed to get StorageProfile: %w", err) } vmSpec := compute.VirtualMachine{ @@ -692,33 +688,33 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert future, err := vmClient.CreateOrUpdate(context.TODO(), config.ResourceGroup, machine.Name, vmSpec) if err != nil { - return nil, fmt.Errorf("trying to create a VM: %v", err) + return nil, fmt.Errorf("trying to create a VM: %w", err) } err = future.WaitForCompletionRef(context.TODO(), vmClient.Client) if err != nil { - return nil, fmt.Errorf("waiting for operation returned: %v", err.Error()) + return nil, fmt.Errorf("waiting for operation returned: %w", err) } vm, err := future.Result(*vmClient) if err != nil { - return nil, fmt.Errorf("decoding result: %v", err.Error()) + return nil, fmt.Errorf("decoding result: %w", err) } // get the actual VM object filled in with additional data vm, err = vmClient.Get(context.TODO(), config.ResourceGroup, machine.Name, "") if err != nil { - return nil, fmt.Errorf("failed to retrieve updated data for VM %q: %v", machine.Name, err) + return nil, fmt.Errorf("failed to retrieve updated data for VM %q: %w", machine.Name, err) } ipAddresses, err := getVMIPAddresses(context.TODO(), config, &vm) if err != nil { - return nil, fmt.Errorf("failed to retrieve IP addresses for VM %q: %v", machine.Name, err.Error()) + return nil, fmt.Errorf("failed to retrieve IP addresses for VM %q: %w", machine.Name, err) } status, err := getVMStatus(context.TODO(), config, machine.Name) if err != nil { - return nil, fmt.Errorf("failed to retrieve status for VM %q: %v", machine.Name, err.Error()) + return nil, fmt.Errorf("failed to retrieve status for VM %q: %w", machine.Name, err) } return &azureVM{vm: &vm, ipAddresses: ipAddresses, status: status}, nil @@ -727,13 +723,13 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { - return false, fmt.Errorf("failed to parse MachineSpec: %v", err) + return false, fmt.Errorf("failed to parse MachineSpec: %w", err) } if kuberneteshelper.HasFinalizer(machine, finalizerVM) { klog.Infof("deleting VM %q", machine.Name) if err = deleteVMsByMachineUID(context.TODO(), config, machine.UID); err != nil { - return false, fmt.Errorf("failed to delete instance for machine %q: %v", machine.Name, err) + return false, fmt.Errorf("failed to delete instance for machine %q: %w", machine.Name, err) } if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { @@ -746,7 +742,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider if kuberneteshelper.HasFinalizer(machine, finalizerDisks) { klog.Infof("deleting disks of VM %q", machine.Name) if err := deleteDisksByMachineUID(context.TODO(), config, machine.UID); err != nil { - return false, fmt.Errorf("failed to remove disks of machine %q: %v", machine.Name, err) + return false, fmt.Errorf("failed to remove disks of machine %q: %w", machine.Name, err) } if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerDisks) @@ -758,7 +754,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider if kuberneteshelper.HasFinalizer(machine, finalizerNIC) { klog.Infof("deleting network interfaces of VM %q", machine.Name) if err := deleteInterfacesByMachineUID(context.TODO(), config, machine.UID); err != nil { - return false, fmt.Errorf("failed to remove network interfaces of machine %q: %v", machine.Name, err) + return false, fmt.Errorf("failed to remove network interfaces of machine %q: %w", machine.Name, err) } if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerNIC) @@ -770,7 +766,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider if kuberneteshelper.HasFinalizer(machine, finalizerPublicIP) { klog.Infof("deleting public IP addresses of VM %q", machine.Name) if err := deleteIPAddressesByMachineUID(context.TODO(), config, machine.UID); err != nil { - return false, fmt.Errorf("failed to remove public IP addresses of machine %q: %v", machine.Name, err) + return false, fmt.Errorf("failed to remove public IP addresses of machine %q: %w", machine.Name, err) } if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizerPublicIP) @@ -798,7 +794,7 @@ func getVMByUID(ctx context.Context, c *config, uid types.UID) (*compute.Virtual for list.NotDone() { allServers = append(allServers, list.Values()...) if err := list.Next(); err != nil { - return nil, fmt.Errorf("failed to iterate the result list: %s", err) + return nil, fmt.Errorf("failed to iterate the result list: %w", err) } } @@ -819,7 +815,7 @@ func getVMStatus(ctx context.Context, c *config, vmName string) (instance.Status iv, err := vmClient.InstanceView(ctx, c.ResourceGroup, vmName) if err != nil { - return instance.StatusUnknown, fmt.Errorf("failed to get instance view for machine %q: %v", vmName, err) + return instance.StatusUnknown, fmt.Errorf("failed to get instance view for machine %q: %w", vmName, err) } if iv.Statuses == nil { @@ -873,26 +869,26 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P func (p *provider) get(machine *clusterv1alpha1.Machine) (*azureVM, error) { config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { - return nil, fmt.Errorf("failed to parse MachineSpec: %v", err) + return nil, fmt.Errorf("failed to parse MachineSpec: %w", err) } vm, err := getVMByUID(context.TODO(), config, machine.UID) if err != nil { - if err == cloudprovidererrors.ErrInstanceNotFound { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return nil, cloudprovidererrors.ErrInstanceNotFound } - return nil, fmt.Errorf("failed to find machine %q by its UID: %v", machine.UID, err) + return nil, fmt.Errorf("failed to find machine %q by its UID: %w", machine.UID, err) } ipAddresses, err := getVMIPAddresses(context.TODO(), config, vm) if err != nil { - return nil, fmt.Errorf("failed to retrieve IP addresses for VM %v: %v", vm.Name, err) + return nil, fmt.Errorf("failed to retrieve IP addresses for VM %v: %w", vm.Name, err) } status, err := getVMStatus(context.TODO(), config, machine.Name) if err != nil { - return nil, fmt.Errorf("failed to retrieve status for VM %v: %v", vm.Name, err) + return nil, fmt.Errorf("failed to retrieve status for VM %v: %w", vm.Name, err) } return &azureVM{vm: vm, ipAddresses: ipAddresses, status: status}, nil @@ -901,7 +897,7 @@ func (p *provider) get(machine *clusterv1alpha1.Machine) (*azureVM, error) { func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { c, _, err := p.getConfig(spec.ProviderSpec) if err != nil { - return "", "", fmt.Errorf("failed to parse config: %v", err) + return "", "", fmt.Errorf("failed to parse config: %w", err) } var avSet string @@ -930,7 +926,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri s, err := azuretypes.CloudConfigToString(cc) if err != nil { - return "", "", fmt.Errorf("failed to convert cloud-config to string: %v", err) + return "", "", fmt.Errorf("failed to convert cloud-config to string: %w", err) } return s, "azure", nil @@ -976,7 +972,7 @@ func validateDiskSKUs(c *config) error { func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, providerConfig, err := p.getConfig(spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to parse config: %v", err) + return fmt.Errorf("failed to parse config: %w", err) } if c.SubscriptionID == "" { @@ -1024,20 +1020,20 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { vmClient, err := getVMClient(c) if err != nil { - return fmt.Errorf("failed to (create) vm client: %v", err.Error()) + return fmt.Errorf("failed to (create) vm client: %w", err) } _, err = vmClient.List(context.TODO(), c.ResourceGroup, "") if err != nil { - return fmt.Errorf("failed to list virtual machines: %v", err.Error()) + return fmt.Errorf("failed to list virtual machines: %w", err) } if _, err := getVirtualNetwork(context.TODO(), c); err != nil { - return fmt.Errorf("failed to get virtual network: %v", err) + return fmt.Errorf("failed to get virtual network: %w", err) } if _, err := getSubnet(context.TODO(), c); err != nil { - return fmt.Errorf("failed to get subnet: %v", err) + return fmt.Errorf("failed to get subnet: %w", err) } if err := validateDiskSKUs(c); err != nil { @@ -1074,7 +1070,7 @@ func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID vmClient, err := getVMClient(config) if err != nil { - return fmt.Errorf("failed to create VM client: %v", err) + return fmt.Errorf("failed to create VM client: %w", err) } var publicIP, publicIPv6 *network.PublicIPAddress @@ -1084,43 +1080,43 @@ func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID sku = network.PublicIPAddressSkuNameStandard _, err = createOrUpdatePublicIPAddress(ctx, publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodDynamic, newUID, config) if err != nil { - return fmt.Errorf("failed to update UID on public IP: %v", err) + return fmt.Errorf("failed to update UID on public IP: %w", err) } } if kuberneteshelper.HasFinalizer(machine, finalizerPublicIP) { _, err = createOrUpdatePublicIPAddress(ctx, publicIPName(ifaceName(machine)), network.IPVersionIPv4, sku, network.IPAllocationMethodStatic, newUID, config) if err != nil { - return fmt.Errorf("failed to update UID on public IP: %v", err) + return fmt.Errorf("failed to update UID on public IP: %w", err) } } if kuberneteshelper.HasFinalizer(machine, finalizerNIC) { _, err = createOrUpdateNetworkInterface(ctx, ifaceName(machine), newUID, config, publicIP, publicIPv6, util.Unspecified) if err != nil { - return fmt.Errorf("failed to update UID on main network interface: %v", err) + return fmt.Errorf("failed to update UID on main network interface: %w", err) } } if kuberneteshelper.HasFinalizer(machine, finalizerDisks) { disksClient, err := getDisksClient(config) if err != nil { - return fmt.Errorf("failed to get disks client: %v", err) + return fmt.Errorf("failed to get disks client: %w", err) } disks, err := getDisksByMachineUID(ctx, disksClient, config, machine.UID) if err != nil { - return fmt.Errorf("failed to get disks: %v", err) + return fmt.Errorf("failed to get disks: %w", err) } for _, disk := range disks { disk.Tags[machineUIDTag] = to.StringPtr(string(newUID)) future, err := disksClient.CreateOrUpdate(ctx, config.ResourceGroup, *disk.Name, disk) if err != nil { - return fmt.Errorf("failed to update UID for disk %s: %v", *disk.Name, err) + return fmt.Errorf("failed to update UID for disk %s: %w", *disk.Name, err) } if err := future.WaitForCompletionRef(ctx, disksClient.Client); err != nil { - return fmt.Errorf("failed waiting for completion of update UID operation for disk %s: %v", *disk.Name, err) + return fmt.Errorf("failed waiting for completion of update UID operation for disk %s: %w", *disk.Name, err) } } } @@ -1134,11 +1130,11 @@ func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID vmSpec := compute.VirtualMachine{Location: &config.Location, Tags: tags} future, err := vmClient.CreateOrUpdate(ctx, config.ResourceGroup, machine.Name, vmSpec) if err != nil { - return fmt.Errorf("failed to update UID of the instance: %v", err) + return fmt.Errorf("failed to update UID of the instance: %w", err) } if err := future.WaitForCompletionRef(ctx, vmClient.Client); err != nil { - return fmt.Errorf("error waiting for instance to have the updated UID: %v", err) + return fmt.Errorf("error waiting for instance to have the updated UID: %w", err) } return nil diff --git a/pkg/cloudprovider/provider/azure/types/cloudconfig.go b/pkg/cloudprovider/provider/azure/types/cloudconfig.go index 0c7484187..6ddb8b5ca 100644 --- a/pkg/cloudprovider/provider/azure/types/cloudconfig.go +++ b/pkg/cloudprovider/provider/azure/types/cloudconfig.go @@ -43,7 +43,7 @@ type CloudConfig struct { func CloudConfigToString(c *CloudConfig) (string, error) { b, err := json.Marshal(c) if err != nil { - return "", fmt.Errorf("failed to unmarshal config: %v", err) + return "", fmt.Errorf("failed to unmarshal config: %w", err) } return string(b), nil diff --git a/pkg/cloudprovider/provider/azure/types/types.go b/pkg/cloudprovider/provider/azure/types/types.go index 10f881d28..de6de2166 100644 --- a/pkg/cloudprovider/provider/azure/types/types.go +++ b/pkg/cloudprovider/provider/azure/types/types.go @@ -21,7 +21,7 @@ import ( providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) -// RawConfig is a direct representation of an Azure machine object's configuration +// RawConfig is a direct representation of an Azure machine object's configuration. type RawConfig struct { SubscriptionID providerconfigtypes.ConfigVarString `json:"subscriptionID,omitempty"` TenantID providerconfigtypes.ConfigVarString `json:"tenantID,omitempty"` @@ -52,7 +52,7 @@ type RawConfig struct { Tags map[string]string `json:"tags,omitempty"` } -// ImagePlan contains azure OS Plan fields for the marketplace images +// ImagePlan contains azure OS Plan fields for the marketplace images. type ImagePlan struct { Name string `json:"name,omitempty"` Publisher string `json:"publisher,omitempty"` diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go index c7234b36d..356593248 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go @@ -67,17 +67,16 @@ func NewTinkerbellDriver(mdConfig *metadataclient.Config, factory ClientFactory, if factory == nil { mdClient, err = metadataclient.NewMetadataClient(mdConfig) if err != nil { - return nil, fmt.Errorf("failed to create metadata client: %v", err) + return nil, fmt.Errorf("failed to create metadata client: %w", err) } if err := tinkclient.Setup(); err != nil { - return nil, fmt.Errorf("failed to setup tink-server client: %v", err) + return nil, fmt.Errorf("failed to setup tink-server client: %w", err) } hwClient = tinkerbellclient.NewHardwareClient(tinkclient.HardwareClient) tmplClient = tinkerbellclient.NewTemplateClient(tinkclient.TemplateClient) wflClient = tinkerbellclient.NewWorkflowClient(tinkclient.WorkflowClient, tinkerbellclient.NewHardwareClient(tinkclient.HardwareClient)) - } else { mdClient, hwClient, tmplClient, wflClient = factory() } @@ -97,7 +96,7 @@ func NewTinkerbellDriver(mdConfig *metadataclient.Config, factory ClientFactory, func (d *driver) GetServer(ctx context.Context, uid types.UID, hwSpec runtime.RawExtension) (plugins.Server, error) { hw := HardwareSpec{} if err := json.Unmarshal(hwSpec.Raw, &hw); err != nil { - return nil, fmt.Errorf("failed to unmarshal tinkerbell hardware spec: %v", err) + return nil, fmt.Errorf("failed to unmarshal tinkerbell hardware spec: %w", err) } fetchedHW, err := d.hardwareClient.Get(ctx, string(uid), hw.GetIPAddress(), @@ -107,7 +106,7 @@ func (d *driver) GetServer(ctx context.Context, uid types.UID, hwSpec runtime.Ra return nil, cloudprovidererrors.ErrInstanceNotFound } - return nil, fmt.Errorf("failed to get hardware: %v", err) + return nil, fmt.Errorf("failed to get hardware: %w", err) } return &HardwareSpec{ @@ -120,7 +119,7 @@ func (d *driver) GetServer(ctx context.Context, uid types.UID, hwSpec runtime.Ra func (d *driver) ProvisionServer(ctx context.Context, uid types.UID, cfg *plugins.CloudConfigSettings, hwSpec runtime.RawExtension) (plugins.Server, error) { hw := HardwareSpec{} if err := json.Unmarshal(hwSpec.Raw, &hw); err != nil { - return nil, fmt.Errorf("failed to unmarshal tinkerbell hardware spec: %v", err) + return nil, fmt.Errorf("failed to unmarshal tinkerbell hardware spec: %w", err) } hw.Hardware.Id = string(uid) _, err := d.hardwareClient.Get(ctx, hw.Hardware.Id, "", "") @@ -128,14 +127,14 @@ func (d *driver) ProvisionServer(ctx context.Context, uid types.UID, cfg *plugin if resourceNotFoundErr(err) { cfg, err := d.metadataClient.GetMachineMetadata() if err != nil { - return nil, fmt.Errorf("failed to get metadata configs: %v", err) + return nil, fmt.Errorf("failed to get metadata configs: %w", err) } hw.Hardware.Network.Interfaces[0].Dhcp.Mac = cfg.MACAddress ip, netmask, _, err := util.CIDRToIPAndNetMask(cfg.CIDR) if err != nil { - return nil, fmt.Errorf("failed to parse CIDR: %v", err) + return nil, fmt.Errorf("failed to parse CIDR: %w", err) } dhcpIP := &hardware.Hardware_DHCP_IP{ Address: ip, @@ -145,7 +144,7 @@ func (d *driver) ProvisionServer(ctx context.Context, uid types.UID, cfg *plugin hw.Hardware.Network.Interfaces[0].Dhcp.Ip = dhcpIP if err := d.hardwareClient.Create(ctx, hw.Hardware.Hardware); err != nil { - return nil, fmt.Errorf("failed to register hardware to tink-server: %v", err) + return nil, fmt.Errorf("failed to register hardware to tink-server: %w", err) } } } @@ -157,7 +156,7 @@ func (d *driver) ProvisionServer(ctx context.Context, uid types.UID, cfg *plugin tmpl := createTemplate(d.TinkServerAddress, d.ImageRepoAddress, cfg) payload, err := yaml.Marshal(tmpl) if err != nil { - return nil, fmt.Errorf("failed marshalling workflow template: %v", err) + return nil, fmt.Errorf("failed marshalling workflow template: %w", err) } workflowTemplate = &tinktmpl.WorkflowTemplate{ @@ -167,13 +166,13 @@ func (d *driver) ProvisionServer(ctx context.Context, uid types.UID, cfg *plugin } if err := d.templateClient.Create(ctx, workflowTemplate); err != nil { - return nil, fmt.Errorf("failed to create workflow template: %v", err) + return nil, fmt.Errorf("failed to create workflow template: %w", err) } } } if _, err := d.workflowClient.Create(ctx, workflowTemplate.Id, hw.GetID()); err != nil { - return nil, fmt.Errorf("failed to provisioing server id %s running template id %s: %v", workflowTemplate.Id, hw.GetID(), err) + return nil, fmt.Errorf("failed to provisioing server id %s running template id %s: %w", workflowTemplate.Id, hw.GetID(), err) } return &hw, nil @@ -182,7 +181,7 @@ func (d *driver) ProvisionServer(ctx context.Context, uid types.UID, cfg *plugin func (d *driver) Validate(hwSpec runtime.RawExtension) error { hw := HardwareSpec{} if err := json.Unmarshal(hwSpec.Raw, &hw); err != nil { - return fmt.Errorf("failed to unmarshal tinkerbell hardware spec: %v", err) + return fmt.Errorf("failed to unmarshal tinkerbell hardware spec: %w", err) } if hw.Hardware.Hardware == nil { @@ -205,7 +204,7 @@ func (d *driver) DeprovisionServer(ctx context.Context, uid types.UID) error { if resourceNotFoundErr(err) { return nil } - return fmt.Errorf("failed to delete tinkerbell hardware data: %v", err) + return fmt.Errorf("failed to delete tinkerbell hardware data: %w", err) } return nil diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver_test.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver_test.go index f405baf0f..229dd6817 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver_test.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver_test.go @@ -99,7 +99,7 @@ func TestDriver_GetServer(t *testing.T) { hardwareSpec: runtime.RawExtension{Raw: []byte("{\n \"hardware\": {\n \"network\": {\n \"interfaces\": [\n {\n \"dhcp\": {\n \"ip\": {\n \"address\": \"10.129.8.90\"\n },\n \"mac\": \"18:C0:4D:B1:18:E3\"\n }\n }\n ]\n }\n }\n}")}, clientFactor: func() (metadata.Client, tinkerbellclient.HardwareClient, tinkerbellclient.TemplateClient, tinkerbellclient.WorkflowClient) { return &fakeMetadataClient{}, &fakeHardwareClient{ - err: &resourceErr{ + err: &resourceError{ resource: "hardware", }, }, &fakeTemplateClient{}, &fakeWorkflowClient{} @@ -130,7 +130,7 @@ func TestDriver_GetServer(t *testing.T) { ctx := context.Background() s, err := d.GetServer(ctx, "0eba0bf8-3772-4b4a-ab9f-6ebe93b90a94", test.hardwareSpec) if err != nil { - if test.errorIsExpected && test.expectedError == err { + if test.errorIsExpected && errors.Is(err, test.expectedError) { return } @@ -168,7 +168,7 @@ func TestDriver_ProvisionServer(t *testing.T) { hardwareSpec: runtime.RawExtension{Raw: []byte("{\n \"hardware\": {\n \"metadata\": {\n \"facility\": {\n \"facility_code\": \"ewr1\",\n \"plan_slug\": \"c2.medium.x86\",\n \"plan_version_slug\": \"\"\n },\n \"instance\": {\n \"operating_system_version\": {\n \"distro\": \"ubuntu\",\n \"os_slug\": \"ubuntu_18_04\",\n \"version\": \"18.04\"\n }\n },\n \"state\": \"\"\n },\n \"network\": {\n \"interfaces\": [\n {\n \"dhcp\": {\n \"arch\": \"x86_64\",\n \"ip\": {\n \"address\": \"10.129.8.90\",\n \"gateway\": \"10.129.8.89\",\n \"netmask\": \"255.255.255.252\"\n },\n \"mac\": \"18:C0:4D:B1:18:E3\",\n \"uefi\": false\n },\n \"netboot\": {\n \"allow_pxe\": true,\n \"allow_workflow\": true\n }\n }\n ]\n }\n }\n}")}, clientFactory: func() (metadata.Client, tinkerbellclient.HardwareClient, tinkerbellclient.TemplateClient, tinkerbellclient.WorkflowClient) { return &fakeMetadataClient{}, &fakeHardwareClient{ - err: &resourceErr{ + err: &resourceError{ resource: "hardware", }, }, &fakeTemplateClient{}, &fakeWorkflowClient{} @@ -220,7 +220,7 @@ func (f *fakeMetadataClient) GetMachineMetadata() (*metadata.MachineMetadata, er } type fakeHardwareClient struct { - err *resourceErr + err *resourceError } func (f *fakeHardwareClient) Get(_ context.Context, _ string, _ string, _ string) (*hardware.Hardware, error) { @@ -337,10 +337,10 @@ func (f *fakeWorkflowClient) Create(_ context.Context, _ string, _ string) (stri return "", nil } -type resourceErr struct { +type resourceError struct { resource string } -func (re *resourceErr) Error() string { +func (re *resourceError) Error() string { return fmt.Sprintf("%s %s", re.resource, tinkerbellclient.ErrNotFound.Error()) } diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata/client.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata/client.go index 9a94f6469..0ef4d0433 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata/client.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata/client.go @@ -81,7 +81,7 @@ func NewMetadataClient(cfg *Config) (Client, error) { func (d *defaultClient) GetMachineMetadata() (*MachineMetadata, error) { req, err := http.NewRequest(http.MethodGet, d.metadataEndpoint, nil) if err != nil { - return nil, fmt.Errorf("failed to create a get metadata request: %v", err) + return nil, fmt.Errorf("failed to create a get metadata request: %w", err) } req.Header.Set("Content-Type", "application/json") @@ -89,7 +89,7 @@ func (d *defaultClient) GetMachineMetadata() (*MachineMetadata, error) { res, err := d.client.Do(req) if err != nil { - return nil, fmt.Errorf("failed to execute get metadata request: %v", err) + return nil, fmt.Errorf("failed to execute get metadata request: %w", err) } if res.StatusCode != http.StatusOK { @@ -97,12 +97,12 @@ func (d *defaultClient) GetMachineMetadata() (*MachineMetadata, error) { } data, err := ioutil.ReadAll(res.Body) if err != nil { - return nil, fmt.Errorf("failed to read response body: %v", err) + return nil, fmt.Errorf("failed to read response body: %w", err) } mdConfig := &MachineMetadata{} if err := json.Unmarshal(data, mdConfig); err != nil { - return nil, fmt.Errorf("failed to unmarshal metadata config: %v", err) + return nil, fmt.Errorf("failed to unmarshal metadata config: %w", err) } return mdConfig, nil diff --git a/pkg/cloudprovider/provider/baremetal/provider.go b/pkg/cloudprovider/provider/baremetal/provider.go index 3956a12b1..0ae28cffc 100644 --- a/pkg/cloudprovider/provider/baremetal/provider.go +++ b/pkg/cloudprovider/provider/baremetal/provider.go @@ -67,7 +67,7 @@ type provider struct { configVarResolver *providerconfig.ConfigVarResolver } -// New returns a new BareMetal provider +// New returns a new BareMetal provider. func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{ configVarResolver: configVarResolver, @@ -96,29 +96,29 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p rawConfig, err := baremetaltypes.GetConfig(*pconfig) if err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal: %v", err) + return nil, nil, fmt.Errorf("failed to unmarshal: %w", err) } c := Config{} endpoint, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.MetadataClient.Endpoint, "METADATA_SERVER_ENDPOINT") if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of \"endpoint\" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to get value of \"endpoint\" field: %w`, err) } authMethod, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.MetadataClient.AuthMethod, "METADATA_SERVER_AUTH_METHOD") if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of \"authMethod\" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to get value of \"authMethod\" field: %w`, err) } username, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.MetadataClient.Username, "METADATA_SERVER_USERNAME") if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of \"username\" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to get value of \"username\" field: %w`, err) } password, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.MetadataClient.Password, "METADATA_SERVER_PASSWORD") if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of \"password\" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to get value of \"password\" field: %w`, err) } token, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.MetadataClient.Token, "METADATA_SERVER_TOKEN") if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of \"token\" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to get value of \"token\" field: %w`, err) } mdCfg := &metadata.Config{ @@ -133,7 +133,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p driverName, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.Driver) if err != nil { - return nil, nil, fmt.Errorf("failed to get baremetal provider's driver name: %v", err) + return nil, nil, fmt.Errorf("failed to get baremetal provider's driver name: %w", err) } c.driverName = plugins.Driver(driverName) @@ -147,12 +147,12 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p }{} if err := json.Unmarshal(c.driverSpec.Raw, &driverConfig); err != nil { - return nil, nil, fmt.Errorf("failed to unmarshal tinkerbell driver spec: %v", err) + return nil, nil, fmt.Errorf("failed to unmarshal tinkerbell driver spec: %w", err) } c.driver, err = tinkerbell.NewTinkerbellDriver(mdCfg, nil, driverConfig.ProvisionerIPAddress, driverConfig.MirrorHost) if err != nil { - return nil, nil, fmt.Errorf("failed to create a tinkerbell driver: %v", err) + return nil, nil, fmt.Errorf("failed to create a tinkerbell driver: %w", err) } default: return nil, nil, fmt.Errorf("unsupported baremetal driver: %s", pconfig.CloudProvider) @@ -169,7 +169,7 @@ func (p provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1 func (p provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, _, err := p.getConfig(spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to parse config: %v", err) + return fmt.Errorf("failed to parse config: %w", err) } if c.driverName == "" { @@ -194,11 +194,11 @@ func (p provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.Pr server, err := c.driver.GetServer(context.Background(), machine.UID, c.driverSpec) if err != nil { - if err == cloudprovidererrors.ErrInstanceNotFound { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return nil, cloudprovidererrors.ErrInstanceNotFound } - return nil, fmt.Errorf("failed to fetch server with the id %s: %v", machine.Name, err) + return nil, fmt.Errorf("failed to fetch server with the id %s: %w", machine.Name, err) } return &bareMetalServer{ @@ -221,12 +221,12 @@ func (p provider) Create(machine *clusterv1alpha1.Machine, data *cloudproviderty ctx := context.Background() if err := util.CreateMachineCloudInitSecret(ctx, userdata, machine.Name, data.Client); err != nil { - return nil, fmt.Errorf("failed to create cloud-init secret for machine %s: %v", machine.Name, err) + return nil, fmt.Errorf("failed to create cloud-init secret for machine %s: %w", machine.Name, err) } token, apiServer, err := util.ExtractTokenAndAPIServer(ctx, userdata, data.Client) if err != nil { - return nil, fmt.Errorf("failed to extarct token and api server address: %v", err) + return nil, fmt.Errorf("failed to extarct token and api server address: %w", err) } cfg := &plugins.CloudConfigSettings{ @@ -238,7 +238,7 @@ func (p provider) Create(machine *clusterv1alpha1.Machine, data *cloudproviderty server, err := c.driver.ProvisionServer(ctx, machine.UID, cfg, c.driverSpec) if err != nil { - return nil, fmt.Errorf("failed to provision server: %v", err) + return nil, fmt.Errorf("failed to provision server: %w", err) } return &bareMetalServer{ @@ -257,20 +257,20 @@ func (p provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidert ctx := context.Background() if err := c.driver.DeprovisionServer(ctx, machine.UID); err != nil { - return false, fmt.Errorf("failed to de-provision server: %v", err) + return false, fmt.Errorf("failed to de-provision server: %w", err) } secret := &corev1.Secret{} if err := data.Client.Get(ctx, types.NamespacedName{Namespace: util.CloudInitNamespace, Name: machine.Name}, secret); err != nil { if !kerrors.IsNotFound(err) { - return false, fmt.Errorf("failed to fetching secret for userdata: %v", err) + return false, fmt.Errorf("failed to fetching secret for userdata: %w", err) } return true, nil } if err := data.Client.Delete(ctx, secret); err != nil { - return false, fmt.Errorf("failed to cleanup secret for userdata: %v", err) + return false, fmt.Errorf("failed to cleanup secret for userdata: %w", err) } return true, nil @@ -280,7 +280,7 @@ func (p provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[st return nil, nil } -func (p provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { +func (p provider) MigrateUID(machine *clusterv1alpha1.Machine, uid types.UID) error { return nil } diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index 4c66c16ac..031b14a34 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -48,7 +48,7 @@ type provider struct { configVarResolver *providerconfig.ConfigVarResolver } -// New returns a digitalocean provider +// New returns a digitalocean provider. func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -124,7 +124,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c := Config{} c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, "DO_TOKEN") if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"token\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"token\" field, error = %w", err) } c.Region, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Region) if err != nil { @@ -168,7 +168,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to parse config: %v", err) + return fmt.Errorf("failed to parse config: %w", err) } if c.Token == "" { @@ -185,7 +185,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { _, err = getSlugForOS(pc.OperatingSystem) if err != nil { - return fmt.Errorf("invalid operating system specified %q: %v", pc.OperatingSystem, err) + return fmt.Errorf("invalid operating system specified %q: %w", pc.OperatingSystem, err) } ctx := context.TODO() @@ -242,11 +242,11 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { // uploadRandomSSHPublicKey generates a random key pair and uploads the public part of the key to // digital ocean because it is not possible to create a droplet without ssh key assigned -// this method returns an error if the key already exists +// this method returns an error if the key already exists. func uploadRandomSSHPublicKey(ctx context.Context, service godo.KeysService) (string, error) { sshkey, err := ssh.NewKey() if err != nil { - return "", fmt.Errorf("failed to generate ssh key: %v", err) + return "", fmt.Errorf("failed to generate ssh key: %w", err) } existingkey, res, err := service.GetByFingerprint(ctx, sshkey.FingerprintMD5) @@ -259,7 +259,7 @@ func uploadRandomSSHPublicKey(ctx context.Context, service godo.KeysService) (st Name: sshkey.Name, }) if err != nil { - return "", doStatusAndErrToTerminalError(rsp.StatusCode, fmt.Errorf("failed to create ssh public key on digitalocean: %v", err)) + return "", doStatusAndErrToTerminalError(rsp.StatusCode, fmt.Errorf("failed to create ssh public key on digitalocean: %w", err)) } return newDoKey.Fingerprint, nil @@ -340,7 +340,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { instance, err := p.get(machine) if err != nil { - if err == cloudprovidererrors.ErrInstanceNotFound { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return true, nil } return false, err @@ -358,7 +358,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertyp doID, err := strconv.Atoi(instance.ID()) if err != nil { - return false, fmt.Errorf("failed to convert instance id %s to int: %v", instance.ID(), err) + return false, fmt.Errorf("failed to convert instance id %s to int: %w", instance.ID(), err) } rsp, err := client.Droplets.Delete(ctx, doID) @@ -408,7 +408,7 @@ func (p *provider) listDroplets(token string) ([]godo.Droplet, error) { for { droplets, resp, err := client.Droplets.List(ctx, opt) if err != nil { - return nil, doStatusAndErrToTerminalError(resp.StatusCode, fmt.Errorf("failed to get droplets: %v", err)) + return nil, doStatusAndErrToTerminalError(resp.StatusCode, fmt.Errorf("failed to get droplets: %w", err)) } result = append(result, droplets...) @@ -428,24 +428,24 @@ func (p *provider) listDroplets(token string) ([]godo.Droplet, error) { return result, nil } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to decode providerconfig: %v", err) + return fmt.Errorf("failed to decode providerconfig: %w", err) } client := getClient(c.Token) droplets, _, err := client.Droplets.List(ctx, &godo.ListOptions{PerPage: 1000}) if err != nil { - return fmt.Errorf("failed to list droplets: %v", err) + return fmt.Errorf("failed to list droplets: %w", err) } // The create does not fail if that tag already exists, it even keep responding with a http/201 - _, response, err := client.Tags.Create(ctx, &godo.TagCreateRequest{Name: string(new)}) + _, response, err := client.Tags.Create(ctx, &godo.TagCreateRequest{Name: string(newUID)}) if err != nil { - return fmt.Errorf("failed to create new UID tag: %v, status code: %v", err, response.StatusCode) + return fmt.Errorf("failed to create new UID tag: %w, status code: %v", err, response.StatusCode) } for _, droplet := range droplets { @@ -453,16 +453,16 @@ func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) e tagResourceRequest := &godo.TagResourcesRequest{ Resources: []godo.Resource{{ID: strconv.Itoa(droplet.ID), Type: godo.DropletResourceType}}, } - _, err = client.Tags.TagResources(ctx, string(new), tagResourceRequest) + _, err = client.Tags.TagResources(ctx, string(newUID), tagResourceRequest) if err != nil { - return fmt.Errorf("failed to tag droplet with new UID tag: %v", err) + return fmt.Errorf("failed to tag droplet with new UID tag: %w", err) } untagResourceRequest := &godo.UntagResourcesRequest{ Resources: []godo.Resource{{ID: strconv.Itoa(droplet.ID), Type: godo.DropletResourceType}}, } _, err = client.Tags.UntagResources(ctx, string(machine.UID), untagResourceRequest) if err != nil { - return fmt.Errorf("failed to remove old UID tag: %v", err) + return fmt.Errorf("failed to remove old UID tag: %w", err) } } } @@ -532,7 +532,7 @@ func (d *doInstance) Status() instance.Status { // can be qualified as a "terminal" error, for more info see v1alpha1.MachineStatus // if the given error doesn't qualify the error passed as -// an argument will be returned +// an argument will be returned. func doStatusAndErrToTerminalError(status int, err error) error { switch status { case http.StatusUnauthorized: diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index 6b1704eb6..da076e57b 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -45,7 +45,7 @@ const ( defaultBillingCycle = "hourly" ) -// New returns a Equinix Metal provider +// New returns a Equinix Metal provider. func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -60,7 +60,7 @@ type Config struct { } // because we have both Config and RawConfig, we need to have func for each -// ideally, these would be merged into one +// ideally, these would be merged into one. func (c *Config) populateDefaults() { if c.BillingCycle == "" { c.BillingCycle = defaultBillingCycle @@ -104,7 +104,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *e // TODO(@ahmedwaleedmalik) Remove this after a release period c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, "PACKET_API_KEY") if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"apiKey\" field, error = %v", err) + return nil, nil, nil, fmt.Errorf("failed to get the value of \"apiKey\" field, error = %w", err) } } c.ProjectID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProjectID, "METAL_PROJECT_ID") @@ -114,28 +114,28 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *e // TODO(@ahmedwaleedmalik) Remove this after a release period c.ProjectID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProjectID, "PACKET_PROJECT_ID") if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"apiKey\" field, error = %v", err) + return nil, nil, nil, fmt.Errorf("failed to get the value of \"apiKey\" field, error = %w", err) } } c.InstanceType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.InstanceType) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"instanceType\" field, error = %v", err) + return nil, nil, nil, fmt.Errorf("failed to get the value of \"instanceType\" field, error = %w", err) } c.BillingCycle, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.BillingCycle) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"billingCycle\" field, error = %v", err) + return nil, nil, nil, fmt.Errorf("failed to get the value of \"billingCycle\" field, error = %w", err) } for i, tag := range rawConfig.Tags { tagValue, err := p.configVarResolver.GetConfigVarStringValue(tag) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to read the value for the Tag at index %d of the \"tags\" field, error = %v", i, err) + return nil, nil, nil, fmt.Errorf("failed to read the value for the Tag at index %d of the \"tags\" field, error = %w", i, err) } c.Tags = append(c.Tags, tagValue) } for i, facility := range rawConfig.Facilities { facilityValue, err := p.configVarResolver.GetConfigVarStringValue(facility) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to read the value for the Tag at index %d of the \"facilities\" field, error = %v", i, err) + return nil, nil, nil, fmt.Errorf("failed to read the value for the Tag at index %d of the \"facilities\" field, error = %w", i, err) } c.Facilities = append(c.Facilities, facilityValue) } @@ -166,7 +166,7 @@ func (p *provider) getMetalDevice(machine *clusterv1alpha1.Machine) (*packngo.De func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, _, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to parse config: %v", err) + return fmt.Errorf("failed to parse config: %w", err) } if c.Token == "" { @@ -181,7 +181,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { _, err = getNameForOS(pc.OperatingSystem) if err != nil { - return fmt.Errorf("invalid/not supported operating system specified %q: %v", pc.OperatingSystem, err) + return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, err) } client := getClient(c.Token) @@ -193,7 +193,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { // get all valid facilities facilities, _, err := client.Facilities.List(nil) if err != nil { - return fmt.Errorf("failed to list facilities: %v", err) + return fmt.Errorf("failed to list facilities: %w", err) } // ensure our requested facilities are in those facilities if missingFacilities := itemsNotInList(facilityProp(facilities, "Code"), c.Facilities); len(missingFacilities) > 0 { @@ -203,7 +203,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { // get all valid plans a.k.a. instance types plans, _, err := client.Plans.List(nil) if err != nil { - return fmt.Errorf("failed to list instance types / plans: %v", err) + return fmt.Errorf("failed to list instance types / plans: %w", err) } // ensure our requested plan is in those plans validPlanNames := planProp(plans, "Name") @@ -257,7 +257,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { instance, err := p.Get(machine, data) if err != nil { - if err == cloudprovidererrors.ErrInstanceNotFound { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return true, nil } return false, err @@ -397,9 +397,7 @@ func (s *metalDevice) Status() instance.Status { } } -/****** -CONVENIENCE INTERNAL FUNCTIONS -******/ +// CONVENIENCE INTERNAL FUNCTIONS. func setProviderSpec(rawConfig equinixmetaltypes.RawConfig, s clusterv1alpha1.ProviderSpec) (*runtime.RawExtension, error) { if s.Value == nil { return nil, fmt.Errorf("machine.spec.providerconfig.value is nil") @@ -438,7 +436,7 @@ func getDeviceByTag(client *packngo.Client, projectID, tag string) (*packngo.Dev return nil, nil } -// given a defined Kubermatic constant for an operating system, return the canonical slug for Equinix Metal +// given a defined Kubermatic constant for an operating system, return the canonical slug for Equinix Metal. func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { switch os { case providerconfigtypes.OperatingSystemUbuntu: @@ -470,10 +468,10 @@ func getTagUID(tag string) (string, error) { // metalErrorToTerminalError judges if the given error // can be qualified as a "terminal" error, for more info see v1alpha1.MachineStatus // -// if the given error doesn't qualify the error passed as an argument will be returned +// if the given error doesn't qualify the error passed as an argument will be returned. func metalErrorToTerminalError(err error, response *packngo.Response, msg string) error { prepareAndReturnError := func() error { - return fmt.Errorf("%s, due to %s", msg, err) + return fmt.Errorf("%s, due to %w", msg, err) } if err != nil { diff --git a/pkg/cloudprovider/provider/fake/provider.go b/pkg/cloudprovider/provider/fake/provider.go index a0655d363..b25950518 100644 --- a/pkg/cloudprovider/provider/fake/provider.go +++ b/pkg/cloudprovider/provider/fake/provider.go @@ -55,7 +55,7 @@ func (f CloudProviderInstance) Status() instance.Status { return instance.StatusUnknown } -// New returns a fake cloud provider +// New returns a fake cloud provider. func New(_ *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{} } @@ -64,7 +64,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, nil } -// Validate returns success or failure based according to its FakeCloudProviderSpec +// Validate returns success or failure based according to its FakeCloudProviderSpec. func (p *provider) Validate(machinespec clusterv1alpha1.MachineSpec) error { pconfig, err := providerconfigtypes.GetConfig(machinespec.ProviderSpec) if err != nil { @@ -93,7 +93,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (string, str return "", "", nil } -// Create creates a cloud instance according to the given machine +// Create creates a cloud instance according to the given machine. func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { return CloudProviderInstance{}, nil } @@ -102,7 +102,7 @@ func (p *provider) Cleanup(_ *clusterv1alpha1.Machine, _ *cloudprovidertypes.Pro return true, nil } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, uid types.UID) error { return nil } diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index 00e544406..2e6e50f75 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -41,12 +41,12 @@ const ( envGoogleServiceAccount = "GOOGLE_SERVICE_ACCOUNT" ) -// imageProjects maps the OS to the Google Cloud image projects +// imageProjects maps the OS to the Google Cloud image projects. var imageProjects = map[providerconfigtypes.OperatingSystem]string{ providerconfigtypes.OperatingSystemUbuntu: "ubuntu-os-cloud", } -// imageFamilies maps the OS to the Google Cloud image projects +// imageFamilies maps the OS to the Google Cloud image projects. var imageFamilies = map[providerconfigtypes.OperatingSystem]string{ providerconfigtypes.OperatingSystemUbuntu: "ubuntu-2004-lts", } @@ -74,7 +74,7 @@ func newCloudProviderSpec(provSpec v1alpha1.ProviderSpec) (*gcetypes.CloudProvid pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { - return nil, nil, fmt.Errorf("cannot unmarshal machine.spec.providerconfig.value: %v", err) + return nil, nil, fmt.Errorf("cannot unmarshal machine.spec.providerconfig.value: %w", err) } if pconfig.OperatingSystemSpec.Raw == nil { @@ -84,7 +84,7 @@ func newCloudProviderSpec(provSpec v1alpha1.ProviderSpec) (*gcetypes.CloudProvid // Retrieve cloud provider specification from cloud provider specification. cpSpec, err := gcetypes.GetConfig(*pconfig) if err != nil { - return nil, nil, fmt.Errorf("cannot unmarshal cloud provider specification: %v", err) + return nil, nil, fmt.Errorf("cannot unmarshal cloud provider specification: %w", err) } return cpSpec, pconfig, nil @@ -131,48 +131,48 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide cfg.serviceAccount, err = resolver.GetConfigVarStringValueOrEnv(cpSpec.ServiceAccount, envGoogleServiceAccount) if err != nil { - return nil, fmt.Errorf("cannot retrieve service account: %v", err) + return nil, fmt.Errorf("cannot retrieve service account: %w", err) } err = cfg.postprocessServiceAccount() if err != nil { - return nil, fmt.Errorf("cannot prepare JWT: %v", err) + return nil, fmt.Errorf("cannot prepare JWT: %w", err) } cfg.zone, err = resolver.GetConfigVarStringValue(cpSpec.Zone) if err != nil { - return nil, fmt.Errorf("cannot retrieve zone: %v", err) + return nil, fmt.Errorf("cannot retrieve zone: %w", err) } cfg.machineType, err = resolver.GetConfigVarStringValue(cpSpec.MachineType) if err != nil { - return nil, fmt.Errorf("cannot retrieve machine type: %v", err) + return nil, fmt.Errorf("cannot retrieve machine type: %w", err) } cfg.diskType, err = resolver.GetConfigVarStringValue(cpSpec.DiskType) if err != nil { - return nil, fmt.Errorf("cannot retrieve disk type: %v", err) + return nil, fmt.Errorf("cannot retrieve disk type: %w", err) } cfg.network, err = resolver.GetConfigVarStringValue(cpSpec.Network) if err != nil { - return nil, fmt.Errorf("cannot retrieve network: %v", err) + return nil, fmt.Errorf("cannot retrieve network: %w", err) } cfg.subnetwork, err = resolver.GetConfigVarStringValue(cpSpec.Subnetwork) if err != nil { - return nil, fmt.Errorf("cannot retrieve subnetwork: %v", err) + return nil, fmt.Errorf("cannot retrieve subnetwork: %w", err) } cfg.preemptible, _, err = resolver.GetConfigVarBoolValue(cpSpec.Preemptible) if err != nil { - return nil, fmt.Errorf("cannot retrieve preemptible: %v", err) + return nil, fmt.Errorf("cannot retrieve preemptible: %w", err) } if cpSpec.AutomaticRestart != nil { automaticRestart, _, err := resolver.GetConfigVarBoolValue(*cpSpec.AutomaticRestart) if err != nil { - return nil, fmt.Errorf("cannot retrieve automaticRestart: %v", err) + return nil, fmt.Errorf("cannot retrieve automaticRestart: %w", err) } cfg.automaticRestart = &automaticRestart @@ -184,7 +184,7 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide if cpSpec.ProvisioningModel != nil { provisioningModel, err := resolver.GetConfigVarStringValue(*cpSpec.ProvisioningModel) if err != nil { - return nil, fmt.Errorf("cannot retrieve provisioningModel: %v", err) + return nil, fmt.Errorf("cannot retrieve provisioningModel: %w", err) } cfg.provisioningModel = &provisioningModel } @@ -195,23 +195,23 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide if cpSpec.AssignPublicIPAddress != nil { cfg.assignPublicIPAddress, _, err = resolver.GetConfigVarBoolValue(*cpSpec.AssignPublicIPAddress) if err != nil { - return nil, fmt.Errorf("failed to retrieve assignPublicIPAddress: %v", err) + return nil, fmt.Errorf("failed to retrieve assignPublicIPAddress: %w", err) } } cfg.multizone, _, err = resolver.GetConfigVarBoolValue(cpSpec.MultiZone) if err != nil { - return nil, fmt.Errorf("failed to retrieve multizone: %v", err) + return nil, fmt.Errorf("failed to retrieve multizone: %w", err) } cfg.regional, _, err = resolver.GetConfigVarBoolValue(cpSpec.Regional) if err != nil { - return nil, fmt.Errorf("failed to retrieve regional: %v", err) + return nil, fmt.Errorf("failed to retrieve regional: %w", err) } cfg.customImage, err = resolver.GetConfigVarStringValue(cpSpec.CustomImage) if err != nil { - return nil, fmt.Errorf("failed to retrieve gce custom image: %v", err) + return nil, fmt.Errorf("failed to retrieve gce custom image: %w", err) } return cfg, nil @@ -222,17 +222,17 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide func (cfg *config) postprocessServiceAccount() error { sa, err := base64.StdEncoding.DecodeString(cfg.serviceAccount) if err != nil { - return fmt.Errorf("failed to decode base64 service account: %v", err) + return fmt.Errorf("failed to decode base64 service account: %w", err) } sam := map[string]string{} err = json.Unmarshal(sa, &sam) if err != nil { - return fmt.Errorf("failed unmarshalling service account: %v", err) + return fmt.Errorf("failed unmarshalling service account: %w", err) } cfg.projectID = sam["project_id"] cfg.jwtConfig, err = google.JWTConfigFromJSON(sa, compute.ComputeScope) if err != nil { - return fmt.Errorf("failed preparing JWT: %v", err) + return fmt.Errorf("failed preparing JWT: %w", err) } return nil } diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index ba3df6677..4d4bd1764 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -21,18 +21,19 @@ limitations under the License. package gce import ( + "errors" "fmt" "net/http" "strconv" "cloud.google.com/go/logging" - monitoring "cloud.google.com/go/monitoring/apiv3" + monitoring "cloud.google.com/go/monitoring/apiv3/v2" compute "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" + cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" gcetypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/gce/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" @@ -81,7 +82,7 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) *Provider { } } -// AddDefaults reads the MachineSpec and applies defaults for provider specific fields +// AddDefaults reads the MachineSpec and applies defaults for provider specific fields. func (p *Provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { // Read cloud provider spec. cpSpec, _, err := newCloudProviderSpec(spec.ProviderSpec) @@ -160,15 +161,16 @@ func (p *Provider) get(machine *clusterv1alpha1.Machine) (*googleInstance, error label := fmt.Sprintf("labels.%s=%s", labelMachineUID, machine.UID) insts, err := svc.Instances.List(cfg.projectID, cfg.zone).Filter(label).Do() if err != nil { - if gerr, ok := err.(*googleapi.Error); ok { + var gerr *googleapi.Error + if errors.As(err, &gerr) { if gerr.Code == http.StatusNotFound { - return nil, errors.ErrInstanceNotFound + return nil, cloudprovidererrors.ErrInstanceNotFound } } return nil, newError(common.InvalidConfigurationMachineError, errRetrieveInstance, err) } if len(insts.Items) == 0 { - return nil, errors.ErrInstanceNotFound + return nil, cloudprovidererrors.ErrInstanceNotFound } if len(insts.Items) > 1 { return nil, newError(common.InvalidConfigurationMachineError, errGotTooManyInstances) @@ -301,7 +303,8 @@ func (p *Provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider // Delete instance. op, err := svc.Instances.Delete(cfg.projectID, cfg.zone, machine.Spec.Name).Do() if err != nil { - if gerr, ok := err.(*googleapi.Error); ok { + var gerr *googleapi.Error + if errors.As(err, &gerr) { if gerr.Code == http.StatusNotFound { return true, nil } @@ -351,7 +354,7 @@ func (p *Provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID // Retrieve instance. inst, err := p.get(machine) if err != nil { - if err == errors.ErrInstanceNotFound { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return nil } return err @@ -385,7 +388,7 @@ func (p *Provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) e // newError creates a terminal error matching to the provider interface. func newError(reason common.MachineStatusError, msg string, args ...interface{}) error { - return errors.TerminalError{ + return cloudprovidererrors.TerminalError{ Reason: reason, Message: fmt.Sprintf(msg, args...), } diff --git a/pkg/cloudprovider/provider/gce/provider_test.go b/pkg/cloudprovider/provider/gce/provider_test.go index 4f5fc405b..f63ef2248 100644 --- a/pkg/cloudprovider/provider/gce/provider_test.go +++ b/pkg/cloudprovider/provider/gce/provider_test.go @@ -82,7 +82,7 @@ func testServiceAccount() string { type testMap map[string]interface{} -// with patches value of m at keypath with val e.g. keypath=x.y val=z then m[x][y] = z +// with patches value of m at keypath with val e.g. keypath=x.y val=z then m[x][y] = z. func (m testMap) with(keypath, val string) testMap { parts := strings.Split(keypath, ".") var curr interface{} = m @@ -103,7 +103,7 @@ func (m testMap) with(keypath, val string) testMap { } func TestValidate(t *testing.T) { - os.Setenv(envGoogleServiceAccount, testServiceAccount()) + t.Setenv(envGoogleServiceAccount, testServiceAccount()) defer os.Unsetenv(envGoogleServiceAccount) rawBytes := func(m map[string]interface{}) []byte { diff --git a/pkg/cloudprovider/provider/gce/service.go b/pkg/cloudprovider/provider/gce/service.go index ed2ee566e..840695c48 100644 --- a/pkg/cloudprovider/provider/gce/service.go +++ b/pkg/cloudprovider/provider/gce/service.go @@ -21,11 +21,12 @@ limitations under the License. package gce import ( + "context" "fmt" "time" - "golang.org/x/oauth2" "google.golang.org/api/compute/v1" + "google.golang.org/api/option" "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" @@ -55,9 +56,10 @@ type service struct { // connectComputeService establishes a service connection to the Compute Engine. func connectComputeService(cfg *config) (*service, error) { - svc, err := compute.New(cfg.jwtConfig.Client(oauth2.NoContext)) + client := cfg.jwtConfig.Client(context.Background()) + svc, err := compute.NewService(context.Background(), option.WithHTTPClient(client)) if err != nil { - return nil, fmt.Errorf("cannot connect to Google Cloud: %v", err) + return nil, fmt.Errorf("cannot connect to Google Cloud: %w", err) } return &service{svc}, nil } @@ -104,9 +106,7 @@ func (svc *service) networkInterfaces(cfg *config) ([]*compute.NetworkInterface, } else { klog.Infof("IP family doesn't specify dual stack: %s", cfg.providerConfig.Network.GetIPFamily()) } - } - return []*compute.NetworkInterface{ifc}, nil } diff --git a/pkg/cloudprovider/provider/gce/types/cloudconfig.go b/pkg/cloudprovider/provider/gce/types/cloudconfig.go index 24e8000b9..9c4201d4b 100644 --- a/pkg/cloudprovider/provider/gce/types/cloudconfig.go +++ b/pkg/cloudprovider/provider/gce/types/cloudconfig.go @@ -67,12 +67,12 @@ func (cc *CloudConfig) AsString() (string, error) { tmpl, err := template.New("cloud-config").Funcs(funcMap).Parse(cloudConfigTemplate) if err != nil { - return "", fmt.Errorf("failed to parse the cloud config template: %v", err) + return "", fmt.Errorf("failed to parse the cloud config template: %w", err) } buf := &bytes.Buffer{} if err := tmpl.Execute(buf, cc); err != nil { - return "", fmt.Errorf("failed to execute cloud config template: %v", err) + return "", fmt.Errorf("failed to execute cloud config template: %w", err) } return buf.String(), nil diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 70de29068..0db5fe0e5 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -50,7 +50,7 @@ type provider struct { configVarResolver *providerconfig.ConfigVarResolver } -// New returns a Hetzner provider +// New returns a Hetzner provider. func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -105,7 +105,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c := Config{} c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, "HZ_TOKEN") if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"token\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"token\" field, error = %w", err) } c.ServerType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ServerType) @@ -187,7 +187,7 @@ func (p *provider) getServerPlacementGroup(ctx context.Context, client *hcloud.C func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to parse config: %v", err) + return fmt.Errorf("failed to parse config: %w", err) } if c.Token == "" { @@ -196,7 +196,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { _, err = getNameForOS(pc.OperatingSystem) if err != nil { - return fmt.Errorf("invalid/not supported operating system specified %q: %v", pc.OperatingSystem, err) + return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, err) } ctx := context.TODO() @@ -208,32 +208,32 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { if c.Location != "" { if _, _, err = client.Location.Get(ctx, c.Location); err != nil { - return fmt.Errorf("failed to get location: %v", err) + return fmt.Errorf("failed to get location: %w", err) } } if c.Datacenter != "" { if _, _, err = client.Datacenter.Get(ctx, c.Datacenter); err != nil { - return fmt.Errorf("failed to get datacenter: %v", err) + return fmt.Errorf("failed to get datacenter: %w", err) } } if c.Image != "" { if _, _, err = client.Image.Get(ctx, c.Image); err != nil { - return fmt.Errorf("failed to get image: %v", err) + return fmt.Errorf("failed to get image: %w", err) } } for _, network := range c.Networks { if _, _, err = client.Network.Get(ctx, network); err != nil { - return fmt.Errorf("failed to get network %q: %v", network, err) + return fmt.Errorf("failed to get network %q: %w", network, err) } } for _, firewall := range c.Firewalls { f, _, err := client.Firewall.Get(ctx, firewall) if err != nil { - return fmt.Errorf("failed to get firewall %q: %v", firewall, err) + return fmt.Errorf("failed to get firewall %q: %w", firewall, err) } if f == nil { return fmt.Errorf("firewall %q does not exist", firewall) @@ -241,7 +241,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { } if _, _, err = client.ServerType.Get(ctx, c.ServerType); err != nil { - return fmt.Errorf("failed to get server type: %v", err) + return fmt.Errorf("failed to get server type: %w", err) } return nil @@ -356,7 +356,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert // spammy. No one will ever get access to the private key. sshkey, err := ssh.NewKey() if err != nil { - return nil, fmt.Errorf("failed to generate ssh key: %v", err) + return nil, fmt.Errorf("failed to generate ssh key: %w", err) } hkey, res, err := client.SSHKey.Create(ctx, hcloud.SSHKeyCreateOpts{ @@ -364,7 +364,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert PublicKey: sshkey.PublicKey, }) if err != nil { - return nil, fmt.Errorf("creating temporary ssh key failed with error %v", err) + return nil, fmt.Errorf("creating temporary ssh key failed with error %w", err) } if res.StatusCode != http.StatusCreated { return nil, fmt.Errorf("got invalid http status code when creating ssh key: expected=%d, god=%d", http.StatusCreated, res.StatusCode) @@ -391,7 +391,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { instance, err := p.Get(machine, data) if err != nil { - if err == cloudprovidererrors.ErrInstanceNotFound { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return true, nil } return false, err @@ -471,7 +471,7 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -487,7 +487,7 @@ func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) e // We didn't use the UID for Hetzner before server, _, err := client.Server.Get(ctx, machine.Spec.Name) if err != nil { - return fmt.Errorf("failed to get server: %v", err) + return fmt.Errorf("failed to get server: %w", err) } if server == nil { klog.Infof("No instance exists for machine %s", machine.Name) @@ -496,10 +496,10 @@ func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) e klog.Infof("Setting UID label for machine %s", machine.Name) _, response, err := client.Server.Update(ctx, server, hcloud.ServerUpdateOpts{ - Labels: map[string]string{machineUIDLabelKey: string(new)}, + Labels: map[string]string{machineUIDLabelKey: string(newUID)}, }) if err != nil { - return fmt.Errorf("failed to update UID label: %v", err) + return fmt.Errorf("failed to update UID label: %w", err) } if response.Response.StatusCode != http.StatusOK { return fmt.Errorf("got unexpected response code %v, expected %v", response.Response.Status, http.StatusOK) @@ -567,10 +567,10 @@ func (s *hetznerServer) Status() instance.Status { // hzErrorToTerminalError judges if the given error // can be qualified as a "terminal" error, for more info see v1alpha1.MachineStatus // -// if the given error doesn't qualify the error passed as an argument will be returned +// if the given error doesn't qualify the error passed as an argument will be returned. func hzErrorToTerminalError(err error, msg string) error { prepareAndReturnError := func() error { - return fmt.Errorf("%s, due to %s", msg, err) + return fmt.Errorf("%s, due to %w", msg, err) } if err != nil { diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 9dbcc6956..8cd0fd4b6 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -79,7 +79,7 @@ type provider struct { configVarResolver *providerconfig.ConfigVarResolver } -// New returns a Kubevirt provider +// New returns a Kubevirt provider. func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -106,18 +106,18 @@ type AffinityType string const ( // Facade for podAffinity, podAntiAffinity, nodeAffinity, nodeAntiAffinity - // HardAffinityType: affinity will include requiredDuringSchedulingIgnoredDuringExecution + // HardAffinityType: affinity will include requiredDuringSchedulingIgnoredDuringExecution. hardAffinityType = "hard" - // SoftAffinityType: affinity will include preferredDuringSchedulingIgnoredDuringExecution + // SoftAffinityType: affinity will include preferredDuringSchedulingIgnoredDuringExecution. softAffinityType = "soft" - // NoAffinityType: affinity section will not be preset + // NoAffinityType: affinity section will not be preset. noAffinityType = "" ) func (p *provider) affinityType(affinityType providerconfigtypes.ConfigVarString) (AffinityType, error) { podAffinityPresetString, err := p.configVarResolver.GetConfigVarStringValue(affinityType) if err != nil { - return "", fmt.Errorf(`failed to parse "podAffinityPreset" field: %v`, err) + return "", fmt.Errorf(`failed to parse "podAffinityPreset" field: %w`, err) } switch strings.ToLower(podAffinityPresetString) { case string(hardAffinityType): @@ -131,7 +131,7 @@ func (p *provider) affinityType(affinityType providerconfigtypes.ConfigVarString return "", fmt.Errorf("unknown affinityType: %s", affinityType) } -// NodeAffinityPreset +// NodeAffinityPreset. type NodeAffinityPreset struct { Type AffinityType Key string @@ -201,20 +201,20 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p config := Config{} config.Kubeconfig, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Auth.Kubeconfig, "KUBEVIRT_KUBECONFIG") if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "kubeconfig" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to get value of "kubeconfig" field: %w`, err) } config.CPUs, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.CPUs) if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "cpus" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to get value of "cpus" field: %w`, err) } config.Memory, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.Memory) if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "memory" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to get value of "memory" field: %w`, err) } config.Namespace = getNamespace() osImage, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.OsImage) if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "sourceURL" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to get value of "sourceURL" field: %w`, err) } if _, err = url.ParseRequestURI(osImage); err == nil { config.OsImage.URL = osImage @@ -223,32 +223,32 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } pvcSize, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.Size) if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "pvcSize" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to get value of "pvcSize" field: %w`, err) } if config.PVCSize, err = resource.ParseQuantity(pvcSize); err != nil { - return nil, nil, fmt.Errorf(`failed to parse value of "pvcSize" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to parse value of "pvcSize" field: %w`, err) } config.StorageClassName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.StorageClassName) if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "storageClassName" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to get value of "storageClassName" field: %w`, err) } config.RestConfig, err = clientcmd.RESTConfigFromKubeConfig([]byte(config.Kubeconfig)) if err != nil { - return nil, nil, fmt.Errorf("failed to decode kubeconfig: %v", err) + return nil, nil, fmt.Errorf("failed to decode kubeconfig: %w", err) } config.FlavorName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Flavor.Name) if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "flavor.name" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to get value of "flavor.name" field: %w`, err) } dnsPolicyString, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.DNSPolicy) if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "dnsPolicy" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to parse "dnsPolicy" field: %w`, err) } if dnsPolicyString != "" { config.DNSPolicy, err = dnsPolicy(dnsPolicyString) if err != nil { - return nil, nil, fmt.Errorf("failed to get dns policy: %v", err) + return nil, nil, fmt.Errorf("failed to get dns policy: %w", err) } } if rawConfig.VirtualMachine.DNSConfig != nil { @@ -256,19 +256,18 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } config.SecondaryDisks = make([]SecondaryDisks, 0, len(rawConfig.VirtualMachine.Template.SecondaryDisks)) for _, sd := range rawConfig.VirtualMachine.Template.SecondaryDisks { - sdSizeString, err := p.configVarResolver.GetConfigVarStringValue(sd.Size) if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "secondaryDisks.size" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to parse "secondaryDisks.size" field: %w`, err) } pvc, err := resource.ParseQuantity(sdSizeString) if err != nil { - return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.size" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.size" field: %w`, err) } scString, err := p.configVarResolver.GetConfigVarStringValue(sd.StorageClassName) if err != nil { - return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.storageClass" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.storageClass" field: %w`, err) } config.SecondaryDisks = append(config.SecondaryDisks, SecondaryDisks{ Size: pvc, @@ -279,15 +278,15 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p // Affinity/AntiAffinity config.PodAffinityPreset, err = p.affinityType(rawConfig.Affinity.PodAffinityPreset) if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "podAffinityPreset" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to parse "podAffinityPreset" field: %w`, err) } config.PodAntiAffinityPreset, err = p.affinityType(rawConfig.Affinity.PodAntiAffinityPreset) if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "podAntiAffinityPreset" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to parse "podAntiAffinityPreset" field: %w`, err) } config.NodeAffinityPreset, err = p.parseNodeAffinityPreset(rawConfig.Affinity.NodeAffinityPreset) if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "nodeAffinityPreset" field: %v`, err) + return nil, nil, fmt.Errorf(`failed to parse "nodeAffinityPreset" field: %w`, err) } return &config, pconfig, nil @@ -298,17 +297,17 @@ func (p *provider) parseNodeAffinityPreset(nodeAffinityPreset kubevirttypes.Node var err error nodeAffinity.Type, err = p.affinityType(nodeAffinityPreset.Type) if err != nil { - return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.type" field: %v`, err) + return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.type" field: %w`, err) } nodeAffinity.Key, err = p.configVarResolver.GetConfigVarStringValue(nodeAffinityPreset.Key) if err != nil { - return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.key" field: %v`, err) + return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.key" field: %w`, err) } nodeAffinity.Values = make([]string, len(nodeAffinityPreset.Values)) for _, v := range nodeAffinityPreset.Values { valueString, err := p.configVarResolver.GetConfigVarStringValue(v) if err != nil { - return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.value" field: %v`, err) + return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.value" field: %w`, err) } nodeAffinity.Values = append(nodeAffinity.Values, valueString) } @@ -338,14 +337,14 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P } sigClient, err := client.New(c.RestConfig, client.Options{}) if err != nil { - return nil, fmt.Errorf("failed to get kubevirt client: %v", err) + return nil, fmt.Errorf("failed to get kubevirt client: %w", err) } ctx := context.Background() virtualMachine := &kubevirtv1.VirtualMachine{} if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: machine.Name}, virtualMachine); err != nil { if !kerrors.IsNotFound(err) { - return nil, fmt.Errorf("failed to get VirtualMachine %s: %v", machine.Name, err) + return nil, fmt.Errorf("failed to get VirtualMachine %s: %w", machine.Name, err) } return nil, cloudprovidererrors.ErrInstanceNotFound } @@ -374,7 +373,7 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P // The pod got deleted, delete the VMI and return ErrNotFound so the VMI // will get recreated if err := sigClient.Delete(ctx, virtualMachineInstance); err != nil { - return nil, fmt.Errorf("failed to delete failed VMI %s: %v", machine.Name, err) + return nil, fmt.Errorf("failed to delete failed VMI %s: %w", machine.Name, err) } return nil, cloudprovidererrors.ErrInstanceNotFound } @@ -384,17 +383,17 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P // We don't use the UID for kubevirt because the name of a VMI must stay stable // in order for the node name to stay stable. The operator is responsible for ensuring -// there are no conflicts, e.G. by using one Namespace per Kubevirt user cluster -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { +// there are no conflicts, e.G. by using one Namespace per Kubevirt user cluster. +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, uid types.UID) error { return nil } func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to parse config: %v", err) + return fmt.Errorf("failed to parse config: %w", err) } - // If VMIPreset is specified, skip CPU and Memory validation + // If VMIPreset is specified, skip CPU and Memory validation. if c.FlavorName == "" { if _, err := parseResources(c.CPUs, c.Memory); err != nil { return err @@ -403,20 +402,20 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { sigClient, err := client.New(c.RestConfig, client.Options{}) if err != nil { - return fmt.Errorf("failed to get kubevirt client: %v", err) + return fmt.Errorf("failed to get kubevirt client: %w", err) } if _, ok := supportedOS[pc.OperatingSystem]; !ok { - return fmt.Errorf("invalid/not supported operating system specified %q: %v", pc.OperatingSystem, providerconfigtypes.ErrOSNotSupported) + return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, providerconfigtypes.ErrOSNotSupported) } if c.DNSPolicy == corev1.DNSNone { if c.DNSConfig == nil || len(c.DNSConfig.Nameservers) == 0 { return fmt.Errorf("dns config must be specified when dns policy is None") } } - // Check if we can reach the API of the target cluster + // Check if we can reach the API of the target cluster. vmi := &kubevirtv1.VirtualMachineInstance{} if err := sigClient.Get(context.Background(), types.NamespacedName{Namespace: c.Namespace, Name: "not-expected-to-exist"}, vmi); err != nil && !kerrors.IsNotFound(err) { - return fmt.Errorf("failed to request VirtualMachineInstances: %v", err) + return fmt.Errorf("failed to request VirtualMachineInstances: %w", err) } return nil @@ -429,7 +428,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { c, _, err := p.getConfig(spec.ProviderSpec) if err != nil { - return "", "", fmt.Errorf("failed to parse config: %v", err) + return "", "", fmt.Errorf("failed to parse config: %w", err) } cc := kubevirttypes.CloudConfig{ Kubeconfig: c.Kubeconfig, @@ -463,20 +462,20 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert // We add the timestamp because the secret name must be different when we recreate the VMI // because its pod got deleted - // The secret has an ownerRef on the VMI so garbace collection will take care of cleaning up + // The secret has an ownerRef on the VMI so garbace collection will take care of cleaning up. terminationGracePeriodSeconds := int64(30) userDataSecretName := fmt.Sprintf("userdata-%s-%s", machine.Name, strconv.Itoa(int(time.Now().Unix()))) resourceRequirements := kubevirtv1.ResourceRequirements{} labels := map[string]string{"kubevirt.io/vm": machine.Name} - // Add a common label to all VirtualMachines spawned by the same MachineDeployment (= MachineDeployment name) + // Add a common label to all VirtualMachines spawned by the same MachineDeployment (= MachineDeployment name). if mdName, err := controllerutil.GetMachineDeploymentNameForMachine(context.Background(), machine, data.Client); err == nil { labels[machineDeploymentLabelKey] = mdName } sigClient, err := client.New(c.RestConfig, client.Options{}) if err != nil { - return nil, fmt.Errorf("failed to get kubevirt client: %v", err) + return nil, fmt.Errorf("failed to get kubevirt client: %w", err) } ctx := context.Background() @@ -556,7 +555,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert } if err := sigClient.Create(ctx, virtualMachine); err != nil { - return nil, fmt.Errorf("failed to create vmi: %v", err) + return nil, fmt.Errorf("failed to create vmi: %w", err) } secret := &corev1.Secret{ @@ -568,10 +567,9 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert Data: map[string][]byte{"userdata": []byte(userdata)}, } if err := sigClient.Create(ctx, secret); err != nil { - return nil, fmt.Errorf("failed to create secret for userdata: %v", err) + return nil, fmt.Errorf("failed to create secret for userdata: %w", err) } return &kubeVirtServer{}, nil - } func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { @@ -584,14 +582,14 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertyp } sigClient, err := client.New(c.RestConfig, client.Options{}) if err != nil { - return false, fmt.Errorf("failed to get kubevirt client: %v", err) + return false, fmt.Errorf("failed to get kubevirt client: %w", err) } ctx := context.Background() vm := &kubevirtv1.VirtualMachine{} if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: machine.Name}, vm); err != nil { if !kerrors.IsNotFound(err) { - return false, fmt.Errorf("failed to get VirtualMachineInstance %s: %v", machine.Name, err) + return false, fmt.Errorf("failed to get VirtualMachineInstance %s: %w", machine.Name, err) } // VMI is gone return true, nil @@ -603,11 +601,11 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertyp func parseResources(cpus, memory string) (*corev1.ResourceList, error) { memoryResource, err := resource.ParseQuantity(memory) if err != nil { - return nil, fmt.Errorf("failed to parse memory requests: %v", err) + return nil, fmt.Errorf("failed to parse memory requests: %w", err) } cpuResource, err := resource.ParseQuantity(cpus) if err != nil { - return nil, fmt.Errorf("failed to parse cpu request: %v", err) + return nil, fmt.Errorf("failed to parse cpu request: %w", err) } return &corev1.ResourceList{ corev1.ResourceMemory: memoryResource, @@ -741,7 +739,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. return dataVolumeTemplates } -// getDataVolumeSource returns DataVolumeSource, HTTP or PVC +// getDataVolumeSource returns DataVolumeSource, HTTP or PVC. func getDataVolumeSource(osImage OSImage) *cdiv1beta1.DataVolumeSource { dataVolumeSource := &cdiv1beta1.DataVolumeSource{} if osImage.URL != "" { @@ -753,7 +751,6 @@ func getDataVolumeSource(osImage OSImage) *cdiv1beta1.DataVolumeSource { Name: nameSpaceAndName[1], } } - } return dataVolumeSource } diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index bd03203d1..8eaab19b1 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -29,12 +29,12 @@ type RawConfig struct { Affinity Affinity `json:"affinity,omitempty"` } -// Auth +// Auth. type Auth struct { Kubeconfig providerconfigtypes.ConfigVarString `json:"kubeconfig,omitempty"` } -// VirtualMachine +// VirtualMachine. type VirtualMachine struct { Flavor Flavor `json:"flavor,omitempty"` Template Template `json:"template,omitempty"` @@ -42,13 +42,13 @@ type VirtualMachine struct { DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` } -// Flavor +// Flavor. type Flavor struct { Name providerconfigtypes.ConfigVarString `json:"name,omitempty"` Profile providerconfigtypes.ConfigVarString `json:"profile,omitempty"` } -// Template +// Template. type Template struct { CPUs providerconfigtypes.ConfigVarString `json:"cpus,omitempty"` Memory providerconfigtypes.ConfigVarString `json:"memory,omitempty"` @@ -56,31 +56,31 @@ type Template struct { SecondaryDisks []SecondaryDisks `json:"secondaryDisks,omitempty"` } -// PrimaryDisk +// PrimaryDisk. type PrimaryDisk struct { Disk OsImage providerconfigtypes.ConfigVarString `json:"osImage,omitempty"` } -// SecondaryDisks +// SecondaryDisks. type SecondaryDisks struct { Disk } -// Disk +// Disk. type Disk struct { Size providerconfigtypes.ConfigVarString `json:"size,omitempty"` StorageClassName providerconfigtypes.ConfigVarString `json:"storageClassName,omitempty"` } -// Affinity +// Affinity. type Affinity struct { PodAffinityPreset providerconfigtypes.ConfigVarString `json:"podAffinityPreset,omitempty"` PodAntiAffinityPreset providerconfigtypes.ConfigVarString `json:"podAntiAffinityPreset,omitempty"` NodeAffinityPreset NodeAffinityPreset `json:"nodeAffinityPreset,omitempty"` } -// NodeAffinityPreset +// NodeAffinityPreset. type NodeAffinityPreset struct { Type providerconfigtypes.ConfigVarString `json:"type,omitempty"` Key providerconfigtypes.ConfigVarString `json:"key,omitempty"` diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index f6fca7fca..7389756b2 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -50,7 +50,7 @@ type provider struct { configVarResolver *providerconfig.ConfigVarResolver } -// New returns a linode provider +// New returns a linode provider. func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -129,7 +129,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c := Config{} c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, "LINODE_TOKEN") if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"token\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"token\" field, error = %w", err) } c.Region, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Region) if err != nil { @@ -166,7 +166,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to parse config: %v", err) + return fmt.Errorf("failed to parse config: %w", err) } if c.Token == "" { @@ -183,7 +183,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { _, err = getSlugForOS(pc.OperatingSystem) if err != nil { - return fmt.Errorf("invalid operating system specified %q: %v", pc.OperatingSystem, err) + return fmt.Errorf("invalid operating system specified %q: %w", pc.OperatingSystem, err) } ctx := context.TODO() @@ -226,7 +226,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert sshkey, err := ssh.NewKey() if err != nil { - return nil, fmt.Errorf("failed to generate ssh key: %v", err) + return nil, fmt.Errorf("failed to generate ssh key: %w", err) } slug, err := getSlugForOS(pc.OperatingSystem) @@ -277,7 +277,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { instance, err := p.Get(machine, data) if err != nil { - if err == cloudprovidererrors.ErrInstanceNotFound { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return true, nil } return false, err @@ -295,7 +295,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider linodeID, err := strconv.Atoi(instance.ID()) if err != nil { - return false, fmt.Errorf("failed to convert instance id %s to int: %v", instance.ID(), err) + return false, fmt.Errorf("failed to convert instance id %s to int: %w", instance.ID(), err) } err = client.DeleteInstance(ctx, linodeID) @@ -343,26 +343,26 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to decode providerconfig: %v", err) + return fmt.Errorf("failed to decode providerconfig: %w", err) } client := getClient(c.Token) listOptions := getListOptions(machine.Spec.Name) linodes, err := client.ListInstances(ctx, listOptions) if err != nil { - return fmt.Errorf("failed to list linodes: %v", err) + return fmt.Errorf("failed to list linodes: %w", err) } for _, linode := range linodes { if sets.NewString(linode.Tags...).Has(string(machine.UID)) { updateOpts := linode.GetUpdateOptions() - tags := []string{string(new)} + tags := []string{string(newUID)} if updateOpts.Tags != nil { oldUID := string(machine.UID) for _, existingTag := range *updateOpts.Tags { @@ -374,7 +374,7 @@ func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) e updateOpts.Tags = &tags _, err = client.UpdateInstance(ctx, linode.ID, updateOpts) if err != nil { - return fmt.Errorf("failed to revise linode UID tags: %v", err) + return fmt.Errorf("failed to revise linode UID tags: %w", err) } } } @@ -438,10 +438,11 @@ func (d *linodeInstance) Status() instance.Status { // can be qualified as a "terminal" error, for more info see v1alpha1.MachineStatus // if the given error doesn't qualify the error passed as -// an argument will be returned +// an argument will be returned. func linodeStatusAndErrToTerminalError(err error) error { status := 0 - if apiErr, ok := err.(*linodego.Error); ok { + var apiErr *linodego.Error + if errors.As(err, &apiErr) { status = apiErr.Code } diff --git a/pkg/cloudprovider/provider/nutanix/client.go b/pkg/cloudprovider/provider/nutanix/client.go index c57a07187..e337e86b8 100644 --- a/pkg/cloudprovider/provider/nutanix/client.go +++ b/pkg/cloudprovider/provider/nutanix/client.go @@ -159,7 +159,7 @@ func createVM(client *ClientSet, name string, conf Config, os providerconfigtype if conf.ProjectName != "" { project, err := getProjectByName(client, conf.ProjectName) if err != nil { - return nil, fmt.Errorf("failed to get project: %v", err) + return nil, fmt.Errorf("failed to get project: %w", err) } request.Metadata.ProjectReference = &nutanixv3.Reference{ @@ -190,7 +190,7 @@ func createVM(client *ClientSet, name string, conf Config, os providerconfigtype taskUUID := resp.Status.ExecutionContext.TaskUUID.(string) if err := waitForCompletion(client, taskUUID, time.Second*10, time.Minute*15); err != nil { - return nil, fmt.Errorf("failed to wait for task: %v", err) + return nil, fmt.Errorf("failed to wait for task: %w", err) } if resp.Metadata.UUID == nil { @@ -198,7 +198,7 @@ func createVM(client *ClientSet, name string, conf Config, os providerconfigtype } if err := waitForPowerState(client, *resp.Metadata.UUID, time.Second*10, time.Minute*10); err != nil { - return nil, fmt.Errorf("failed to wait for power state: %v", err) + return nil, fmt.Errorf("failed to wait for power state: %w", err) } vm, err := client.Prism.V3.GetVM(*resp.Metadata.UUID) @@ -212,7 +212,7 @@ func createVM(client *ClientSet, name string, conf Config, os providerconfigtype addresses, err := getIPs(client, *vm.Metadata.UUID, time.Second*5, time.Minute*10) if err != nil { - return nil, fmt.Errorf("failed to get addresses: %v", err) + return nil, fmt.Errorf("failed to get addresses: %w", err) } return Server{ @@ -259,7 +259,6 @@ func getProjectByName(client *ClientSet, name string) (*nutanixv3.Project, error Reason: common.InvalidConfigurationMachineError, Message: fmt.Sprintf("no project found for name==%s", name), } - } for _, project := range projects.Entities { @@ -394,7 +393,6 @@ func waitForCompletion(client *ClientSet, taskID string, interval time.Duration, default: return false, fmt.Errorf("unknown status: %s", *task.Status) } - }) } diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 5ad6b0212..6a7cdfe68 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -198,32 +198,32 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { config, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to parse machineSpec: %v", err) + return fmt.Errorf("failed to parse machineSpec: %w", err) } client, err := GetClientSet(config) if err != nil { - return fmt.Errorf("failed to construct client: %v", err) + return fmt.Errorf("failed to construct client: %w", err) } cluster, err := getClusterByName(client, config.ClusterName) if err != nil { - return fmt.Errorf("failed to get cluster: %v", err) + return fmt.Errorf("failed to get cluster: %w", err) } if config.ProjectName != "" { if _, err := getProjectByName(client, config.ProjectName); err != nil { - return fmt.Errorf("failed to get project: %v", err) + return fmt.Errorf("failed to get project: %w", err) } } if _, err := getSubnetByName(client, config.SubnetName, *cluster.Metadata.UUID); err != nil { - return fmt.Errorf("failed to get subnet: %v", err) + return fmt.Errorf("failed to get subnet: %w", err) } image, err := getImageByName(client, config.ImageName) if err != nil { - return fmt.Errorf("failed to get image: %v", err) + return fmt.Errorf("failed to get image: %w", err) } var imageSizeBytes int64 @@ -246,7 +246,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert if err != nil { _, cleanupErr := p.Cleanup(machine, data) if cleanupErr != nil { - return nil, fmt.Errorf("cleaning up failed with err %v after creation failed with err %v", cleanupErr, err) + return nil, fmt.Errorf("cleaning up failed with err %v after creation failed with err %w", cleanupErr, err) } return nil, err } @@ -307,7 +307,7 @@ func (p *provider) cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider vm, err := getVMByName(client, machine.Name, projectID) if err != nil { - if err == cloudprovidererrors.ErrInstanceNotFound { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { // VM is gone already return true, nil } @@ -332,7 +332,7 @@ func (p *provider) cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider } if err := waitForCompletion(client, taskID, time.Second*5, time.Minute*10); err != nil { - return false, fmt.Errorf("failed to wait for completion: %v", err) + return false, fmt.Errorf("failed to wait for completion: %w", err) } return true, nil @@ -403,11 +403,11 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertype }, nil } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new ktypes.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, uid ktypes.UID) error { return nil } -// GetCloudConfig returns an empty cloud configuration for Nutanix as no CCM exists +// GetCloudConfig returns an empty cloud configuration for Nutanix as no CCM exists. func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } @@ -417,7 +417,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { - return labels, fmt.Errorf("failed to parse config: %v", err) + return labels, fmt.Errorf("failed to parse config: %w", err) } labels["size"] = fmt.Sprintf("%d-cpus-%d-mb", config.CPUs, config.MemoryMB) diff --git a/pkg/cloudprovider/provider/openstack/helper.go b/pkg/cloudprovider/provider/openstack/helper.go index 05901f1bc..b7475fd2d 100644 --- a/pkg/cloudprovider/provider/openstack/helper.go +++ b/pkg/cloudprovider/provider/openstack/helper.go @@ -88,7 +88,7 @@ func getNewComputeV2(client *gophercloud.ProviderClient, c *Config) (*gopherclou // Validation - empty value default to microversion 2.0=2.1 version, err := strconv.ParseFloat(c.ComputeAPIVersion, 32) if err != nil || version < 2.0 { - return nil, fmt.Errorf("invalid computeAPIVersion: %v", err) + return nil, fmt.Errorf("invalid computeAPIVersion: %w", err) } // See https://github.com/gophercloud/gophercloud/blob/master/docs/MICROVERSIONS.md @@ -277,7 +277,7 @@ func ensureKubernetesSecurityGroupExist(client *gophercloud.ProviderClient, regi _, err = getSecurityGroup(client, region, name) if err != nil { - if err == errNotFound { + if errors.Is(err, errNotFound) { sg, err := ossecuritygroups.Create(netClient, ossecuritygroups.CreateOpts{Name: name}).Extract() if err != nil { return osErrorToTerminalError(err, fmt.Sprintf("failed to create security group %s", name)) @@ -385,7 +385,7 @@ NetworkLoop: for _, network := range networks { for _, subnet := range network.Subnets { _, err := getSubnet(netClient, subnet) - if err == errNotFound { + if errors.Is(err, errNotFound) { continue } else if err != nil { return nil, err diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index b5dc24fe0..16deef23d 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -70,7 +70,7 @@ type provider struct { portReadinessWaiter portReadinessWaiterFunc } -// New returns a openstack provider +// New returns a openstack provider. func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{ configVarResolver: configVarResolver, @@ -118,28 +118,28 @@ const ( ovhAuthURL = "auth.cloud.ovh.net" ) -// Protects floating ip assignment +// Protects floating ip assignment. var floatingIPAssignLock = &sync.Mutex{} -// Get the Project name from config or env var. If not defined fallback to tenant name +// Get the Project name from config or env var. If not defined fallback to tenant name. func (p *provider) getProjectNameOrTenantName(rawConfig *openstacktypes.RawConfig) (string, error) { projectName, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProjectName, "OS_PROJECT_NAME") if err == nil && len(projectName) > 0 { return projectName, nil } - //fallback to tenantName + //fallback to tenantName. return p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.TenantName, "OS_TENANT_NAME") } -// Get the Project id from config or env var. If not defined fallback to tenant id +// Get the Project id from config or env var. If not defined fallback to tenant id. func (p *provider) getProjectIDOrTenantID(rawConfig *openstacktypes.RawConfig) (string, error) { projectID, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProjectID, "OS_PROJECT_ID") if err == nil && len(projectID) > 0 { return projectID, nil } - //fallback to tenantName + //fallback to tenantName. return p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.TenantID, "OS_TENANT_ID") } @@ -147,31 +147,31 @@ func (p *provider) getConfigAuth(c *Config, rawConfig *openstacktypes.RawConfig) var err error c.ApplicationCredentialID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ApplicationCredentialID, "OS_APPLICATION_CREDENTIAL_ID") if err != nil { - return fmt.Errorf("failed to get the value of \"applicationCredentialID\" field, error = %v", err) + return fmt.Errorf("failed to get the value of \"applicationCredentialID\" field, error = %w", err) } if c.ApplicationCredentialID != "" { klog.V(6).Infof("applicationCredentialID from configuration or environment was found.") c.ApplicationCredentialSecret, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ApplicationCredentialSecret, "OS_APPLICATION_CREDENTIAL_SECRET") if err != nil { - return fmt.Errorf("failed to get the value of \"applicationCredentialSecret\" field, error = %v", err) + return fmt.Errorf("failed to get the value of \"applicationCredentialSecret\" field, error = %w", err) } return nil } c.Username, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Username, "OS_USER_NAME") if err != nil { - return fmt.Errorf("failed to get the value of \"username\" field, error = %v", err) + return fmt.Errorf("failed to get the value of \"username\" field, error = %w", err) } c.Password, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Password, "OS_PASSWORD") if err != nil { - return fmt.Errorf("failed to get the value of \"password\" field, error = %v", err) + return fmt.Errorf("failed to get the value of \"password\" field, error = %w", err) } c.ProjectName, err = p.getProjectNameOrTenantName(rawConfig) if err != nil { - return fmt.Errorf("failed to get the value of \"projectName\" field or fallback to \"tenantName\" field, error = %v", err) + return fmt.Errorf("failed to get the value of \"projectName\" field or fallback to \"tenantName\" field, error = %w", err) } c.ProjectID, err = p.getProjectIDOrTenantID(rawConfig) if err != nil { - return fmt.Errorf("failed to get the value of \"projectID\" or fallback to\"tenantID\" field, error = %v", err) + return fmt.Errorf("failed to get the value of \"projectID\" or fallback to\"tenantID\" field, error = %w", err) } return nil } @@ -198,16 +198,16 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p cfg := Config{} cfg.IdentityEndpoint, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.IdentityEndpoint, "OS_AUTH_URL") if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"identityEndpoint\" field, error = %v", err) + return nil, nil, nil, fmt.Errorf("failed to get the value of \"identityEndpoint\" field, error = %w", err) } - // Retrieve authentication config, username/password or application credentials + // Retrieve authentication config, username/password or application credentials. err = p.getConfigAuth(&cfg, rawConfig) if err != nil { - return nil, nil, nil, fmt.Errorf("failed to retrieve authentication credentials, error = %v", err) + return nil, nil, nil, fmt.Errorf("failed to retrieve authentication credentials, error = %w", err) } - // Ignore Region not found as Region might not be found and we can default it later + // Ignore Region not found as Region might not be found and we can default it later. cfg.Region, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Region, "OS_REGION_NAME") if err != nil { klog.V(6).Infof("Region from configuration or environment variable not found") @@ -215,15 +215,15 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p cfg.InstanceReadyCheckPeriod, err = p.configVarResolver.GetConfigVarDurationValueOrDefault(rawConfig.InstanceReadyCheckPeriod, 5*time.Second) if err != nil { - return nil, nil, nil, fmt.Errorf(`failed to get the value of "InstanceReadyCheckPeriod" field, error = %v`, err) + return nil, nil, nil, fmt.Errorf(`failed to get the value of "InstanceReadyCheckPeriod" field, error = %w`, err) } cfg.InstanceReadyCheckTimeout, err = p.configVarResolver.GetConfigVarDurationValueOrDefault(rawConfig.InstanceReadyCheckTimeout, 10*time.Second) if err != nil { - return nil, nil, nil, fmt.Errorf(`failed to get the value of "InstanceReadyCheckTimeout" field, error = %v`, err) + return nil, nil, nil, fmt.Errorf(`failed to get the value of "InstanceReadyCheckTimeout" field, error = %w`, err) } - // We ignore errors here because the OS domain is only required when using Identity API V3 + // We ignore errors here because the OS domain is only required when using Identity API V3. cfg.DomainName, _ = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.DomainName, "OS_DOMAIN_NAME") cfg.TokenID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.TokenID) if err != nil { @@ -328,7 +328,7 @@ func getClient(c *Config) (*gophercloud.ProviderClient, error) { Username: c.Username, Password: c.Password, DomainName: c.DomainName, - // gophercloud internally store projectName/projectID under tenantName/TenantID. We store it under projectName + // gophercloud internally store projectName/projectID under tenantName/TenantID. We store it under projectName. // to be coherent with KPP code TenantName: c.ProjectName, TenantID: c.ProjectID, @@ -342,7 +342,7 @@ func getClient(c *Config) (*gophercloud.ProviderClient, error) { return nil, err } if pc != nil { - // use the util's HTTP client to benefit, among other things, from its CA bundle + // use the util's HTTP client to benefit, among other things, from its CA bundle. pc.HTTPClient = cloudproviderutil.HTTPClientConfig{LogPrefix: "[OpenStack API]"}.New() } @@ -443,7 +443,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, pc, _, err := p.getConfig(spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to parse config: %v", err) + return fmt.Errorf("failed to parse config: %w", err) } if c.ApplicationCredentialID == "" { @@ -477,31 +477,31 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { client, err := p.clientGetter(c) if err != nil { - return fmt.Errorf("failed to get a openstack client: %v", err) + return fmt.Errorf("failed to get a openstack client: %w", err) } - // Required fields + // Required fields. if !strings.Contains(c.IdentityEndpoint, ovhAuthURL) { if _, err := getRegion(client, c.Region); err != nil { - return fmt.Errorf("failed to get region %q: %v", c.Region, err) + return fmt.Errorf("failed to get region %q: %w", c.Region, err) } } // Get OS Compute Client computeClient, err := getNewComputeV2(client, c) if err != nil { - return fmt.Errorf("failed to get compute client: %v", err) + return fmt.Errorf("failed to get compute client: %w", err) } - // Get OS Image Client + // Get OS Image Client. imageClient, err := goopenstack.NewImageServiceV2(client, gophercloud.EndpointOpts{Region: c.Region}) if err != nil { - return fmt.Errorf("failed to get image client: %v", err) + return fmt.Errorf("failed to get image client: %w", err) } image, err := getImageByName(imageClient, c) if err != nil { - return fmt.Errorf("failed to get image %q: %v", c.Image, err) + return fmt.Errorf("failed to get image %q: %w", c.Image, err) } if c.RootDiskSizeGB != nil { if *c.RootDiskSizeGB < image.MinDiskGigabytes { @@ -511,7 +511,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { } if _, err := getFlavor(computeClient, c); err != nil { - return fmt.Errorf("failed to get flavor %q: %v", c.Flavor, err) + return fmt.Errorf("failed to get flavor %q: %w", c.Flavor, err) } netClient, err := goopenstack.NewNetworkV2(client, gophercloud.EndpointOpts{Region: c.Region}) @@ -520,35 +520,35 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { } if _, err := getNetwork(netClient, c.Network); err != nil { - return fmt.Errorf("failed to get network %q: %v", c.Network, err) + return fmt.Errorf("failed to get network %q: %w", c.Network, err) } if _, err := getSubnet(netClient, c.Subnet); err != nil { - return fmt.Errorf("failed to get subnet %q: %v", c.Subnet, err) + return fmt.Errorf("failed to get subnet %q: %w", c.Subnet, err) } if c.FloatingIPPool != "" { if _, err := getNetwork(netClient, c.FloatingIPPool); err != nil { - return fmt.Errorf("failed to get floating ip pool %q: %v", c.FloatingIPPool, err) + return fmt.Errorf("failed to get floating ip pool %q: %w", c.FloatingIPPool, err) } } if _, err := getAvailabilityZone(computeClient, c); err != nil { - return fmt.Errorf("failed to get availability zone %q: %v", c.AvailabilityZone, err) + return fmt.Errorf("failed to get availability zone %q: %w", c.AvailabilityZone, err) } if pc.OperatingSystem == providerconfigtypes.OperatingSystemSLES { - return fmt.Errorf("invalid/not supported operating system specified %q: %v", pc.OperatingSystem, providerconfigtypes.ErrOSNotSupported) + return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, providerconfigtypes.ErrOSNotSupported) } - // Optional fields + // Optional fields. if len(c.SecurityGroups) != 0 { for _, s := range c.SecurityGroups { if _, err := getSecurityGroup(client, c.Region, s); err != nil { - return fmt.Errorf("failed to get security group %q: %v", s, err) + return fmt.Errorf("failed to get security group %q: %w", s, err) } } } - // validate reserved tags + // validate reserved tags. if _, ok := c.Tags[machineUIDMetaKey]; ok { return fmt.Errorf("the tag with the given name =%s is reserved, choose a different one", machineUIDMetaKey) } @@ -580,7 +580,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return nil, osErrorToTerminalError(err, fmt.Sprintf("failed to get flavor %s", cfg.Flavor)) } - // Get OS Image Client + // Get OS Image Client. imageClient, err := goopenstack.NewImageServiceV2(client, gophercloud.EndpointOpts{Region: cfg.Region}) if err != nil { return nil, osErrorToTerminalError(err, "failed to get a image client") @@ -606,12 +606,12 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert klog.V(2).Infof("creating security group %s for worker nodes", securityGroupName) err = ensureKubernetesSecurityGroupExist(client, cfg.Region, securityGroupName) if err != nil { - return nil, fmt.Errorf("Error occurred creating security groups: %v", err) + return nil, fmt.Errorf("Error occurred creating security groups: %w", err) } securityGroups = append(securityGroups, securityGroupName) } - // we check against reserved tags in Validation method + // we check against reserved tags in Validation method. allTags := cfg.Tags allTags[machineUIDMetaKey] = string(machine.UID) @@ -661,7 +661,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert } else { // Image ID should only be set in server options when block device // mapping is not used. Otherwish an error may occur with some - // OpenStack providers/versions .e.g. OpenTelekom Cloud + // OpenStack providers/versions .e.g. OpenTelekom Cloud. serverOpts.ImageRef = image.ID if err := osservers.Create(computeClient, createOpts).ExtractInto(&server); err != nil { @@ -674,7 +674,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert klog.V(2).Infof("port for instance %q did not became active due to: %v", server.ID, err) } - // Find a free FloatingIP or allocate a new one + // Find a free FloatingIP or allocate a new one. if err := assignFloatingIPToInstance(data.Update, machine, netClient, server.ID, cfg.FloatingIPPool, cfg.Region, network); err != nil { defer deleteInstanceDueToFatalLogged(computeClient, server.ID) return nil, fmt.Errorf("failed to assign a floating ip to instance %s: %w", server.ID, err) @@ -695,7 +695,7 @@ func waitForPort(netClient *gophercloud.ServiceClient, serverID string, networkI if isTerminalErr, _, _ := cloudprovidererrors.IsTerminalError(tErr); isTerminalErr { return true, tErr } - // Only log the error but don't exit. in case of a network failure we want to retry + // Only log the error but don't exit. in case of a network failure we want to retry. klog.V(2).Infof("failed to get current instance port %s: %v", serverID, err) return false, nil } @@ -704,12 +704,12 @@ func waitForPort(netClient *gophercloud.ServiceClient, serverID string, networkI } if err := wait.Poll(checkPeriod, checkTimeout, portIsReady); err != nil { - if err == wait.ErrWaitTimeout { + if errors.Is(err, wait.ErrWaitTimeout) { // In case we have a timeout, include the timeout details return fmt.Errorf("instance port became not active after %f seconds", checkTimeout.Seconds()) } // Some terminal error happened - return fmt.Errorf("failed to wait for instance port to become active: %v", err) + return fmt.Errorf("failed to wait for instance port to become active: %w", err) } klog.V(2).Infof("Instance %q port became active after %f seconds", serverID, time.Since(started).Seconds()) @@ -719,7 +719,7 @@ func waitForPort(netClient *gophercloud.ServiceClient, serverID string, networkI func deleteInstanceDueToFatalLogged(computeClient *gophercloud.ServiceClient, serverID string) { klog.V(0).Infof("Deleting instance %s due to fatal error during machine creation...", serverID) if err := osservers.Delete(computeClient, serverID).ExtractErr(); err != nil { - utilruntime.HandleError(fmt.Errorf("failed to delete the instance %s. Please take care of manually deleting the instance: %v", serverID, err)) + utilruntime.HandleError(fmt.Errorf("failed to delete the instance %s. Please take care of manually deleting the instance: %w", serverID, err)) return } klog.V(0).Infof("Instance %s got deleted", serverID) @@ -733,10 +733,10 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider instance, err := p.Get(machine, data) if err != nil { - if err == cloudprovidererrors.ErrInstanceNotFound { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { if hasFloatingIPReleaseFinalizer { if err := p.cleanupFloatingIP(machine, data.Update); err != nil { - return false, fmt.Errorf("failed to clean up floating ip: %v", err) + return false, fmt.Errorf("failed to clean up floating ip: %w", err) } } return true, nil @@ -816,7 +816,7 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return cloudprovidererrors.TerminalError{ @@ -853,10 +853,10 @@ func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) e for _, s := range allServers { if s.Metadata[machineUIDMetaKey] == string(machine.UID) { metadataOpts := osservers.MetadataOpts(s.Metadata) - metadataOpts[machineUIDMetaKey] = string(new) + metadataOpts[machineUIDMetaKey] = string(newUID) response := osservers.UpdateMetadata(computeClient, s.ID, metadataOpts) if response.Err != nil { - return fmt.Errorf("failed to update instance metadata with new UID: %v", err) + return fmt.Errorf("failed to update instance metadata with new UID: %w", err) } } } @@ -867,7 +867,7 @@ func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) e func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { c, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { - return "", "", fmt.Errorf("failed to parse config: %v", err) + return "", "", fmt.Errorf("failed to parse config: %w", err) } cc := &openstacktypes.CloudConfig{ @@ -898,7 +898,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri s, err := openstacktypes.CloudConfigToString(cc) if err != nil { - return "", "", fmt.Errorf("failed to convert the cloud-config to string: %v", err) + return "", "", fmt.Errorf("failed to convert the cloud-config to string: %w", err) } return s, "openstack", nil } @@ -959,16 +959,18 @@ func (d *osInstance) Status() instance.Status { // osErrorToTerminalError judges if the given error // can be qualified as a "terminal" error, for more info see v1alpha1.MachineStatus // -// if the given error doesn't qualify the error passed as an argument will be returned +// if the given error doesn't qualify the error passed as an argument will be returned. func osErrorToTerminalError(err error, msg string) error { - if errUnauthorized, ok := err.(gophercloud.ErrDefault401); ok { + var errUnauthorized gophercloud.ErrDefault401 + if errors.As(err, &errUnauthorized) { return cloudprovidererrors.TerminalError{ Reason: common.InvalidConfigurationMachineError, Message: fmt.Sprintf("A request has been rejected due to invalid credentials which were taken from the MachineSpec: %v", errUnauthorized), } } - if errForbidden, ok := err.(gophercloud.ErrDefault403); ok { + var errForbidden gophercloud.ErrDefault403 + if errors.As(err, &errForbidden) { terr := cloudprovidererrors.TerminalError{ Reason: common.InvalidConfigurationMachineError, Message: fmt.Sprintf("%s. The request against the OpenStack API is forbidden: %s", msg, errForbidden.Error()), @@ -993,10 +995,10 @@ func osErrorToTerminalError(err error, msg string) error { return terr } - return fmt.Errorf("%s, due to %s", msg, err) + return fmt.Errorf("%s, due to %w", msg, err) } -// forbiddenResponse is a potential response body from the OpenStack API when the request is forbidden (code: 403) +// forbiddenResponse is a potential response body from the OpenStack API when the request is forbidden (code: 403). type forbiddenResponse struct { Forbidden struct { Message string `json:"message"` @@ -1025,17 +1027,17 @@ func (p *provider) cleanupFloatingIP(machine *clusterv1alpha1.Machine, updater c } netClient, err := goopenstack.NewNetworkV2(client, gophercloud.EndpointOpts{Region: c.Region}) if err != nil { - return fmt.Errorf("failed to create the networkv2 client for region %s: %v", c.Region, err) + return fmt.Errorf("failed to create the networkv2 client for region %s: %w", c.Region, err) } if err := osfloatingips.Delete(netClient, floatingIPID).ExtractErr(); err != nil && err.Error() != "Resource not found" { - return fmt.Errorf("failed to delete floating ip %s: %v", floatingIPID, err) + return fmt.Errorf("failed to delete floating ip %s: %w", floatingIPID, err) } if err := updater(machine, func(m *clusterv1alpha1.Machine) { finalizers := sets.NewString(m.Finalizers...) finalizers.Delete(floatingIPReleaseFinalizer) m.Finalizers = finalizers.List() }); err != nil { - return fmt.Errorf("failed to delete %s finalizer from Machine: %v", floatingIPReleaseFinalizer, err) + return fmt.Errorf("failed to delete %s finalizer from Machine: %w", floatingIPReleaseFinalizer, err) } return nil @@ -1044,7 +1046,7 @@ func (p *provider) cleanupFloatingIP(machine *clusterv1alpha1.Machine, updater c func assignFloatingIPToInstance(machineUpdater cloudprovidertypes.MachineUpdater, machine *clusterv1alpha1.Machine, netClient *gophercloud.ServiceClient, instanceID, floatingIPPoolName, region string, network *osnetworks.Network) error { port, err := getInstancePort(netClient, instanceID, network.ID) if err != nil { - return fmt.Errorf("failed to get instance port for network %s in region %s: %v", network.ID, region, err) + return fmt.Errorf("failed to get instance port for network %s in region %s: %w", network.ID, region, err) } floatingIPPool, err := getNetwork(netClient, floatingIPPoolName) @@ -1076,7 +1078,7 @@ func assignFloatingIPToInstance(machineUpdater cloudprovidertypes.MachineUpdater } m.Annotations[floatingIPIDAnnotationKey] = ip.ID }); err != nil { - return fmt.Errorf("failed to add floating ip release finalizer after allocating floating ip: %v", err) + return fmt.Errorf("failed to add floating ip release finalizer after allocating floating ip: %w", err) } } else { freeIP := freeFloatingIps[0] @@ -1084,7 +1086,7 @@ func assignFloatingIPToInstance(machineUpdater cloudprovidertypes.MachineUpdater PortID: &port.ID, }).Extract() if err != nil { - return fmt.Errorf("failed to update FloatingIP %s(%s): %v", freeIP.ID, freeIP.FloatingIP, err) + return fmt.Errorf("failed to update FloatingIP %s(%s): %w", freeIP.ID, freeIP.FloatingIP, err) } // We're now going to wait 3 seconds and check if the IP is still ours. If not, we're going to fail @@ -1092,7 +1094,7 @@ func assignFloatingIPToInstance(machineUpdater cloudprovidertypes.MachineUpdater time.Sleep(floatingReassignIPCheckPeriod) currentIP, err := osfloatingips.Get(netClient, ip.ID).Extract() if err != nil { - return fmt.Errorf("failed to load FloatingIP %s after assignment has been done: %v", ip.FloatingIP, err) + return fmt.Errorf("failed to load FloatingIP %s after assignment has been done: %w", ip.FloatingIP, err) } // Verify if the port is still the one we set it to if currentIP.PortID != port.ID { diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index 8bebddd7e..249815af6 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -38,7 +38,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/pointer" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) const expectedServerRequest = `{ @@ -270,7 +270,7 @@ func TestCreateServer(t *testing.T) { ExpectServerCreated(t, tt.wantServerReq) p := &provider{ // Note that configVarResolver is not used in this test as the getConfigFunc is mocked. - configVarResolver: providerconfig.NewConfigVarResolver(context.Background(), fakeclient.NewFakeClient()), + configVarResolver: providerconfig.NewConfigVarResolver(context.Background(), fakectrlruntimeclient.NewClientBuilder().Build()), // mock client config getter clientGetter: func(c *Config) (*gophercloud.ProviderClient, error) { pc := client.ServiceClient() @@ -334,7 +334,9 @@ func TestProjectAuthVarsAreCorrectlyLoaded(t *testing.T) { t.Run(tt.name, func(t *testing.T) { p := &provider{ // Note that configVarResolver is not used in this test as the getConfigFunc is mocked. - configVarResolver: providerconfig.NewConfigVarResolver(context.Background(), fakeclient.NewFakeClient()), + configVarResolver: providerconfig.NewConfigVarResolver(context.Background(), fakectrlruntimeclient. + NewClientBuilder(). + Build()), } conf, _, _, _ := p.getConfig(v1alpha1.ProviderSpec{ Value: &runtime.RawExtension{ @@ -383,7 +385,7 @@ func ExpectServerCreated(t *testing.T, expectedServer string) { w.WriteHeader(http.StatusAccepted) w.Header().Add("Content-Type", "application/json") - fmt.Fprintf(w, string(srvRes)) + fmt.Fprint(w, string(srvRes)) }) th.Mux.HandleFunc("/os-volumes_boot", func(w http.ResponseWriter, r *http.Request) { th.TestMethod(t, r, "POST") @@ -392,7 +394,7 @@ func ExpectServerCreated(t *testing.T, expectedServer string) { w.WriteHeader(http.StatusAccepted) w.Header().Add("Content-Type", "application/json") - fmt.Fprintf(w, string(srvRes)) + fmt.Fprint(w, string(srvRes)) }) // Handle listing images v2. diff --git a/pkg/cloudprovider/provider/openstack/types/cloudconfig.go b/pkg/cloudprovider/provider/openstack/types/cloudconfig.go index cdcc64121..83899c292 100644 --- a/pkg/cloudprovider/provider/openstack/types/cloudconfig.go +++ b/pkg/cloudprovider/provider/openstack/types/cloudconfig.go @@ -117,7 +117,7 @@ type GlobalOpts struct { Region string } -// CloudConfig is used to read and store information from the cloud configuration file +// CloudConfig is used to read and store information from the cloud configuration file. type CloudConfig struct { Global GlobalOpts LoadBalancer LoadBalancerOpts @@ -132,12 +132,12 @@ func CloudConfigToString(c *CloudConfig) (string, error) { tpl, err := template.New("cloud-config").Funcs(funcMap).Parse(cloudConfigTpl) if err != nil { - return "", fmt.Errorf("failed to parse the cloud config template: %v", err) + return "", fmt.Errorf("failed to parse the cloud config template: %w", err) } buf := &bytes.Buffer{} if err := tpl.Execute(buf, c); err != nil { - return "", fmt.Errorf("failed to execute cloud config template: %v", err) + return "", fmt.Errorf("failed to execute cloud config template: %w", err) } return buf.String(), nil diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go index b506210d6..4e66e32a3 100644 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ b/pkg/cloudprovider/provider/scaleway/provider.go @@ -45,7 +45,7 @@ type provider struct { configVarResolver *providerconfig.ConfigVarResolver } -// New returns a Scaleway provider +// New returns a Scaleway provider. func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -68,7 +68,7 @@ func (c *Config) getInstanceAPI() (*instance.API, error) { scw.WithUserAgent("kubermatic/machine-controller"), ) if err != nil { - return nil, fmt.Errorf("failed to initialize the scaleway client: %s", err.Error()) + return nil, fmt.Errorf("failed to initialize the scaleway client: %w", err) } return instance.NewAPI(client), nil @@ -108,19 +108,19 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c := Config{} c.AccessKey, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AccessKey, scw.ScwAccessKeyEnv) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"access_key\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"access_key\" field, error = %w", err) } c.SecretKey, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.SecretKey, scw.ScwSecretKeyEnv) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"secret_key\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"secret_key\" field, error = %w", err) } c.ProjectID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ProjectID) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"project_id\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"project_id\" field, error = %w", err) } c.Zone, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Zone) if err != nil { - return nil, nil, fmt.Errorf("failed to get the value of \"zone\" field, error = %v", err) + return nil, nil, fmt.Errorf("failed to get the value of \"zone\" field, error = %w", err) } c.CommercialType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.CommercialType) if err != nil { @@ -142,7 +142,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to parse config: %v", err) + return fmt.Errorf("failed to parse config: %w", err) } if !validation.IsAccessKey(c.AccessKey) { @@ -166,7 +166,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { _, err = getImageNameForOS(pc.OperatingSystem) if err != nil { - return fmt.Errorf("invalid operating system specified %q: %v", pc.OperatingSystem, err) + return fmt.Errorf("invalid operating system specified %q: %w", pc.OperatingSystem, err) } return nil @@ -224,7 +224,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { i, err := p.get(machine) if err != nil { - if err == cloudprovidererrors.ErrInstanceNotFound { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return true, nil } return false, err @@ -319,10 +319,10 @@ func (p *provider) get(machine *clusterv1alpha1.Machine) (*scwServer, error) { return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to decode providerconfig: %v", err) + return fmt.Errorf("failed to decode providerconfig: %w", err) } api, err := c.getInstanceAPI() if err != nil { @@ -335,7 +335,7 @@ func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) e } oldTags := server.server.Tags - newTags := []string{string(new)} + newTags := []string{string(newUID)} for _, oldTag := range oldTags { if oldTag != string(machine.UID) { newTags = append(newTags, oldTag) @@ -415,32 +415,35 @@ func (s *scwServer) Status() cloudInstance.Status { // can be qualified as a "terminal" error, for more info see v1alpha1.MachineStatus // if the given error doesn't qualify the error passed as -// an argument will be returned +// an argument will be returned. func scalewayErrToTerminalError(err error) error { - switch err.(type) { - case *scw.PermissionsDeniedError: + var deinedErr *scw.PermissionsDeniedError + var invalidArgErr *scw.InvalidArgumentsError + var outOfStackErr *scw.OutOfStockError + var quotaErr *scw.QuotasExceededError + + if errors.As(err, &deinedErr) { return cloudprovidererrors.TerminalError{ Reason: common.InvalidConfigurationMachineError, Message: "A request has been rejected due to invalid credentials which were taken from the MachineSpec", } - case *scw.InvalidArgumentsError: + } else if errors.As(err, &invalidArgErr) { return cloudprovidererrors.TerminalError{ Reason: common.InvalidConfigurationMachineError, Message: "A request has been rejected due to invalid arguments which were taken from the MachineSpec", } - case *scw.OutOfStockError: + } else if errors.As(err, &outOfStackErr) { return cloudprovidererrors.TerminalError{ Reason: common.InsufficientResourcesMachineError, Message: "A request has been rejected due to out of stocks", } - case *scw.QuotasExceededError: + } else if errors.As(err, "aErr) { return cloudprovidererrors.TerminalError{ Reason: common.InsufficientResourcesMachineError, Message: "A request has been rejected due to insufficient quotas", } - default: - return err } + return err } func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { diff --git a/pkg/cloudprovider/provider/vsphere/client.go b/pkg/cloudprovider/provider/vsphere/client.go index b5e30afd0..1d2d00b1d 100644 --- a/pkg/cloudprovider/provider/vsphere/client.go +++ b/pkg/cloudprovider/provider/vsphere/client.go @@ -60,13 +60,13 @@ func NewSession(ctx context.Context, config *Config) (*Session, error) { } if err = client.Login(ctx, url.UserPassword(config.Username, config.Password)); err != nil { - return nil, fmt.Errorf("failed vsphere login: %v", err) + return nil, fmt.Errorf("failed vsphere login: %w", err) } finder := find.NewFinder(client.Client, true) dc, err := finder.Datacenter(ctx, config.Datacenter) if err != nil { - return nil, fmt.Errorf("failed to get vsphere datacenter: %v", err) + return nil, fmt.Errorf("failed to get vsphere datacenter: %w", err) } finder.SetDatacenter(dc) @@ -77,10 +77,10 @@ func NewSession(ctx context.Context, config *Config) (*Session, error) { }, nil } -// Logout closes the idling vCenter connections +// Logout closes the idling vCenter connections. func (s *Session) Logout() { if err := s.Client.Logout(context.Background()); err != nil { - utilruntime.HandleError(fmt.Errorf("vsphere client failed to logout: %s", err)) + utilruntime.HandleError(fmt.Errorf("vsphere client failed to logout: %w", err)) } } @@ -101,10 +101,10 @@ func NewRESTSession(ctx context.Context, config *Config) (*RESTSession, error) { }, nil } -// Logout closes the idling vCenter connections +// Logout closes the idling vCenter connections. func (s *RESTSession) Logout(ctx context.Context) { if err := s.Client.Logout(ctx); err != nil { - utilruntime.HandleError(fmt.Errorf("vsphere REST client failed to logout: %s", err)) + utilruntime.HandleError(fmt.Errorf("vsphere REST client failed to logout: %w", err)) } } diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go index d69297451..86ec839d7 100644 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ b/pkg/cloudprovider/provider/vsphere/helper.go @@ -49,7 +49,7 @@ local-hostname: {{ .Hostname }}` func createClonedVM(ctx context.Context, vmName string, config *Config, session *Session, os providerconfigtypes.OperatingSystem, containerLinuxUserdata string) (*object.VirtualMachine, error) { tpl, err := session.Finder.VirtualMachine(ctx, config.TemplateVMName) if err != nil { - return nil, fmt.Errorf("failed to get template vm: %v", err) + return nil, fmt.Errorf("failed to get template vm: %w", err) } // Find the target folder, if its included in the provider config. @@ -62,13 +62,13 @@ func createClonedVM(ctx context.Context, vmName string, config *Config, session // The target folder must already exist. targetVMFolder, err = session.Finder.Folder(ctx, config.Folder) if err != nil { - return nil, fmt.Errorf("failed to get target folder: %v", err) + return nil, fmt.Errorf("failed to get target folder: %w", err) } } else { // Do not query datacenter folders unless required datacenterFolders, err := session.Datacenter.Folders(ctx) if err != nil { - return nil, fmt.Errorf("failed to get datacenter folders: %v", err) + return nil, fmt.Errorf("failed to get datacenter folders: %w", err) } targetVMFolder = datacenterFolders.VmFolder } @@ -85,12 +85,12 @@ func createClonedVM(ctx context.Context, vmName string, config *Config, session } datastoreref, err := resolveDatastoreRef(ctx, config, session, tpl, targetVMFolder, &cloneSpec) if err != nil { - return nil, fmt.Errorf("failed to resolve datastore: %v", err) + return nil, fmt.Errorf("failed to resolve datastore: %w", err) } resourcepoolref, err := resolveResourcePoolRef(ctx, config, session, tpl) if err != nil { - return nil, fmt.Errorf("failed to resolve resourcePool: %v", err) + return nil, fmt.Errorf("failed to resolve resourcePool: %w", err) } cloneSpec.Location.Datastore = datastoreref @@ -100,21 +100,21 @@ func createClonedVM(ctx context.Context, vmName string, config *Config, session // It's nicer to tell which specific action failed due to lacking permissions. clonedVMTask, err := tpl.Clone(ctx, targetVMFolder, vmName, cloneSpec) if err != nil { - return nil, fmt.Errorf("failed to clone template vm: %v", err) + return nil, fmt.Errorf("failed to clone template vm: %w", err) } if err := clonedVMTask.Wait(ctx); err != nil { - return nil, fmt.Errorf("error when waiting for result of clone task: %v", err) + return nil, fmt.Errorf("error when waiting for result of clone task: %w", err) } virtualMachine, err := session.Finder.VirtualMachine(ctx, vmName) if err != nil { - return nil, fmt.Errorf("failed to get virtual machine object after cloning: %v", err) + return nil, fmt.Errorf("failed to get virtual machine object after cloning: %w", err) } vmDevices, err := virtualMachine.Device(ctx) if err != nil { - return nil, fmt.Errorf("failed to list devices of template VM: %v", err) + return nil, fmt.Errorf("failed to list devices of template VM: %w", err) } var vAppAconfig *types.VmConfigSpec @@ -126,7 +126,7 @@ func createClonedVM(ctx context.Context, vmName string, config *Config, session // which we'll extract from that template. var mvm mo.VirtualMachine if err := virtualMachine.Properties(ctx, virtualMachine.Reference(), []string{"config", "config.vAppConfig", "config.vAppConfig.property"}, &mvm); err != nil { - return nil, fmt.Errorf("failed to extract vapp properties for flatcar: %v", err) + return nil, fmt.Errorf("failed to extract vapp properties for flatcar: %w", err) } var propertySpecs []types.VAppPropertySpec @@ -178,7 +178,7 @@ func createClonedVM(ctx context.Context, vmName string, config *Config, session if config.DiskSizeGB != nil { disks, err := getDisksFromVM(ctx, virtualMachine) if err != nil { - return nil, fmt.Errorf("failed to get disks from VM: %v", err) + return nil, fmt.Errorf("failed to get disks from VM: %w", err) } // If this is wrong, the resulting error is `Invalid operation for device '0` // so verify again this is legit @@ -196,7 +196,7 @@ func createClonedVM(ctx context.Context, vmName string, config *Config, session if config.VMNetName != "" { networkSpecs, err := GetNetworkSpecs(ctx, session, vmDevices, config.VMNetName) if err != nil { - return nil, fmt.Errorf("failed to get network specifications: %v", err) + return nil, fmt.Errorf("failed to get network specifications: %w", err) } deviceSpecs = append(deviceSpecs, networkSpecs...) } @@ -212,17 +212,17 @@ func createClonedVM(ctx context.Context, vmName string, config *Config, session } reconfigureTask, err := virtualMachine.Reconfigure(ctx, vmConfig) if err != nil { - return nil, fmt.Errorf("failed to reconfigure the VM: %v", err) + return nil, fmt.Errorf("failed to reconfigure the VM: %w", err) } if err := reconfigureTask.Wait(ctx); err != nil { - return nil, fmt.Errorf("error when waiting for result of the reconfigure task: %v", err) + return nil, fmt.Errorf("error when waiting for result of the reconfigure task: %w", err) } // Ubuntu won't boot with attached floppy device, because it tries to write to it // which fails, because the floppy device does not contain a floppy disk // Upstream issue: https://bugs.launchpad.net/cloud-images/+bug/1573095 if err := removeFloppyDevice(ctx, virtualMachine); err != nil { - return nil, fmt.Errorf("failed to remove floppy device: %v", err) + return nil, fmt.Errorf("failed to remove floppy device: %w", err) } return virtualMachine, nil @@ -234,7 +234,7 @@ func resolveDatastoreRef(ctx context.Context, config *Config, session *Session, klog.Infof("Choosing initial datastore placement for vm %s from datastore cluster %s", vm.Name(), config.DatastoreCluster) storagePod, err := session.Finder.DatastoreCluster(ctx, config.DatastoreCluster) if err != nil { - return nil, fmt.Errorf("failed to get datastore cluster: %v", err) + return nil, fmt.Errorf("failed to get datastore cluster: %w", err) } // Build pod selection spec from config spec @@ -263,7 +263,7 @@ func resolveDatastoreRef(ctx context.Context, config *Config, session *Session, storageResourceManager := object.NewStorageResourceManager(session.Client.Client) result, err := storageResourceManager.RecommendDatastores(ctx, storagePlacementSpec) if err != nil { - return nil, fmt.Errorf("error occurred while getting storage placement recommendation: %v", err) + return nil, fmt.Errorf("error occurred while getting storage placement recommendation: %w", err) } // Get the recommendations @@ -279,7 +279,7 @@ func resolveDatastoreRef(ctx context.Context, config *Config, session *Session, } else if config.DatastoreCluster == "" && config.Datastore != "" { datastore, err := session.Finder.Datastore(ctx, config.Datastore) if err != nil { - return nil, fmt.Errorf("failed to get datastore: %v", err) + return nil, fmt.Errorf("failed to get datastore: %w", err) } return types.NewReference(datastore.Reference()), nil } else { @@ -293,24 +293,24 @@ func uploadAndAttachISO(ctx context.Context, session *Session, vmRef *object.Vir // Get the datastore where VM files are located datastore, err := getDatastoreFromVM(ctx, session, vmRef) if err != nil { - return fmt.Errorf("error getting datastore from VM %s: %v", vmRef.Name(), err) + return fmt.Errorf("error getting datastore from VM %s: %w", vmRef.Name(), err) } klog.V(3).Infof("Uploading userdata ISO to datastore %+v, destination iso is %s\n", datastore, remoteIsoFilePath) if err := datastore.UploadFile(ctx, localIsoFilePath, remoteIsoFilePath, &p); err != nil { - return fmt.Errorf("failed to upload iso: %v", err) + return fmt.Errorf("failed to upload iso: %w", err) } klog.V(3).Infof("Uploaded ISO file %s", localIsoFilePath) // Find the cd-rom device and insert the cloud init iso file into it. devices, err := vmRef.Device(ctx) if err != nil { - return fmt.Errorf("failed to get devices: %v", err) + return fmt.Errorf("failed to get devices: %w", err) } // passing empty cd-rom name so that the first one gets returned cdrom, err := devices.FindCdrom("") if err != nil { - return fmt.Errorf("failed to find cdrom device: %v", err) + return fmt.Errorf("failed to find cdrom device: %w", err) } cdrom.Connectable.StartConnected = true iso := datastore.Path(remoteIsoFilePath) @@ -322,11 +322,11 @@ func generateLocalUserdataISO(userdata, name string) (string, error) { // take a directory as input userdataDir, err := ioutil.TempDir(localTempDir, name) if err != nil { - return "", fmt.Errorf("failed to create local temp directory for userdata at %s: %v", userdataDir, err) + return "", fmt.Errorf("failed to create local temp directory for userdata at %s: %w", userdataDir, err) } defer func() { if err := os.RemoveAll(userdataDir); err != nil { - utilruntime.HandleError(fmt.Errorf("error cleaning up local userdata tempdir %s: %v", userdataDir, err)) + utilruntime.HandleError(fmt.Errorf("error cleaning up local userdata tempdir %s: %w", userdataDir, err)) } }() @@ -336,7 +336,7 @@ func generateLocalUserdataISO(userdata, name string) (string, error) { metadataTmpl, err := template.New("metadata").Parse(metaDataTemplate) if err != nil { - return "", fmt.Errorf("failed to parse metadata template: %v", err) + return "", fmt.Errorf("failed to parse metadata template: %w", err) } metadata := &bytes.Buffer{} templateContext := struct { @@ -347,15 +347,15 @@ func generateLocalUserdataISO(userdata, name string) (string, error) { Hostname: name, } if err = metadataTmpl.Execute(metadata, templateContext); err != nil { - return "", fmt.Errorf("failed to render metadata: %v", err) + return "", fmt.Errorf("failed to render metadata: %w", err) } if err := ioutil.WriteFile(userdataFilePath, []byte(userdata), 0644); err != nil { - return "", fmt.Errorf("failed to locally write userdata file to %s: %v", userdataFilePath, err) + return "", fmt.Errorf("failed to locally write userdata file to %s: %w", userdataFilePath, err) } if err := ioutil.WriteFile(metadataFilePath, metadata.Bytes(), 0644); err != nil { - return "", fmt.Errorf("failed to locally write metadata file to %s: %v", userdataFilePath, err) + return "", fmt.Errorf("failed to locally write metadata file to %s: %w", userdataFilePath, err) } var command string @@ -373,7 +373,7 @@ func generateLocalUserdataISO(userdata, name string) (string, error) { cmd := exec.Command(command, args...) if output, err := cmd.CombinedOutput(); err != nil { - return "", fmt.Errorf("error executing command `%s %s`: output: `%s`, error: `%v`", command, args, string(output), err) + return "", fmt.Errorf("error executing command `%s %s`: output: `%s`, error: `%w`", command, args, string(output), err) } return isoFilePath, nil @@ -382,7 +382,7 @@ func generateLocalUserdataISO(userdata, name string) (string, error) { func removeFloppyDevice(ctx context.Context, virtualMachine *object.VirtualMachine) error { vmDevices, err := virtualMachine.Device(ctx) if err != nil { - return fmt.Errorf("failed to get device list: %v", err) + return fmt.Errorf("failed to get device list: %w", err) } // If there is more than one floppy device attached, you will simply get the first one. We @@ -392,11 +392,11 @@ func removeFloppyDevice(ctx context.Context, virtualMachine *object.VirtualMachi if err.Error() == "no floppy device found" { return nil } - return fmt.Errorf("failed to find floppy: %v", err) + return fmt.Errorf("failed to find floppy: %w", err) } if err := virtualMachine.RemoveDevice(ctx, false, floppyDevice); err != nil { - return fmt.Errorf("failed to remove floppy device: %v", err) + return fmt.Errorf("failed to remove floppy device: %w", err) } return nil @@ -405,7 +405,7 @@ func removeFloppyDevice(ctx context.Context, virtualMachine *object.VirtualMachi func getDisksFromVM(ctx context.Context, vm *object.VirtualMachine) ([]*types.VirtualDisk, error) { var props mo.VirtualMachine if err := vm.Properties(ctx, vm.Reference(), nil, &props); err != nil { - return nil, fmt.Errorf("error getting VM template reference: %v", err) + return nil, fmt.Errorf("error getting VM template reference: %w", err) } l := object.VirtualDeviceList(props.Config.Hardware.Device) disks := l.SelectByType((*types.VirtualDisk)(nil)) @@ -436,7 +436,7 @@ func getDatastoreFromVM(ctx context.Context, session *Session, vmRef *object.Vir var props mo.VirtualMachine // Obtain VM properties if err := vmRef.Properties(ctx, vmRef.Reference(), nil, &props); err != nil { - return nil, fmt.Errorf("error getting VM properties: %v", err) + return nil, fmt.Errorf("error getting VM properties: %w", err) } datastorePathObj := new(object.DatastorePath) isSuccess := datastorePathObj.FromString(props.Summary.Config.VmPathName) @@ -450,7 +450,7 @@ func resolveResourcePoolRef(ctx context.Context, config *Config, session *Sessio if config.ResourcePool != "" { targetResourcePool, err := session.Finder.ResourcePool(ctx, config.ResourcePool) if err != nil { - return nil, fmt.Errorf("failed to get target resourcepool: %v", err) + return nil, fmt.Errorf("failed to get target resourcepool: %w", err) } return types.NewReference(targetResourcePool.Reference()), nil } @@ -460,7 +460,7 @@ func resolveResourcePoolRef(ctx context.Context, config *Config, session *Sessio func createAndAttachTags(ctx context.Context, config *Config, vm *object.VirtualMachine) error { restAPISession, err := NewRESTSession(ctx, config) if err != nil { - return fmt.Errorf("failed to create REST API session: %v", err) + return fmt.Errorf("failed to create REST API session: %w", err) } defer restAPISession.Logout(ctx) tagManager := tags.NewManager(restAPISession.Client) @@ -468,7 +468,7 @@ func createAndAttachTags(ctx context.Context, config *Config, vm *object.Virtual for _, tag := range config.Tags { tagID, err := tagManager.CreateTag(ctx, &tag) if err != nil { - return fmt.Errorf("failed to create tag: %v %v", tag, err) + return fmt.Errorf("failed to create tag: %v %w", tag, err) } if err := tagManager.AttachTag(ctx, tagID, vm.Reference()); err != nil { @@ -479,10 +479,10 @@ func createAndAttachTags(ctx context.Context, config *Config, vm *object.Virtual Name: tag.Name, CategoryID: tag.CategoryID, }); errDelete != nil { - return fmt.Errorf("failed to attach tag to VM and delete the orphan tag: %v, attach error: %v, delete error: %v", tag, err, errDelete) + return fmt.Errorf("failed to attach tag to VM and delete the orphan tag: %v, attach error: %v, delete error: %w", tag, err, errDelete) } klog.V(3).Infof("Failed to attach tag %v. The tag was successfully deleted", tag) - return fmt.Errorf("failed to attach tag to VM: %v %v", tag, err) + return fmt.Errorf("failed to attach tag to VM: %v %w", tag, err) } } return nil @@ -491,20 +491,20 @@ func createAndAttachTags(ctx context.Context, config *Config, vm *object.Virtual func deleteTags(ctx context.Context, config *Config, vm *object.VirtualMachine) error { restAPISession, err := NewRESTSession(ctx, config) if err != nil { - return fmt.Errorf("failed to create REST API session: %v", err) + return fmt.Errorf("failed to create REST API session: %w", err) } defer restAPISession.Logout(ctx) tagManager := tags.NewManager(restAPISession.Client) tags, err := tagManager.GetAttachedTags(ctx, vm.Reference()) if err != nil { - return fmt.Errorf("failed to get attached tags for the VM: %s, %v", vm.Name(), err) + return fmt.Errorf("failed to get attached tags for the VM: %s, %w", vm.Name(), err) } klog.V(3).Info("Deleting tags") for _, tag := range tags { err := tagManager.DeleteTag(ctx, &tag) if err != nil { - return fmt.Errorf("failed to delete tag: %v %v", tag, err) + return fmt.Errorf("failed to delete tag: %v %w", tag, err) } } diff --git a/pkg/cloudprovider/provider/vsphere/network.go b/pkg/cloudprovider/provider/vsphere/network.go index 0498cef6b..6bb4ecbcb 100644 --- a/pkg/cloudprovider/provider/vsphere/network.go +++ b/pkg/cloudprovider/provider/vsphere/network.go @@ -43,11 +43,11 @@ func GetNetworkSpecs(ctx context.Context, session *Session, devices object.Virtu // Add new NICs based on the machine config. ref, err := session.Finder.Network(ctx, network) if err != nil { - return nil, fmt.Errorf("failed to find network %q: %v", network, err) + return nil, fmt.Errorf("failed to find network %q: %w", network, err) } backing, err := ref.EthernetCardBackingInfo(ctx) if err != nil { - return nil, fmt.Errorf("failed to create new ethernet card backing info for network %q: %v", network, err) + return nil, fmt.Errorf("failed to create new ethernet card backing info for network %q: %w", network, err) } dev, err := object.EthernetCardTypes().CreateEthernetCard(ethCardType, backing) if err != nil { diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index cb1c7cef5..09c49b2c1 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -204,19 +204,19 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { defer cancel() config, pc, _, err := p.getConfig(spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to get config: %v", err) + return fmt.Errorf("failed to get config: %w", err) } session, err := NewSession(ctx, config) if err != nil { - return fmt.Errorf("failed to create vCenter session: %v", err) + return fmt.Errorf("failed to create vCenter session: %w", err) } defer session.Logout() if config.Tags != nil { restAPISession, err := NewRESTSession(ctx, config) if err != nil { - return fmt.Errorf("failed to create REST API session: %v", err) + return fmt.Errorf("failed to create REST API session: %w", err) } defer restAPISession.Logout(ctx) tagManager := tags.NewManager(restAPISession.Client) @@ -239,18 +239,18 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { // present, otherwise an error is raised. if config.DatastoreCluster != "" && config.Datastore == "" { if _, err := session.Finder.DatastoreCluster(ctx, config.DatastoreCluster); err != nil { - return fmt.Errorf("failed to get datastore cluster %s: %v", config.DatastoreCluster, err) + return fmt.Errorf("failed to get datastore cluster %s: %w", config.DatastoreCluster, err) } } else if config.Datastore != "" && config.DatastoreCluster == "" { if _, err := session.Finder.Datastore(ctx, config.Datastore); err != nil { - return fmt.Errorf("failed to get datastore %s: %v", config.Datastore, err) + return fmt.Errorf("failed to get datastore %s: %w", config.Datastore, err) } } else { - return fmt.Errorf("one between datastore and datastore cluster should be specified: %v", err) + return fmt.Errorf("one between datastore and datastore cluster should be specified: %w", err) } if _, err := session.Finder.Folder(ctx, config.Folder); err != nil { - return fmt.Errorf("failed to get folder %q: %v", config.Folder, err) + return fmt.Errorf("failed to get folder %q: %w", config.Folder, err) } if _, err := p.get(ctx, config.Folder, spec, session.Finder); err == nil { @@ -259,18 +259,18 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { if config.ResourcePool != "" { if _, err := session.Finder.ResourcePool(ctx, config.ResourcePool); err != nil { - return fmt.Errorf("failed to get resourcepool %q: %v", config.ResourcePool, err) + return fmt.Errorf("failed to get resourcepool %q: %w", config.ResourcePool, err) } } templateVM, err := session.Finder.VirtualMachine(ctx, config.TemplateVMName) if err != nil { - return fmt.Errorf("failed to get template vm %q: %v", config.TemplateVMName, err) + return fmt.Errorf("failed to get template vm %q: %w", config.TemplateVMName, err) } disks, err := getDisksFromVM(ctx, templateVM) if err != nil { - return fmt.Errorf("failed to get disks from VM: %v", err) + return fmt.Errorf("failed to get disks from VM: %w", err) } if diskLen := len(disks); diskLen != 1 { return fmt.Errorf("expected vm to have exactly one disk, had %d", diskLen) @@ -282,7 +282,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { } } if pc.OperatingSystem == providerconfigtypes.OperatingSystemSLES { - return fmt.Errorf("invalid/not supported operating system specified %q: %v", pc.OperatingSystem, providerconfigtypes.ErrOSNotSupported) + return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, providerconfigtypes.ErrOSNotSupported) } return nil } @@ -299,7 +299,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert if err != nil { _, cleanupErr := p.Cleanup(machine, data) if cleanupErr != nil { - return nil, fmt.Errorf("cleaning up failed with err %v after creation failed with err %v", cleanupErr, err) + return nil, fmt.Errorf("cleaning up failed with err %v after creation failed with err %w", cleanupErr, err) } return nil, err } @@ -311,12 +311,12 @@ func (p *provider) create(machine *clusterv1alpha1.Machine, userdata string) (in config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { - return nil, fmt.Errorf("failed to parse config: %v", err) + return nil, fmt.Errorf("failed to parse config: %w", err) } session, err := NewSession(ctx, config) if err != nil { - return nil, fmt.Errorf("failed to create vCenter session: %v", err) + return nil, fmt.Errorf("failed to create vCenter session: %w", err) } defer session.Logout() @@ -333,23 +333,23 @@ func (p *provider) create(machine *clusterv1alpha1.Machine, userdata string) (in containerLinuxUserdata, ) if err != nil { - return nil, machineInvalidConfigurationTerminalError(fmt.Errorf("failed to create cloned vm: '%v'", err)) + return nil, machineInvalidConfigurationTerminalError(fmt.Errorf("failed to create cloned vm: '%w'", err)) } if err := createAndAttachTags(ctx, config, virtualMachine); err != nil { - return nil, fmt.Errorf("failed create and attach tags: %v", err) + return nil, fmt.Errorf("failed create and attach tags: %w", err) } if pc.OperatingSystem != providerconfigtypes.OperatingSystemFlatcar { localUserdataIsoFilePath, err := generateLocalUserdataISO(userdata, machine.Spec.Name) if err != nil { - return nil, fmt.Errorf("failed to generate local userdadata iso: %v", err) + return nil, fmt.Errorf("failed to generate local userdadata iso: %w", err) } defer func() { err := os.Remove(localUserdataIsoFilePath) if err != nil { - utilruntime.HandleError(fmt.Errorf("failed to clean up local userdata iso file at %s: %v", localUserdataIsoFilePath, err)) + utilruntime.HandleError(fmt.Errorf("failed to clean up local userdata iso file at %s: %w", localUserdataIsoFilePath, err)) } }() @@ -357,22 +357,22 @@ func (p *provider) create(machine *clusterv1alpha1.Machine, userdata string) (in // Destroy VM to avoid a leftover. destroyTask, vmErr := virtualMachine.Destroy(ctx) if vmErr != nil { - return nil, fmt.Errorf("failed to destroy vm %s after failing upload and attach userdata iso: %v / %v", virtualMachine.Name(), err, vmErr) + return nil, fmt.Errorf("failed to destroy vm %s after failing upload and attach userdata iso: %w / %v", virtualMachine.Name(), err, vmErr) } if vmErr := destroyTask.Wait(ctx); vmErr != nil { - return nil, fmt.Errorf("failed to destroy vm %s after failing upload and attach userdata iso: %v / %v", virtualMachine.Name(), err, vmErr) + return nil, fmt.Errorf("failed to destroy vm %s after failing upload and attach userdata iso: %w / %v", virtualMachine.Name(), err, vmErr) } - return nil, machineInvalidConfigurationTerminalError(fmt.Errorf("failed to upload and attach userdata iso: %v", err)) + return nil, machineInvalidConfigurationTerminalError(fmt.Errorf("failed to upload and attach userdata iso: %w", err)) } } powerOnTask, err := virtualMachine.PowerOn(ctx) if err != nil { - return nil, fmt.Errorf("failed to power on machine: %v", err) + return nil, fmt.Errorf("failed to power on machine: %w", err) } if err := powerOnTask.Wait(ctx); err != nil { - return nil, fmt.Errorf("error when waiting for vm powerOn task: %v", err) + return nil, fmt.Errorf("error when waiting for vm powerOn task: %w", err) } return Server{name: virtualMachine.Name(), status: instance.StatusRunning, id: virtualMachine.Reference().Value}, nil @@ -384,12 +384,12 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { - return false, fmt.Errorf("failed to parse config: %v", err) + return false, fmt.Errorf("failed to parse config: %w", err) } session, err := NewSession(ctx, config) if err != nil { - return false, fmt.Errorf("failed to create vCenter session: %v", err) + return false, fmt.Errorf("failed to create vCenter session: %w", err) } defer session.Logout() @@ -398,16 +398,16 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider if cloudprovidererrors.IsNotFound(err) { return true, nil } - return false, fmt.Errorf("failed to get instance from vSphere: %v", err) + return false, fmt.Errorf("failed to get instance from vSphere: %w", err) } if err := deleteTags(ctx, config, virtualMachine); err != nil { - return false, fmt.Errorf("failed to delete tags: %v", err) + return false, fmt.Errorf("failed to delete tags: %w", err) } powerState, err := virtualMachine.PowerState(ctx) if err != nil { - return false, fmt.Errorf("failed to get virtual machine power state: %v", err) + return false, fmt.Errorf("failed to get virtual machine power state: %w", err) } // We cannot destroy a VM that's powered on, but we also @@ -415,21 +415,21 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider if powerState != types.VirtualMachinePowerStatePoweredOff { powerOffTask, err := virtualMachine.PowerOff(ctx) if err != nil { - return false, fmt.Errorf("failed to poweroff vm %s: %v", virtualMachine.Name(), err) + return false, fmt.Errorf("failed to poweroff vm %s: %w", virtualMachine.Name(), err) } if err = powerOffTask.Wait(ctx); err != nil { - return false, fmt.Errorf("failed to poweroff vm %s: %v", virtualMachine.Name(), err) + return false, fmt.Errorf("failed to poweroff vm %s: %w", virtualMachine.Name(), err) } } virtualMachineDeviceList, err := virtualMachine.Device(ctx) if err != nil { - return false, fmt.Errorf("failed to get devices for virtual machine: %v", err) + return false, fmt.Errorf("failed to get devices for virtual machine: %w", err) } pvs := &corev1.PersistentVolumeList{} if err := data.Client.List(data.Ctx, pvs); err != nil { - return false, fmt.Errorf("failed to list PVs: %v", err) + return false, fmt.Errorf("failed to list PVs: %w", err) } for _, pv := range pvs.Items { @@ -441,7 +441,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider fileName := device.GetVirtualDevice().Backing.(types.BaseVirtualDeviceFileBackingInfo).GetVirtualDeviceFileBackingInfo().FileName if pv.Spec.VsphereVolume.VolumePath == fileName { if err := virtualMachine.RemoveDevice(ctx, true, device); err != nil { - return false, fmt.Errorf("error detaching pv-backing disk %s: %v", fileName, err) + return false, fmt.Errorf("error detaching pv-backing disk %s: %w", fileName, err) } } } @@ -450,14 +450,14 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider datastore, err := getDatastoreFromVM(ctx, session, virtualMachine) if err != nil { - return false, fmt.Errorf("Error getting datastore from VM %s: %v", virtualMachine.Name(), err) + return false, fmt.Errorf("Error getting datastore from VM %s: %w", virtualMachine.Name(), err) } destroyTask, err := virtualMachine.Destroy(ctx) if err != nil { - return false, fmt.Errorf("failed to destroy vm %s: %v", virtualMachine.Name(), err) + return false, fmt.Errorf("failed to destroy vm %s: %w", virtualMachine.Name(), err) } if err := destroyTask.Wait(ctx); err != nil { - return false, fmt.Errorf("failed to destroy vm %s: %v", virtualMachine.Name(), err) + return false, fmt.Errorf("failed to destroy vm %s: %w", virtualMachine.Name(), err) } if pc.OperatingSystem != providerconfigtypes.OperatingSystemFlatcar { @@ -467,7 +467,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider if err.Error() == fmt.Sprintf("File [%s] %s was not found", datastore.Name(), virtualMachine.Name()) { return true, nil } - return false, fmt.Errorf("failed to delete storage of deleted instance %s: %v", virtualMachine.Name(), err) + return false, fmt.Errorf("failed to delete storage of deleted instance %s: %w", virtualMachine.Name(), err) } } @@ -480,12 +480,12 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertype config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { - return nil, fmt.Errorf("failed to parse config: %v", err) + return nil, fmt.Errorf("failed to parse config: %w", err) } session, err := NewSession(ctx, config) if err != nil { - return nil, fmt.Errorf("failed to create vCenter session: %v", err) + return nil, fmt.Errorf("failed to create vCenter session: %w", err) } defer session.Logout() @@ -497,16 +497,16 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertype powerState, err := virtualMachine.PowerState(ctx) if err != nil { - return nil, fmt.Errorf("failed to get powerstate: %v", err) + return nil, fmt.Errorf("failed to get powerstate: %w", err) } if powerState != types.VirtualMachinePowerStatePoweredOn { powerOnTask, err := virtualMachine.PowerOn(ctx) if err != nil { - return nil, fmt.Errorf("failed to power on instance that was in state %q: %v", powerState, err) + return nil, fmt.Errorf("failed to power on instance that was in state %q: %w", powerState, err) } if err := powerOnTask.Wait(ctx); err != nil { - return nil, fmt.Errorf("failed waiting for instance to be powered on: %v", err) + return nil, fmt.Errorf("failed waiting for instance to be powered on: %w", err) } // We must return here because the vendored code for determining if the guest // utils are running yields an NPD when using with an instance that is not running @@ -517,13 +517,13 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertype addresses := map[string]corev1.NodeAddressType{} isGuestToolsRunning, err := virtualMachine.IsToolsRunning(ctx) if err != nil { - return nil, fmt.Errorf("failed to check if guest utils are running: %v", err) + return nil, fmt.Errorf("failed to check if guest utils are running: %w", err) } if isGuestToolsRunning { var moVirtualMachine mo.VirtualMachine pc := property.DefaultCollector(session.Client.Client) if err := pc.RetrieveOne(ctx, virtualMachine.Reference(), []string{"guest"}, &moVirtualMachine); err != nil { - return nil, fmt.Errorf("failed to retrieve guest info: %v", err) + return nil, fmt.Errorf("failed to retrieve guest info: %w", err) } for _, nic := range moVirtualMachine.Guest.Net { @@ -541,14 +541,14 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertype return Server{name: virtualMachine.Name(), status: instance.StatusRunning, addresses: addresses, id: virtualMachine.Reference().Value}, nil } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, new ktypes.UID) error { +func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, uid ktypes.UID) error { return nil } func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { c, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { - return "", "", fmt.Errorf("failed to parse config: %v", err) + return "", "", fmt.Errorf("failed to parse config: %w", err) } passedURL := c.VSphereURL @@ -559,7 +559,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri u, err := url.Parse(passedURL) if err != nil { - return "", "", fmt.Errorf("failed to parse '%s' as url: %v", passedURL, err) + return "", "", fmt.Errorf("failed to parse '%s' as url: %w", passedURL, err) } workingDir := c.Folder @@ -596,7 +596,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri s, err := vspheretypes.CloudConfigToString(cc) if err != nil { - return "", "", fmt.Errorf("failed to convert the cloud-config to string: %v", err) + return "", "", fmt.Errorf("failed to convert the cloud-config to string: %w", err) } return s, "vsphere", nil @@ -625,7 +625,7 @@ func (p *provider) get(ctx context.Context, folder string, spec clusterv1alpha1. if err.Error() == fmt.Sprintf("vm '%s' not found", path) { return nil, cloudprovidererrors.ErrInstanceNotFound } - return nil, fmt.Errorf("failed to list virtual machines: %v", err) + return nil, fmt.Errorf("failed to list virtual machines: %w", err) } if len(virtualMachineList) == 0 { diff --git a/pkg/cloudprovider/provider/vsphere/provider_test.go b/pkg/cloudprovider/provider/vsphere/provider_test.go index 4503b2fa6..3987483e0 100644 --- a/pkg/cloudprovider/provider/vsphere/provider_test.go +++ b/pkg/cloudprovider/provider/vsphere/provider_test.go @@ -30,7 +30,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/providerconfig" "k8s.io/utils/pointer" - fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) type vsphereProviderSpecConf struct { @@ -169,7 +169,9 @@ func TestValidate(t *testing.T) { password, _ := simulator.DefaultLogin.Password() p := &provider{ // Note that configVarResolver is not used in this test as the getConfigFunc is mocked. - configVarResolver: providerconfig.NewConfigVarResolver(context.Background(), fakeclient.NewFakeClient()), + configVarResolver: providerconfig.NewConfigVarResolver(context.Background(), fakectrlruntimeclient. + NewClientBuilder(). + Build()), } tt.args.User = username tt.args.Password = password diff --git a/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go b/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go index 132fa8b5c..073262a8b 100644 --- a/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go +++ b/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go @@ -88,7 +88,7 @@ type VirtualCenterConfig struct { Datacenters string `gcfg:"datacenters"` } -// CloudConfig is used to read and store information from the cloud configuration file +// CloudConfig is used to read and store information from the cloud configuration file. type CloudConfig struct { Global GlobalOpts Disk DiskOpts @@ -103,12 +103,12 @@ func CloudConfigToString(c *CloudConfig) (string, error) { tpl, err := template.New("cloud-config").Funcs(funcMap).Parse(cloudConfigTpl) if err != nil { - return "", fmt.Errorf("failed to parse the cloud config template: %v", err) + return "", fmt.Errorf("failed to parse the cloud config template: %w", err) } buf := &bytes.Buffer{} if err := tpl.Execute(buf, c); err != nil { - return "", fmt.Errorf("failed to execute cloud config template: %v", err) + return "", fmt.Errorf("failed to execute cloud config template: %w", err) } return buf.String(), nil diff --git a/pkg/cloudprovider/provider/vsphere/types/types.go b/pkg/cloudprovider/provider/vsphere/types/types.go index 2c1760de8..835af02e7 100644 --- a/pkg/cloudprovider/provider/vsphere/types/types.go +++ b/pkg/cloudprovider/provider/vsphere/types/types.go @@ -47,7 +47,7 @@ type RawConfig struct { AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` } -// Tag represents vsphere tag +// Tag represents vsphere tag. type Tag struct { Description string `json:"description,omitempty"` Name string `json:"name"` diff --git a/pkg/cloudprovider/types/types.go b/pkg/cloudprovider/types/types.go index fdc756fe7..5dc22fb89 100644 --- a/pkg/cloudprovider/types/types.go +++ b/pkg/cloudprovider/types/types.go @@ -28,7 +28,7 @@ import ( ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) -// Provider exposed all required functions to interact with a cloud provider +// Provider exposed all required functions to interact with a cloud provider. type Provider interface { // AddDefaults will read the MachineSpec and apply defaults for provider specific fields AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) @@ -69,41 +69,41 @@ type Provider interface { // MigrateUID is called when the controller migrates types and the UID of the machine object changes // All cloud providers that use Machine.UID to uniquely identify resources must implement this - MigrateUID(machine *clusterv1alpha1.Machine, new types.UID) error + MigrateUID(machine *clusterv1alpha1.Machine, uid types.UID) error // SetMetricsForMachines allows providers to provide provider-specific metrics. This may be implemented // as no-op SetMetricsForMachines(machines clusterv1alpha1.MachineList) error } -// MachineModifier defines a function to modify a machine +// MachineModifier defines a function to modify a machine. type MachineModifier func(*clusterv1alpha1.Machine) -// MachineUpdater defines a function to persist an update to a machine +// MachineUpdater defines a function to persist an update to a machine. type MachineUpdater func(*clusterv1alpha1.Machine, ...MachineModifier) error -// ProviderData is the struct the cloud providers get when creating or deleting an instance +// ProviderData is the struct the cloud providers get when creating or deleting an instance. type ProviderData struct { Ctx context.Context Update MachineUpdater Client ctrlruntimeclient.Client } -// GetMachineUpdater returns an MachineUpdater based on the passed in context and ctrlruntimeclient.Client +// GetMachineUpdater returns an MachineUpdater based on the passed in context and ctrlruntimeclient.Client. func GetMachineUpdater(ctx context.Context, client ctrlruntimeclient.Client) MachineUpdater { return func(machine *clusterv1alpha1.Machine, modifiers ...MachineModifier) error { if len(modifiers) == 0 { return nil } - // Store name here, because the machine can be nil if an update failed + // Store name here, because the machine can be nil if an update failed. namespacedName := types.NamespacedName{Namespace: machine.Namespace, Name: machine.Name} return retry.RetryOnConflict(retry.DefaultBackoff, func() error { if err := client.Get(ctx, namespacedName, machine); err != nil { return err } - // Check if we actually change something and only update if that is the case + // Check if we actually change something and only update if that is the case. unmodifiedMachine := machine.DeepCopy() for _, modify := range modifiers { modify(machine) diff --git a/pkg/cloudprovider/util/cloud_init_settings.go b/pkg/cloudprovider/util/cloud_init_settings.go index b4fa4c5b6..fc199b92d 100644 --- a/pkg/cloudprovider/util/cloud_init_settings.go +++ b/pkg/cloudprovider/util/cloud_init_settings.go @@ -40,7 +40,7 @@ const ( func ExtractAPIServerToken(ctx context.Context, client ctrlruntimeclient.Client) (string, error) { secretList := corev1.SecretList{} if err := client.List(ctx, &secretList, &ctrlruntimeclient.ListOptions{Namespace: CloudInitNamespace}); err != nil { - return "", fmt.Errorf("failed to list secrets in namespace %s: %v", CloudInitNamespace, err) + return "", fmt.Errorf("failed to list secrets in namespace %s: %w", CloudInitNamespace, err) } for _, secret := range secretList.Items { @@ -56,18 +56,17 @@ func ExtractAPIServerToken(ctx context.Context, client ctrlruntimeclient.Client) } return "", errors.New("failed to fetch api server token") - } func ExtractTokenAndAPIServer(ctx context.Context, userdata string, client ctrlruntimeclient.Client) (token string, apiServer string, err error) { secretList := corev1.SecretList{} if err := client.List(ctx, &secretList, &ctrlruntimeclient.ListOptions{Namespace: CloudInitNamespace}); err != nil { - return "", "", fmt.Errorf("failed to list secrets in namespace %s: %v", CloudInitNamespace, err) + return "", "", fmt.Errorf("failed to list secrets in namespace %s: %w", CloudInitNamespace, err) } apiServer, err = extractAPIServer(userdata) if err != nil { - return "", "", fmt.Errorf("failed to extract api server address: %v", err) + return "", "", fmt.Errorf("failed to extract api server address: %w", err) } for _, secret := range secretList.Items { @@ -97,11 +96,11 @@ func CreateMachineCloudInitSecret(ctx context.Context, userdata, machineName str Data: map[string][]byte{"cloud_init": []byte(userdata)}, } if err := client.Create(ctx, secret); err != nil { - return fmt.Errorf("failed to create secret for userdata: %v", err) + return fmt.Errorf("failed to create secret for userdata: %w", err) } } - return fmt.Errorf("failed to fetch cloud-init secret: %v", err) + return fmt.Errorf("failed to fetch cloud-init secret: %w", err) } return nil @@ -119,14 +118,14 @@ func extractAPIServer(userdata string) (string, error) { }{} if err := yaml.Unmarshal([]byte(userdata), files); err != nil { - return "", fmt.Errorf("failed to unmarshal userdata: %v", err) + return "", fmt.Errorf("failed to unmarshal userdata: %w", err) } for _, file := range files.WriteFiles { if file.Path == "/etc/kubernetes/bootstrap-kubelet.conf" { config, err := clientcmd.RESTConfigFromKubeConfig([]byte(file.Content)) if err != nil { - return "", fmt.Errorf("failed to get kubeconfig from userdata: %v", err) + return "", fmt.Errorf("failed to get kubeconfig from userdata: %w", err) } return config.Host, nil diff --git a/pkg/cloudprovider/util/cloud_init_settings_test.go b/pkg/cloudprovider/util/cloud_init_settings_test.go index e1cf26a0d..a4bf41131 100644 --- a/pkg/cloudprovider/util/cloud_init_settings_test.go +++ b/pkg/cloudprovider/util/cloud_init_settings_test.go @@ -23,7 +23,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client/fake" + "k8s.io/client-go/kubernetes/scheme" + fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) var testData = []struct { @@ -53,7 +54,11 @@ var testData = []struct { func TestCloudInitGeneration(t *testing.T) { for _, test := range testData { t.Run(test.name, func(t *testing.T) { - fakeClient := fake.NewFakeClient(test.secret) + fakeClient := fakectrlruntimeclient. + NewClientBuilder(). + WithScheme(scheme.Scheme). + WithObjects(test.secret). + Build() userdata, err := ioutil.ReadFile(test.userdata) if err != nil { diff --git a/pkg/cloudprovider/util/http.go b/pkg/cloudprovider/util/http.go index 44d0c0272..06ce4da53 100644 --- a/pkg/cloudprovider/util/http.go +++ b/pkg/cloudprovider/util/http.go @@ -37,7 +37,7 @@ const defaultClientTimeout = 15 * time.Second var ( // CABundle is set globally once by the main() function // and is used to overwrite the default set of CA certificates - // loaded from the host system/pod + // loaded from the host system/pod. CABundle *x509.CertPool ) @@ -47,7 +47,7 @@ var ( func SetCABundleFile(filename string) error { content, err := ioutil.ReadFile(filename) if err != nil { - return fmt.Errorf("failed to read file: %v", err) + return fmt.Errorf("failed to read file: %w", err) } CABundle = x509.NewCertPool() diff --git a/pkg/cloudprovider/util/net.go b/pkg/cloudprovider/util/net.go index fc640f1b6..f682ca05c 100644 --- a/pkg/cloudprovider/util/net.go +++ b/pkg/cloudprovider/util/net.go @@ -31,7 +31,7 @@ const ( func CIDRToIPAndNetMask(ipv4 string) (string, string, int, error) { ip, ipNet, err := net.ParseCIDR(ipv4) if err != nil { - return "", "", 0, fmt.Errorf("failed to parse CIDR prefix: %v", err) + return "", "", 0, fmt.Errorf("failed to parse CIDR prefix: %w", err) } if len(ipNet.Mask) != 4 { @@ -60,7 +60,7 @@ func GenerateRandMAC() (net.HardwareAddr, error) { return mac, nil } -// IPFamily IPv4 | IPv6 | IPv4+IPv6 +// IPFamily IPv4 | IPv6 | IPv4+IPv6. type IPFamily string const ( @@ -70,7 +70,7 @@ const ( DualStack IPFamily = "IPv4+IPv6" ) -// IsLinkLocal checks if given ip address is link local +// IsLinkLocal checks if given ip address is link local.. func IsLinkLocal(ipAddr string) bool { addr := net.ParseIP(ipAddr) return addr.IsLinkLocalMulticast() || addr.IsLinkLocalUnicast() diff --git a/pkg/cloudprovider/util/util.go b/pkg/cloudprovider/util/util.go index 064867540..63a15a83b 100644 --- a/pkg/cloudprovider/util/util.go +++ b/pkg/cloudprovider/util/util.go @@ -28,7 +28,6 @@ import ( func RemoveFinalizerOnInstanceNotFound(finalizer string, machine *v1alpha1.Machine, provider *cloudprovidertypes.ProviderData) (bool, error) { - if !kuberneteshelper.HasFinalizer(machine, finalizer) { return true, nil } @@ -36,9 +35,7 @@ func RemoveFinalizerOnInstanceNotFound(finalizer string, if err := provider.Update(machine, func(updatedMachine *v1alpha1.Machine) { updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizer) }); err != nil { - return false, fmt.Errorf("failed updating machine %v finzaliers: %v", machine.Name, err) - + return false, fmt.Errorf("failed updating machine %v finzaliers: %w", machine.Name, err) } - return true, nil } diff --git a/pkg/cloudprovider/util/util_test.go b/pkg/cloudprovider/util/util_test.go index d607842d3..1c15707bd 100644 --- a/pkg/cloudprovider/util/util_test.go +++ b/pkg/cloudprovider/util/util_test.go @@ -36,15 +36,18 @@ func TestRemoveFinalizerOnInstanceNotFound(t *testing.T) { t.Fatalf("failed to add clusterv1alpha1 to scheme: %v", err) } - var fakeClient = fakectrlruntimeclient.NewFakeClient( - &v1alpha1.Machine{ + var fakeClient = fakectrlruntimeclient. + NewClientBuilder(). + WithScheme(scheme.Scheme). + WithObjects(&v1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{ Name: "test_machine", Finalizers: []string{ "test_finalizer_1", "test_finalizer_2"}, }, - }) + }). + Build() var testCases = []struct { name string diff --git a/pkg/cloudprovider/validationwrapper.go b/pkg/cloudprovider/validationwrapper.go index 976ba129e..6cd7c45f5 100644 --- a/pkg/cloudprovider/validationwrapper.go +++ b/pkg/cloudprovider/validationwrapper.go @@ -31,22 +31,22 @@ type cachingValidationWrapper struct { actualProvider cloudprovidertypes.Provider } -// NewValidationCacheWrappingCloudProvider returns a wrapped cloudprovider +// NewValidationCacheWrappingCloudProvider returns a wrapped cloudprovider. func NewValidationCacheWrappingCloudProvider(actualProvider cloudprovidertypes.Provider) cloudprovidertypes.Provider { return &cachingValidationWrapper{actualProvider: actualProvider} } -// AddDefaults just calls the underlying cloudproviders AddDefaults +// AddDefaults just calls the underlying cloudproviders AddDefaults. func (w *cachingValidationWrapper) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { return w.actualProvider.AddDefaults(spec) } // Validate tries to get the validation result from the cache and if not found, calls the -// cloudproviders Validate and saves that to the cache +// cloudproviders Validate and saves that to the cache. func (w *cachingValidationWrapper) Validate(spec v1alpha1.MachineSpec) error { result, exists, err := cache.Get(spec) if err != nil { - return fmt.Errorf("error getting validation result from cache: %v", err) + return fmt.Errorf("error getting validation result from cache: %w", err) } if exists { klog.V(6).Infof("Got cache hit for validation") @@ -56,38 +56,38 @@ func (w *cachingValidationWrapper) Validate(spec v1alpha1.MachineSpec) error { klog.V(6).Infof("Got cache miss for validation") err = w.actualProvider.Validate(spec) if err := cache.Set(spec, err); err != nil { - return fmt.Errorf("failed to set cache after validation: %v", err) + return fmt.Errorf("failed to set cache after validation: %w", err) } return err } -// Get just calls the underlying cloudproviders Get +// Get just calls the underlying cloudproviders Get. func (w *cachingValidationWrapper) Get(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { return w.actualProvider.Get(machine, data) } -// GetCloudConfig just calls the underlying cloudproviders GetCloudConfig +// GetCloudConfig just calls the underlying cloudproviders GetCloudConfig. func (w *cachingValidationWrapper) GetCloudConfig(spec v1alpha1.MachineSpec) (string, string, error) { return w.actualProvider.GetCloudConfig(spec) } -// Create just calls the underlying cloudproviders Create +// Create just calls the underlying cloudproviders Create. func (w *cachingValidationWrapper) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { return w.actualProvider.Create(machine, data, userdata) } -// Cleanup just calls the underlying cloudproviders Cleanup +// Cleanup just calls the underlying cloudproviders Cleanup. func (w *cachingValidationWrapper) Cleanup(m *v1alpha1.Machine, mcd *cloudprovidertypes.ProviderData) (bool, error) { return w.actualProvider.Cleanup(m, mcd) } -// MigrateUID just calls the underlying cloudproviders MigrateUID -func (w *cachingValidationWrapper) MigrateUID(m *v1alpha1.Machine, new types.UID) error { - return w.actualProvider.MigrateUID(m, new) +// MigrateUID just calls the underlying cloudproviders MigrateUID. +func (w *cachingValidationWrapper) MigrateUID(m *v1alpha1.Machine, newUID types.UID) error { + return w.actualProvider.MigrateUID(m, newUID) } -// MachineMetricsLabels just calls the underlying cloudproviders MachineMetricsLabels +// MachineMetricsLabels just calls the underlying cloudproviders MachineMetricsLabels. func (w *cachingValidationWrapper) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { return w.actualProvider.MachineMetricsLabels(machine) } diff --git a/pkg/clusterinfo/configmap.go b/pkg/clusterinfo/configmap.go index 5d654fcae..02dfc19b4 100644 --- a/pkg/clusterinfo/configmap.go +++ b/pkg/clusterinfo/configmap.go @@ -22,6 +22,7 @@ import ( "fmt" "io/ioutil" "net" + "strconv" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -78,7 +79,7 @@ func (p *KubeconfigProvider) getKubeconfigFromConfigMap(ctx context.Context) (*c func (p *KubeconfigProvider) buildKubeconfigFromEndpoint(ctx context.Context) (*clientcmdapi.Config, error) { e, err := p.kubeClient.CoreV1().Endpoints(metav1.NamespaceDefault).Get(ctx, kubernetesEndpointsName, metav1.GetOptions{}) if err != nil { - return nil, fmt.Errorf("failed to get endpoint from lister: %v", err) + return nil, fmt.Errorf("failed to get endpoint from lister: %w", err) } if len(e.Subsets) == 0 { @@ -109,11 +110,11 @@ func (p *KubeconfigProvider) buildKubeconfigFromEndpoint(ctx context.Context) (* if port == nil { return nil, errors.New("no secure port in the subset") } - url := fmt.Sprintf("https://%s:%d", ip.String(), port.Port) + url := fmt.Sprintf("https://%s", net.JoinHostPort(ip.String(), strconv.Itoa(int(port.Port)))) caData, err := getCAData(p.clientConfig) if err != nil { - return nil, fmt.Errorf("failed to get ca data from config: %v", err) + return nil, fmt.Errorf("failed to get ca data from config: %w", err) } return &clientcmdapi.Config{ diff --git a/pkg/containerruntime/config.go b/pkg/containerruntime/config.go index 13149b705..13faf255f 100644 --- a/pkg/containerruntime/config.go +++ b/pkg/containerruntime/config.go @@ -54,7 +54,7 @@ func BuildConfig(opts Opts) (Config, error) { _, err := url.Parse(trimmedMirror) if err != nil { - return Config{}, fmt.Errorf("incorrect mirror provided: %v", err) + return Config{}, fmt.Errorf("incorrect mirror provided: %w", err) } registryMirrors = append(registryMirrors, trimmedMirror) diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index 29f3e8012..ebf7a8c5a 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -227,7 +227,7 @@ type containerdRegistryTLSConfig struct { } // AuthConfig is a COPY of github.com/containerd/containerd/pkg/cri/config.AuthConfig. -// AuthConfig contains the config related to authentication to a specific registry +// AuthConfig contains the config related to authentication to a specific registry. type AuthConfig struct { // Username is the username to login the registry. Username string `toml:"username,omitempty" json:"username,omitempty"` diff --git a/pkg/controller/machine/bootstrap.go b/pkg/controller/machine/bootstrap.go index 92916f4a6..6e86b8fe3 100644 --- a/pkg/controller/machine/bootstrap.go +++ b/pkg/controller/machine/bootstrap.go @@ -37,7 +37,6 @@ import ( ) func getOSMBootstrapUserdata(ctx context.Context, client ctrlruntimeclient.Client, req plugin.UserDataRequest, secretName string) (string, error) { - var clusterName string for key := range req.Kubeconfig.Clusters { clusterName = key @@ -45,18 +44,18 @@ func getOSMBootstrapUserdata(ctx context.Context, client ctrlruntimeclient.Clien token, err := util.ExtractAPIServerToken(ctx, client) if err != nil { - return "", fmt.Errorf("failed to fetch api-server token: %v", err) + return "", fmt.Errorf("failed to fetch api-server token: %w", err) } // Retrieve provider config from machine pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) if err != nil { - return "", fmt.Errorf("failed to get providerSpec: %v", err) + return "", fmt.Errorf("failed to get providerSpec: %w", err) } bootstrapKubeconfig, err := helper.StringifyKubeconfig(req.Kubeconfig) if err != nil { - return "", fmt.Errorf("failed to format bootstrap kubeconfig: %v", err) + return "", fmt.Errorf("failed to format bootstrap kubeconfig: %w", err) } // Regardless if the provisioningUtility is set to use cloud-init, we only allow using ignition to provision flatcar @@ -65,11 +64,11 @@ func getOSMBootstrapUserdata(ctx context.Context, client ctrlruntimeclient.Clien return getOSMBootstrapUserDataForIgnition(req, pconfig.SSHPublicKeys, token, secretName, clusterName, bootstrapKubeconfig) } - // cloud-init is used for all other operating systems + // cloud-init is used for all other operating systems. return getOSMBootstrapUserDataForCloudInit(req, pconfig, token, secretName, clusterName, bootstrapKubeconfig) } -// getOSMBootstrapUserDataForIgnition returns the userdata for the ignition bootstrap config +// getOSMBootstrapUserDataForIgnition returns the userdata for the ignition bootstrap config. func getOSMBootstrapUserDataForIgnition(req plugin.UserDataRequest, sshPublicKeys []string, token, secretName, clusterName, bootstrapKfg string) (string, error) { data := struct { Token string @@ -82,16 +81,16 @@ func getOSMBootstrapUserDataForIgnition(req plugin.UserDataRequest, sshPublicKey } bsScript, err := template.New("bootstrap-script").Parse(ignitionBootstrapBinContentTemplate) if err != nil { - return "", fmt.Errorf("failed to parse bootstrapBinContentTemplate template for ignition: %v", err) + return "", fmt.Errorf("failed to parse bootstrapBinContentTemplate template for ignition: %w", err) } script := &bytes.Buffer{} err = bsScript.Execute(script, data) if err != nil { - return "", fmt.Errorf("failed to execute bootstrapBinContentTemplate template for ignition: %v", err) + return "", fmt.Errorf("failed to execute bootstrapBinContentTemplate template for ignition: %w", err) } bsIgnitionConfig, err := template.New("bootstrap-ignition-config").Funcs(sprig.TxtFuncMap()).Parse(ignitionTemplate) if err != nil { - return "", fmt.Errorf("failed to parse bootstrap-ignition-config template: %v", err) + return "", fmt.Errorf("failed to parse bootstrap-ignition-config template: %w", err) } ignitionConfig := &bytes.Buffer{} @@ -109,13 +108,13 @@ func getOSMBootstrapUserDataForIgnition(req plugin.UserDataRequest, sshPublicKey BootstrapKubeconfig: bootstrapKfg, }) if err != nil { - return "", fmt.Errorf("failed to execute ignitionTemplate template: %v", err) + return "", fmt.Errorf("failed to execute ignitionTemplate template: %w", err) } return convert.ToIgnition(ignitionConfig.String()) } -// getOSMBootstrapUserDataForCloudInit returns the userdata for the cloud-init bootstrap script +// getOSMBootstrapUserDataForCloudInit returns the userdata for the cloud-init bootstrap script. func getOSMBootstrapUserDataForCloudInit(req plugin.UserDataRequest, pconfig *providerconfigtypes.Config, token, secretName, clusterName, bootstrapKfg string) (string, error) { data := struct { Token string @@ -143,23 +142,23 @@ func getOSMBootstrapUserDataForCloudInit(req plugin.UserDataRequest, pconfig *pr case providerconfigtypes.OperatingSystemUbuntu: bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapAptBinContentTemplate) if err != nil { - return "", fmt.Errorf("failed to parse bootstrapAptBinContentTemplate template: %v", err) + return "", fmt.Errorf("failed to parse bootstrapAptBinContentTemplate template: %w", err) } case providerconfigtypes.OperatingSystemCentOS: data.EnterpriseLinux = true bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapYumBinContentTemplate) if err != nil { - return "", fmt.Errorf("failed to parse bootstrapYumBinContentTemplate template: %v", err) + return "", fmt.Errorf("failed to parse bootstrapYumBinContentTemplate template: %w", err) } case providerconfigtypes.OperatingSystemAmazonLinux2: bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapYumBinContentTemplate) if err != nil { - return "", fmt.Errorf("failed to parse bootstrapYumBinContentTemplate template: %v", err) + return "", fmt.Errorf("failed to parse bootstrapYumBinContentTemplate template: %w", err) } case providerconfigtypes.OperatingSystemSLES: bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapZypperBinContentTemplate) if err != nil { - return "", fmt.Errorf("failed to parse bootstrapZypperBinContentTemplate template: %v", err) + return "", fmt.Errorf("failed to parse bootstrapZypperBinContentTemplate template: %w", err) } case providerconfigtypes.OperatingSystemRHEL: rhelConfig, err = rhel.LoadConfig(pconfig.OperatingSystemSpec) @@ -168,18 +167,18 @@ func getOSMBootstrapUserDataForCloudInit(req plugin.UserDataRequest, pconfig *pr } bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapYumBinContentTemplate) if err != nil { - return "", fmt.Errorf("failed to parse bootstrapYumBinContentTemplate template: %v", err) + return "", fmt.Errorf("failed to parse bootstrapYumBinContentTemplate template: %w", err) } } script := &bytes.Buffer{} err = bsScript.Execute(script, data) if err != nil { - return "", fmt.Errorf("failed to execute bootstrap script template: %v", err) + return "", fmt.Errorf("failed to execute bootstrap script template: %w", err) } bsCloudInit, err := template.New("bootstrap-cloud-init").Parse(cloudInitTemplate) if err != nil { - return "", fmt.Errorf("failed to parse download-binaries template: %v", err) + return "", fmt.Errorf("failed to parse download-binaries template: %w", err) } cloudInit := &bytes.Buffer{} @@ -199,7 +198,7 @@ func getOSMBootstrapUserDataForCloudInit(req plugin.UserDataRequest, pconfig *pr RHELConfig: rhelConfig, }) if err != nil { - return "", fmt.Errorf("failed to execute cloudInitTemplate template: %v", err) + return "", fmt.Errorf("failed to execute cloudInitTemplate template: %w", err) } return cloudInit.String(), nil } diff --git a/pkg/controller/machine/kubeconfig.go b/pkg/controller/machine/kubeconfig.go index ca960a47e..f4c91036f 100644 --- a/pkg/controller/machine/kubeconfig.go +++ b/pkg/controller/machine/kubeconfig.go @@ -43,7 +43,7 @@ const ( tokenSecretKey string = "token-secret" expirationKey string = "expiration" tokenFormatter string = "%s.%s" - // Keep this short, userdata is limited + // Keep this short, userdata is limited. contextIdentifier string = "c" ) @@ -54,12 +54,12 @@ func (r *Reconciler) createBootstrapKubeconfig(ctx context.Context, name string) if r.bootstrapTokenServiceAccountName != nil { token, err = r.getTokenFromServiceAccount(ctx, *r.bootstrapTokenServiceAccountName) if err != nil { - return nil, fmt.Errorf("failed to get token from ServiceAccount %s/%s: %v", r.bootstrapTokenServiceAccountName.Namespace, r.bootstrapTokenServiceAccountName.Name, err) + return nil, fmt.Errorf("failed to get token from ServiceAccount %s/%s: %w", r.bootstrapTokenServiceAccountName.Namespace, r.bootstrapTokenServiceAccountName.Name, err) } } else { token, err = r.createBootstrapToken(ctx, name) if err != nil { - return nil, fmt.Errorf("failed to create bootstrap token: %v", err) + return nil, fmt.Errorf("failed to create bootstrap token: %w", err) } } @@ -100,14 +100,14 @@ func (r *Reconciler) getTokenFromServiceAccount(ctx context.Context, name types. sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: name.Name, Namespace: name.Namespace}} raw, err := r.getAsUnstructured(ctx, sa) if err != nil { - return "", fmt.Errorf("failed to get serviceAccount %q: %v", name.String(), err) + return "", fmt.Errorf("failed to get serviceAccount %q: %w", name.String(), err) } sa = raw.(*corev1.ServiceAccount) for _, serviceAccountSecretName := range sa.Secrets { serviceAccountSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: sa.Namespace, Name: serviceAccountSecretName.Name}} raw, err = r.getAsUnstructured(ctx, serviceAccountSecret) if err != nil { - return "", fmt.Errorf("failed to get serviceAccountSecret: %v", err) + return "", fmt.Errorf("failed to get serviceAccountSecret: %w", err) } serviceAccountSecret = raw.(*corev1.Secret) if serviceAccountSecret.Type != corev1.SecretTypeServiceAccountToken { @@ -149,7 +149,7 @@ func (r *Reconciler) createBootstrapToken(ctx context.Context, name string) (str } if err := r.client.Create(ctx, &secret); err != nil { - return "", fmt.Errorf("failed to create bootstrap token secret: %v", err) + return "", fmt.Errorf("failed to create bootstrap token secret: %w", err) } return fmt.Sprintf(tokenFormatter, tokenID, tokenSecret), nil @@ -168,7 +168,7 @@ func (r *Reconciler) updateSecretExpirationAndGetToken(ctx context.Context, secr return "", err } - //If the token is close to expire, reset it's expiration time + // If the token is close to expire, reset it's expiration time if time.Until(expirationTime).Minutes() < 30 { secret.Data[expirationKey] = []byte(metav1.Now().Add(1 * time.Hour).Format(time.RFC3339)) } else { @@ -176,7 +176,7 @@ func (r *Reconciler) updateSecretExpirationAndGetToken(ctx context.Context, secr } if err := r.client.Update(ctx, secret); err != nil { - return "", fmt.Errorf("failed to update secret: %v", err) + return "", fmt.Errorf("failed to update secret: %w", err) } return token, nil } @@ -215,7 +215,7 @@ func (r *Reconciler) getAsUnstructured(ctx context.Context, obj runtime.Object) } kinds, _, err := scheme.Scheme.ObjectKinds(obj) if err != nil { - return nil, fmt.Errorf("failed to get kinds for object: %v", err) + return nil, fmt.Errorf("failed to get kinds for object: %w", err) } if len(kinds) == 0 { return nil, fmt.Errorf("found no kind for object %t", obj) @@ -228,15 +228,15 @@ func (r *Reconciler) getAsUnstructured(ctx context.Context, obj runtime.Object) name := types.NamespacedName{Name: metaObj.GetName(), Namespace: metaObj.GetNamespace()} if err := r.client.Get(ctx, name, target); err != nil { - return nil, fmt.Errorf("failed to get object: %v", err) + return nil, fmt.Errorf("failed to get object: %w", err) } rawJSON, err := target.MarshalJSON() if err != nil { - return nil, fmt.Errorf("failed to marshal unstructured.Unstructured: %v", err) + return nil, fmt.Errorf("failed to marshal unstructured.Unstructured: %w", err) } if err := json.Unmarshal(rawJSON, obj); err != nil { - return nil, fmt.Errorf("failed to marshal unstructured.Unstructued into %T: %v", obj, err) + return nil, fmt.Errorf("failed to marshal unstructured.Unstructued into %T: %w", obj, err) } return obj, nil } diff --git a/pkg/controller/machine/kubeconfig_test.go b/pkg/controller/machine/kubeconfig_test.go index b058d3f4f..731080762 100644 --- a/pkg/controller/machine/kubeconfig_test.go +++ b/pkg/controller/machine/kubeconfig_test.go @@ -24,8 +24,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" ctrlruntimefake "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -60,7 +60,11 @@ func TestUpdateSecretExpirationAndGetToken(t *testing.T) { data[tokenIDKey] = []byte("tokenID") data[expirationKey] = []byte(testCase.initialExperirationTime.Format(time.RFC3339)) secret.Data = data - reconciler.client = ctrlruntimefake.NewFakeClient(runtime.Object(secret)) + reconciler.client = ctrlruntimefake. + NewClientBuilder(). + WithScheme(scheme.Scheme). + WithObjects(secret). + Build() if _, err := reconciler.updateSecretExpirationAndGetToken(ctx, secret); err != nil { t.Fatalf("Unexpected error running updateSecretExpirationAndGetToken: %v", err) @@ -92,6 +96,5 @@ func TestUpdateSecretExpirationAndGetToken(t *testing.T) { if time.Until(expirationTimeParsed).Minutes() < 0 { t.Errorf("Error, secret expiration is in the past!") } - } } diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 95870092b..fda75629a 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -83,7 +83,7 @@ const ( // AnnotationMachineUninitialized indicates that a machine is not yet // ready to be worked on by the machine-controller. The machine-controller // will ignore all machines that have this anotation with any value - // Its value should consist of one or more initializers, separated by a comma + // Its value should consist of one or more initializers, separated by a comma. AnnotationMachineUninitialized = "machine-controller.kubermatic.io/initializers" deletionRetryWaitPeriod = 10 * time.Second @@ -92,13 +92,13 @@ const ( NodeOwnerLabelName = "machine-controller/owned-by" // AnnotationAutoscalerIdentifier is used by the cluster-autoscaler - // cluster-api provider to match Nodes to Machines + // cluster-api provider to match Nodes to Machines. AnnotationAutoscalerIdentifier = "cluster.k8s.io/machine" provisioningSuffix = "osc-provisioning" ) -// Reconciler is the controller implementation for machine resources +// Reconciler is the controller implementation for machine resources. type Reconciler struct { kubeClient kubernetes.Interface client ctrlruntimeclient.Client @@ -197,7 +197,7 @@ func Add( } m, err := userdatamanager.New() if err != nil { - return fmt.Errorf("failed to create userdatamanager: %v", err) + return fmt.Errorf("failed to create userdatamanager: %w", err) } reconciler.userDataManager = m @@ -221,7 +221,7 @@ func Add( handler.EnqueueRequestsFromMapFunc(func(node client.Object) (result []reconcile.Request) { machinesList := &clusterv1alpha1.MachineList{} if err := mgr.GetClient().List(ctx, machinesList); err != nil { - utilruntime.HandleError(fmt.Errorf("failed to list machines in lister: %v", err)) + utilruntime.HandleError(fmt.Errorf("failed to list machines in lister: %w", err)) return } @@ -281,14 +281,14 @@ func Add( } // clearMachineError is a convenience function to remove a error on the machine if its set. -// It does not return an error as it's used around the sync handler +// It does not return an error as it's used around the sync handler. func (r *Reconciler) clearMachineError(machine *clusterv1alpha1.Machine) { if machine.Status.ErrorMessage != nil || machine.Status.ErrorReason != nil { if err := r.updateMachine(machine, func(m *clusterv1alpha1.Machine) { m.Status.ErrorMessage = nil m.Status.ErrorReason = nil }); err != nil { - utilruntime.HandleError(fmt.Errorf("failed to update machine: %v", err)) + utilruntime.HandleError(fmt.Errorf("failed to update machine: %w", err)) } } } @@ -317,7 +317,7 @@ func (r *Reconciler) updateMachine(m *clusterv1alpha1.Machine, modify ...cloudpr } // updateMachine updates machine's ErrorMessage and ErrorReason regardless if they were set or not -// this essentially overwrites previous values +// this essentially overwrites previous values. func (r *Reconciler) updateMachineError(machine *clusterv1alpha1.Machine, reason common.MachineStatusError, message string) error { return r.updateMachine(machine, func(m *clusterv1alpha1.Machine) { m.Status.ErrorMessage = &message @@ -327,22 +327,22 @@ func (r *Reconciler) updateMachineError(machine *clusterv1alpha1.Machine, reason // updateMachineErrorIfTerminalError is a convenience method that will update machine's Status if the given err is terminal // and at the same time terminal error will be returned to the caller -// otherwise it will return formatted error according to errMsg +// otherwise it will return formatted error according to errMsg. func (r *Reconciler) updateMachineErrorIfTerminalError(machine *clusterv1alpha1.Machine, stReason common.MachineStatusError, stMessage string, err error, errMsg string) error { if ok, _, _ := cloudprovidererrors.IsTerminalError(err); ok { if errNested := r.updateMachineError(machine, stReason, stMessage); errNested != nil { - return fmt.Errorf("failed to update machine error after due to %v, terminal error = %v", errNested, stMessage) + return fmt.Errorf("failed to update machine error after due to %w, terminal error = %v", errNested, stMessage) } return err } - return fmt.Errorf("%s, due to %v", errMsg, err) + return fmt.Errorf("%s, due to %w", errMsg, err) } func (r *Reconciler) createProviderInstance(prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { - // Ensure finalizer is there + // Ensure finalizer is there. _, err := r.ensureDeleteFinalizerExists(machine) if err != nil { - return nil, fmt.Errorf("failed to add %q finalizer: %v", FinalizerDeleteInstance, err) + return nil, fmt.Errorf("failed to add %q finalizer: %w", FinalizerDeleteInstance, err) } i, err := prov.Create(machine, r.providerData, userdata) if err != nil { @@ -397,12 +397,12 @@ func (r *Reconciler) reconcile(ctx context.Context, machine *clusterv1alpha1.Mac providerConfig, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) if err != nil { - return nil, fmt.Errorf("failed to get provider config: %v", err) + return nil, fmt.Errorf("failed to get provider config: %w", err) } skg := providerconfig.NewConfigVarResolver(ctx, r.client) prov, err := cloudprovider.ForProvider(providerConfig.CloudProvider, skg) if err != nil { - return nil, fmt.Errorf("failed to get cloud provider %q: %v", providerConfig.CloudProvider, err) + return nil, fmt.Errorf("failed to get cloud provider %q: %w", providerConfig.CloudProvider, err) } // step 2: check if a user requested to delete the machine @@ -413,7 +413,7 @@ func (r *Reconciler) reconcile(ctx context.Context, machine *clusterv1alpha1.Mac // Step 3: Essentially creates an instance for the given machine. userdataPlugin, err := r.userDataManager.ForOS(providerConfig.OperatingSystem) if err != nil { - return nil, fmt.Errorf("failed to userdata provider for '%s': %v", providerConfig.OperatingSystem, err) + return nil, fmt.Errorf("failed to userdata provider for '%s': %w", providerConfig.OperatingSystem, err) } // case 3.2: creates an instance if there is no node associated with the given machine @@ -430,14 +430,14 @@ func (r *Reconciler) reconcile(ctx context.Context, machine *clusterv1alpha1.Mac m.Status.NodeRef = nil }) } - return nil, fmt.Errorf("failed to check if node for machine exists: '%s'", err) + return nil, fmt.Errorf("failed to check if node for machine exists: '%w'", err) } if nodeIsReady(node) { // We must do this to ensure the informers in the machineSet and machineDeployment controller // get triggered as soon as a ready node exists for a machine if err := r.ensureMachineHasNodeReadyCondition(machine); err != nil { - return nil, fmt.Errorf("failed to set nodeReady condition on machine: %v", err) + return nil, fmt.Errorf("failed to set nodeReady condition on machine: %w", err) } } else { // Node is not ready anymore? Maybe it got deleted @@ -485,7 +485,7 @@ func (r *Reconciler) shouldCleanupVolumes(ctx context.Context, machine *clusterv return true, nil } -// evictIfNecessary checks if the machine has a node and evicts it if necessary +// evictIfNecessary checks if the machine has a node and evicts it if necessary. func (r *Reconciler) shouldEvict(ctx context.Context, machine *clusterv1alpha1.Machine) (bool, error) { // If the deletion got triggered a few hours ago, skip eviction. // We assume here that the eviction is blocked by misconfiguration or a misbehaving kubelet and/or controller-runtime @@ -516,7 +516,7 @@ func (r *Reconciler) shouldEvict(ctx context.Context, machine *clusterv1alpha1.M // * There is at least one Node that is schedulable (`.Spec.Unschedulable == false`) machines := &clusterv1alpha1.MachineList{} if err := r.client.List(ctx, machines); err != nil { - return false, fmt.Errorf("failed to get machines from lister: %v", err) + return false, fmt.Errorf("failed to get machines from lister: %w", err) } for _, machine := range machines.Items { if machine.Status.NodeRef == nil { @@ -525,7 +525,7 @@ func (r *Reconciler) shouldEvict(ctx context.Context, machine *clusterv1alpha1.M } nodes := &corev1.NodeList{} if err := r.client.List(ctx, nodes); err != nil { - return false, fmt.Errorf("failed to get nodes from lister: %v", err) + return false, fmt.Errorf("failed to get nodes from lister: %w", err) } for _, node := range nodes.Items { // Don't consider our own node a valid target @@ -559,13 +559,13 @@ func (r *Reconciler) deleteMachine(ctx context.Context, prov cloudprovidertypes. if shouldEvict { evictedSomething, err = eviction.New(ctx, machine.Status.NodeRef.Name, r.client, r.kubeClient).Run() if err != nil { - return nil, fmt.Errorf("failed to evict node %s: %v", machine.Status.NodeRef.Name, err) + return nil, fmt.Errorf("failed to evict node %s: %w", machine.Status.NodeRef.Name, err) } } if shouldCleanUpVolumes { deletedSomething, volumesFree, err = poddeletion.New(ctx, machine.Status.NodeRef.Name, r.client, r.kubeClient).Run() if err != nil { - return nil, fmt.Errorf("failed to delete pods bound to volumes running on node %s: %v", machine.Status.NodeRef.Name, err) + return nil, fmt.Errorf("failed to delete pods bound to volumes running on node %s: %w", machine.Status.NodeRef.Name, err) } } @@ -605,7 +605,7 @@ func (r *Reconciler) retrieveNodesRelatedToMachine(ctx context.Context, machine node := &corev1.Node{} if err := r.client.Get(ctx, objKey, node); err != nil { if !kerrors.IsNotFound(err) { - return nil, fmt.Errorf("failed to get node %s: %v", machine.Status.NodeRef.Name, err) + return nil, fmt.Errorf("failed to get node %s: %w", machine.Status.NodeRef.Name, err) } klog.V(2).Infof("node %q does not longer exist for machine %q", machine.Status.NodeRef.Name, machine.Spec.Name) } else { @@ -614,20 +614,20 @@ func (r *Reconciler) retrieveNodesRelatedToMachine(ctx context.Context, machine } else { selector, err := labels.Parse(NodeOwnerLabelName + "=" + string(machine.UID)) if err != nil { - return nil, fmt.Errorf("failed to parse label selector: %v", err) + return nil, fmt.Errorf("failed to parse label selector: %w", err) } listOpts := &ctrlruntimeclient.ListOptions{LabelSelector: selector} nodeList := &corev1.NodeList{} if err := r.client.List(ctx, nodeList, listOpts); err != nil { - return nil, fmt.Errorf("failed to list nodes: %v", err) + return nil, fmt.Errorf("failed to list nodes: %w", err) } if len(nodeList.Items) == 0 { // We just want log that we didn't found the node. klog.V(3).Infof("No node found for the machine %s", machine.Spec.Name) } - for _, node := range nodeList.Items { - nodes = append(nodes, &node) + for i := range nodeList.Items { + nodes = append(nodes, &nodeList.Items[i]) } } @@ -654,13 +654,13 @@ func (r *Reconciler) deleteCloudProviderInstance(prov cloudprovidertypes.Provide machineConfig, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) if err != nil { - return nil, fmt.Errorf("failed to get provider config: %v", err) + return nil, fmt.Errorf("failed to get provider config: %w", err) } if machineConfig.OperatingSystem == providerconfigtypes.OperatingSystemRHEL { rhelConfig, err := rhel.LoadConfig(machineConfig.OperatingSystemSpec) if err != nil { - return nil, fmt.Errorf("failed to get rhel os specs: %v", err) + return nil, fmt.Errorf("failed to get rhel os specs: %w", err) } machineName := machine.Name @@ -674,7 +674,7 @@ func (r *Reconciler) deleteCloudProviderInstance(prov cloudprovidertypes.Provide if rhelConfig.RHSMOfflineToken != "" { if err := r.redhatSubscriptionManager.UnregisterInstance(rhelConfig.RHSMOfflineToken, machineName); err != nil { - return nil, fmt.Errorf("failed to delete subscription for machine name %s: %v", machine.Name, err) + return nil, fmt.Errorf("failed to delete subscription for machine name %s: %w", machine.Name, err) } } @@ -686,14 +686,13 @@ func (r *Reconciler) deleteCloudProviderInstance(prov cloudprovidertypes.Provide rhelConfig.RHELSubscriptionManagerPassword, rhelConfig.RHELSatelliteServer) if err != nil { - return nil, fmt.Errorf("failed to delete redhat satellite host for machine name %s: %v", machine.Name, err) + return nil, fmt.Errorf("failed to delete redhat satellite host for machine name %s: %w", machine.Name, err) } - } } if err := rhsm.RemoveRHELSubscriptionFinalizer(machine, r.updateMachine); err != nil { - return nil, fmt.Errorf("failed to remove redhat subscription finalizer: %v", err) + return nil, fmt.Errorf("failed to remove redhat subscription finalizer: %w", err) } } @@ -738,19 +737,18 @@ func (r *Reconciler) ensureInstanceExistsForMachine( // case 2: retrieving instance from provider was not successful if err != nil { - // case 2.1: instance was not found and we are going to create one - if err == cloudprovidererrors.ErrInstanceNotFound { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { klog.V(3).Infof("Validated machine spec of %s", machine.Name) kubeconfig, err := r.createBootstrapKubeconfig(ctx, machine.Name) if err != nil { - return nil, fmt.Errorf("failed to create bootstrap kubeconfig: %v", err) + return nil, fmt.Errorf("failed to create bootstrap kubeconfig: %w", err) } cloudConfig, kubeletCloudProviderName, err := prov.GetCloudConfig(machine.Spec) if err != nil { - return nil, fmt.Errorf("failed to render cloud config: %v", err) + return nil, fmt.Errorf("failed to render cloud config: %w", err) } // grab kubelet featureGates from the annotations @@ -772,7 +770,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( registryCredentials, err := containerruntime.GetContainerdAuthConfig(ctx, r.client, r.nodeSettings.RegistryCredentialsSecretRef) if err != nil { - return nil, fmt.Errorf("failed to get containerd auth config: %v", err) + return nil, fmt.Errorf("failed to get containerd auth config: %w", err) } crRuntime := r.nodeSettings.ContainerRuntime @@ -809,7 +807,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( if r.useOSM { referencedMachineDeployment, err := controllerutil.GetMachineDeploymentNameForMachine(ctx, machine, r.client) if err != nil { - return nil, fmt.Errorf("failed to find machine's MachineDployment: %v", err) + return nil, fmt.Errorf("failed to find machine's MachineDployment: %w", err) } cloudConfigSecretName := fmt.Sprintf("%s-%s-%s", @@ -827,17 +825,17 @@ func (r *Reconciler) ensureInstanceExistsForMachine( userdata, err = getOSMBootstrapUserdata(ctx, r.client, req, cloudConfigSecretName) if err != nil { - return nil, fmt.Errorf("failed get OSM userdata: %v", err) + return nil, fmt.Errorf("failed get OSM userdata: %w", err) } userdata, err = cleanupTemplateOutput(userdata) if err != nil { - return nil, fmt.Errorf("failed to cleanup user-data template: %v", err) + return nil, fmt.Errorf("failed to cleanup user-data template: %w", err) } } else { userdata, err = userdataPlugin.UserData(req) if err != nil { - return nil, fmt.Errorf("failed get userdata: %v", err) + return nil, fmt.Errorf("failed get userdata: %w", err) } } @@ -848,7 +846,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( } if providerConfig.OperatingSystem == providerconfigtypes.OperatingSystemRHEL { if err := rhsm.AddRHELSubscriptionFinalizer(machine, r.updateMachine); err != nil { - return nil, fmt.Errorf("failed to add redhat subscription finalizer: %v", err) + return nil, fmt.Errorf("failed to add redhat subscription finalizer: %w", err) } } r.recorder.Event(machine, corev1.EventTypeNormal, "Created", "Successfully created instance") @@ -864,7 +862,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( } // case 2.3: transient error was returned, requeue the request and try again in the future - return nil, fmt.Errorf("failed to get instance from provider: %v", err) + return nil, fmt.Errorf("failed to get instance from provider: %w", err) } // Instance exists, so ensure finalizer does as well machine, err = r.ensureDeleteFinalizerExists(machine) @@ -884,7 +882,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( if err := r.updateMachine(machine, func(m *clusterv1alpha1.Machine) { m.Status.Addresses = machineAddresses }); err != nil { - return nil, fmt.Errorf("failed to update machine after setting .status.addresses: %v", err) + return nil, fmt.Errorf("failed to update machine after setting .status.addresses: %w", err) } return r.ensureNodeOwnerRefAndConfigSource(ctx, providerInstance, machine, providerConfig) } @@ -892,7 +890,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( func (r *Reconciler) ensureNodeOwnerRefAndConfigSource(ctx context.Context, providerInstance instance.Instance, machine *clusterv1alpha1.Machine, providerConfig *providerconfigtypes.Config) (*reconcile.Result, error) { node, exists, err := r.getNode(ctx, providerInstance, providerConfig.CloudProvider) if err != nil { - return nil, fmt.Errorf("failed to get node for machine %s: %v", machine.Name, err) + return nil, fmt.Errorf("failed to get node for machine %s: %w", machine.Name, err) } if exists { @@ -900,20 +898,20 @@ func (r *Reconciler) ensureNodeOwnerRefAndConfigSource(ctx context.Context, prov if err := r.updateNode(ctx, node, func(n *corev1.Node) { n.Labels[NodeOwnerLabelName] = string(machine.UID) }); err != nil { - return nil, fmt.Errorf("failed to update node %q after adding owner label: %v", node.Name, err) + return nil, fmt.Errorf("failed to update node %q after adding owner label: %w", node.Name, err) } } if node.Spec.ConfigSource == nil && machine.Spec.ConfigSource != nil { - if err := r.updateNode(ctx, node, func(n *corev1.Node) { - n.Spec.ConfigSource = machine.Spec.ConfigSource + if err := r.updateNode(ctx, node, func(node *corev1.Node) { + node.Spec.ConfigSource = machine.Spec.ConfigSource }); err != nil { - return nil, fmt.Errorf("failed to update node %s after setting the config source: %v", node.Name, err) + return nil, fmt.Errorf("failed to update node %s after setting the config source: %w", node.Name, err) } klog.V(3).Infof("Added config source to node %s (machine %s)", node.Name, machine.Name) } if err := r.updateMachineStatus(machine, node); err != nil { - return nil, fmt.Errorf("failed to update machine status: %v", err) + return nil, fmt.Errorf("failed to update machine status: %w", err) } } else { // If the machine has an owner Ref and joinClusterTimeout is configured and reached, delete it to have it re-created by the MachineSet controller @@ -922,7 +920,7 @@ func (r *Reconciler) ensureNodeOwnerRefAndConfigSource(ctx context.Context, prov if time.Since(machine.CreationTimestamp.Time) > *r.joinClusterTimeout { klog.V(3).Infof("Join cluster timeout expired for machine %s, deleting it", machine.Name) if err := r.client.Delete(ctx, machine); err != nil { - return nil, fmt.Errorf("failed to delete machine %s/%s that didn't join cluster within expected period of %s: %v", + return nil, fmt.Errorf("failed to delete machine %s/%s that didn't join cluster within expected period of %s: %w", machine.Namespace, machine.Name, r.joinClusterTimeout.String(), err) } return nil, nil @@ -998,7 +996,7 @@ func (r *Reconciler) ensureNodeLabelsAnnotationsAndTaints(ctx context.Context, n if len(modifiers) > 0 { if err := r.updateNode(ctx, node, modifiers...); err != nil { - return fmt.Errorf("failed to update node %s after setting labels/annotations/taints: %v", node.Name, err) + return fmt.Errorf("failed to update node %s after setting labels/annotations/taints: %w", node.Name, err) } r.recorder.Event(machine, corev1.EventTypeNormal, "LabelsAnnotationsTaintsUpdated", "Successfully updated labels/annotations/taints") klog.V(3).Infof("Added labels/annotations/taints to node %s (machine %s)", node.Name, machine.Name) @@ -1014,7 +1012,7 @@ func (r *Reconciler) updateMachineStatus(machine *clusterv1alpha1.Machine, node ref, err := reference.GetReference(scheme.Scheme, node) if err != nil { - return fmt.Errorf("failed to get node reference for %s : %v", node.Name, err) + return fmt.Errorf("failed to get node reference for %s : %w", node.Name, err) } if !equality.Semantic.DeepEqual(machine.Status.NodeRef, ref) || machine.Status.Versions == nil || @@ -1023,7 +1021,7 @@ func (r *Reconciler) updateMachineStatus(machine *clusterv1alpha1.Machine, node m.Status.NodeRef = ref m.Status.Versions = &clusterv1alpha1.MachineVersionInfo{Kubelet: node.Status.NodeInfo.KubeletVersion} }); err != nil { - return fmt.Errorf("failed to update machine after setting its status: %v", err) + return fmt.Errorf("failed to update machine after setting its status: %w", err) } } @@ -1091,7 +1089,7 @@ func (r *Reconciler) ReadinessChecks(ctx context.Context) map[string]healthcheck "valid-info-kubeconfig": func() error { cm, err := r.kubeconfigProvider.GetKubeconfig(ctx) if err != nil { - err := fmt.Errorf("failed to get cluster-info configmap: %v", err) + err := fmt.Errorf("failed to get cluster-info configmap: %w", err) klog.V(2).Info(err) return err } @@ -1125,7 +1123,7 @@ func (r *Reconciler) ensureDeleteFinalizerExists(machine *clusterv1alpha1.Machin finalizers.Insert(FinalizerDeleteNode) m.Finalizers = finalizers.List() }); err != nil { - return nil, fmt.Errorf("failed to update machine after adding the delete instance finalizer: %v", err) + return nil, fmt.Errorf("failed to update machine after adding the delete instance finalizer: %w", err) } klog.V(3).Infof("Added delete finalizer to machine %s", machine.Name) } diff --git a/pkg/controller/machine/machine_test.go b/pkg/controller/machine/machine_test.go index a9322ca32..d528ec4f6 100644 --- a/pkg/controller/machine/machine_test.go +++ b/pkg/controller/machine/machine_test.go @@ -32,13 +32,13 @@ import ( corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" "k8s.io/klog" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ctrlruntimefake "sigs.k8s.io/controller-runtime/pkg/client/fake" + fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) func init() { @@ -190,11 +190,15 @@ func TestController_GetNode(t *testing.T) { t.Run(test.name, func(t *testing.T) { ctx := context.Background() - nodes := []runtime.Object{} + nodes := []ctrlruntimeclient.Object{} for _, node := range nodeList { nodes = append(nodes, node) } - client := ctrlruntimefake.NewFakeClient(nodes...) + client := fakectrlruntimeclient.NewClientBuilder(). + WithScheme(scheme.Scheme). + WithObjects(nodes...). + Build() + reconciler := Reconciler{client: client} node, exists, err := reconciler.getNode(ctx, test.instance, test.provider) @@ -298,7 +302,10 @@ func TestControllerDeletesMachinesOnJoinTimeout(t *testing.T) { providerConfig := &providerconfigtypes.Config{CloudProvider: providerconfigtypes.CloudProviderFake} - client := ctrlruntimefake.NewFakeClient(node, machine) + client := fakectrlruntimeclient.NewClientBuilder(). + WithScheme(scheme.Scheme). + WithObjects(node, machine). + Build() reconciler := Reconciler{ client: client, @@ -318,7 +325,6 @@ func TestControllerDeletesMachinesOnJoinTimeout(t *testing.T) { } }) } - } func durationPtr(d time.Duration) *time.Duration { @@ -332,14 +338,14 @@ func TestControllerShouldEvict(t *testing.T) { tests := []struct { name string machine *clusterv1alpha1.Machine - additionalMachines []runtime.Object - existingNodes []runtime.Object + additionalMachines []ctrlruntimeclient.Object + existingNodes []ctrlruntimeclient.Object shouldEvict bool }{ { name: "skip eviction due to eviction timeout", shouldEvict: false, - existingNodes: []runtime.Object{&corev1.Node{ + existingNodes: []ctrlruntimeclient.Object{&corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "existing-node", }, @@ -379,7 +385,7 @@ func TestControllerShouldEvict(t *testing.T) { }, { name: "Skip eviction due to no available target", - existingNodes: []runtime.Object{&corev1.Node{ + existingNodes: []ctrlruntimeclient.Object{&corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "existing-node", }, @@ -396,7 +402,7 @@ func TestControllerShouldEvict(t *testing.T) { { name: "Eviction possible because of second node", shouldEvict: true, - existingNodes: []runtime.Object{&corev1.Node{ + existingNodes: []ctrlruntimeclient.Object{&corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "existing-node", }}, &corev1.Node{ @@ -416,7 +422,7 @@ func TestControllerShouldEvict(t *testing.T) { { name: "Eviction possible because of machine without noderef", shouldEvict: true, - existingNodes: []runtime.Object{&corev1.Node{ + existingNodes: []ctrlruntimeclient.Object{&corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "existing-node", }}, &corev1.Node{ @@ -432,7 +438,7 @@ func TestControllerShouldEvict(t *testing.T) { NodeRef: &corev1.ObjectReference{Name: "existing-node"}, }, }, - additionalMachines: []runtime.Object{ + additionalMachines: []ctrlruntimeclient.Object{ &clusterv1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{ Name: "new-machine-without-a-node", @@ -446,10 +452,14 @@ func TestControllerShouldEvict(t *testing.T) { t.Run(test.name, func(t *testing.T) { ctx := context.Background() - objects := []runtime.Object{test.machine} + objects := []ctrlruntimeclient.Object{test.machine} objects = append(objects, test.existingNodes...) objects = append(objects, test.additionalMachines...) - client := ctrlruntimefake.NewFakeClient(objects...) + + client := ctrlruntimefake.NewClientBuilder(). + WithScheme(scheme.Scheme). + WithObjects(objects...). + Build() reconciler := &Reconciler{ client: client, @@ -584,12 +594,15 @@ func TestControllerDeleteNodeForMachine(t *testing.T) { t.Run(test.name, func(t *testing.T) { ctx := context.Background() - objects := []runtime.Object{test.machine} + objects := []ctrlruntimeclient.Object{test.machine} for _, n := range test.nodes { objects = append(objects, n) } - client := ctrlruntimefake.NewFakeClient(objects...) + client := fakectrlruntimeclient.NewClientBuilder(). + WithScheme(scheme.Scheme). + WithObjects(objects...). + Build() providerData := &cloudprovidertypes.ProviderData{ Ctx: ctx, diff --git a/pkg/controller/machine/metrics.go b/pkg/controller/machine/metrics.go index 0b2bb45ee..a5d375a93 100644 --- a/pkg/controller/machine/metrics.go +++ b/pkg/controller/machine/metrics.go @@ -101,7 +101,8 @@ func (l *machineMetricLabels) Counter(value uint) prometheus.Counter { } counterVec := prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: metricsPrefix + "machines", + Name: metricsPrefix + "machines_total", + Help: "Total number of machines", }, labelNames) counter := counterVec.With(labels) @@ -117,7 +118,7 @@ func NewMachineCollector(ctx context.Context, client ctrlruntimeclient.Client) * metricGatheringExecutor := func() { machines := &clusterv1alpha1.MachineList{} if err := client.List(ctx, machines); err != nil { - utilruntime.HandleError(fmt.Errorf("failed to list machines for SetMetricsForMachines: %v", err)) + utilruntime.HandleError(fmt.Errorf("failed to list machines for SetMetricsForMachines: %w", err)) return } var machineList clusterv1alpha1.MachineList @@ -132,7 +133,7 @@ func NewMachineCollector(ctx context.Context, client ctrlruntimeclient.Client) * for _, machine := range machines.Items { providerConfig, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) if err != nil { - utilruntime.HandleError(fmt.Errorf("failed to get providerSpec for SetMetricsForMachines: %v", err)) + utilruntime.HandleError(fmt.Errorf("failed to get providerSpec for SetMetricsForMachines: %w", err)) continue } if _, exists := providerMachineMap[providerConfig.CloudProvider]; !exists { @@ -144,15 +145,14 @@ func NewMachineCollector(ctx context.Context, client ctrlruntimeclient.Client) * for provider, providerMachineList := range providerMachineMap { prov, err := cloudprovider.ForProvider(provider, skg) if err != nil { - utilruntime.HandleError(fmt.Errorf("failed to get cloud provider for SetMetricsForMachines:: %q: %v", provider, err)) + utilruntime.HandleError(fmt.Errorf("failed to get cloud provider for SetMetricsForMachines:: %q: %w", provider, err)) continue } if err := prov.SetMetricsForMachines(*providerMachineList); err != nil { - utilruntime.HandleError(fmt.Errorf("failed to call prov.SetInstanceNumberForMachines: %v", err)) + utilruntime.HandleError(fmt.Errorf("failed to call prov.SetInstanceNumberForMachines: %w", err)) continue } } - } for { metricGatheringExecutor() @@ -218,19 +218,19 @@ func (mc MachineCollector) Collect(ch chan<- prometheus.Metric) { providerConfig, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) if err != nil { - utilruntime.HandleError(fmt.Errorf("failed to determine providerSpec for machine %s: %v", machine.Name, err)) + utilruntime.HandleError(fmt.Errorf("failed to determine providerSpec for machine %s: %w", machine.Name, err)) continue } provider, err := cloudprovider.ForProvider(providerConfig.CloudProvider, cvr) if err != nil { - utilruntime.HandleError(fmt.Errorf("failed to determine provider provider: %v", err)) + utilruntime.HandleError(fmt.Errorf("failed to determine provider provider: %w", err)) continue } labels, err := provider.MachineMetricsLabels(&machine) if err != nil { - utilruntime.HandleError(fmt.Errorf("failed to determine machine metrics labels: %v", err)) + utilruntime.HandleError(fmt.Errorf("failed to determine machine metrics labels: %w", err)) continue } diff --git a/pkg/controller/machinedeployment/machinedeployment_controller.go b/pkg/controller/machinedeployment/machinedeployment_controller.go index 7aa7f4c74..37528d471 100644 --- a/pkg/controller/machinedeployment/machinedeployment_controller.go +++ b/pkg/controller/machinedeployment/machinedeployment_controller.go @@ -41,7 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" ) -// controllerName is the name of this controller +// controllerName is the name of this controller. const controllerName = "machinedeployment-controller" var ( @@ -166,7 +166,6 @@ func (r *ReconcileMachineDeployment) reconcile(ctx context.Context, d *v1alpha1. } if !contains(d.Finalizers, metav1.FinalizerDeleteDependents) { - d.Finalizers = append(d.ObjectMeta.Finalizers, metav1.FinalizerDeleteDependents) if err := r.Client.Update(ctx, d); err != nil { klog.Infof("Failed to add finalizers to MachineSet %q: %v", d.Name, err) @@ -205,7 +204,6 @@ func (r *ReconcileMachineDeployment) reconcile(ctx context.Context, d *v1alpha1. // getMachineSetsForDeployment returns a list of MachineSets associated with a MachineDeployment. func (r *ReconcileMachineDeployment) getMachineSetsForDeployment(ctx context.Context, d *v1alpha1.MachineDeployment) ([]*v1alpha1.MachineSet, error) { - // List all MachineSets to find those we own but that no longer match our selector. machineSets := &v1alpha1.MachineSetList{} listOptions := &client.ListOptions{Namespace: d.Namespace} diff --git a/pkg/controller/machinedeployment/rolling.go b/pkg/controller/machinedeployment/rolling.go index 4ca3496e8..3267d9487 100644 --- a/pkg/controller/machinedeployment/rolling.go +++ b/pkg/controller/machinedeployment/rolling.go @@ -228,7 +228,7 @@ func (r *ReconcileMachineDeployment) cleanupUnhealthyReplicas(ctx context.Contex } // scaleDownOldMachineSetsForRollingUpdate scales down old machine sets when deployment strategy is "RollingUpdate". -// Need check maxUnavailable to ensure availability +// Need check maxUnavailable to ensure availability. func (r *ReconcileMachineDeployment) scaleDownOldMachineSetsForRollingUpdate(ctx context.Context, allMSs []*v1alpha1.MachineSet, oldMSs []*v1alpha1.MachineSet, deployment *v1alpha1.MachineDeployment) (int32, error) { if deployment.Spec.Replicas == nil { return 0, errors.Errorf("spec replicas for deployment %v is nil, this is unexpected", deployment.Name) diff --git a/pkg/controller/machinedeployment/sync.go b/pkg/controller/machinedeployment/sync.go index 51ede6fa7..8e700c4aa 100644 --- a/pkg/controller/machinedeployment/sync.go +++ b/pkg/controller/machinedeployment/sync.go @@ -331,7 +331,7 @@ func (r *ReconcileMachineDeployment) scale(ctx context.Context, deployment *clus return nil } -// syncDeploymentStatus checks if the status is up-to-date and sync it if necessary +// syncDeploymentStatus checks if the status is up-to-date and sync it if necessary. func (r *ReconcileMachineDeployment) syncDeploymentStatus(ctx context.Context, allMSs []*clusterv1alpha1.MachineSet, newMS *clusterv1alpha1.MachineSet, d *clusterv1alpha1.MachineDeployment) error { newStatus := calculateStatus(allMSs, newMS, d) if reflect.DeepEqual(d.Status, newStatus) { @@ -467,7 +467,7 @@ func (r *ReconcileMachineDeployment) updateMachineDeployment(ctx context.Context return updateMachineDeployment(ctx, r.Client, d, modify) } -// We have this as standalone variant to be able to use it from the tests +// We have this as standalone variant to be able to use it from the tests. func updateMachineDeployment(ctx context.Context, c client.Client, d *clusterv1alpha1.MachineDeployment, modify func(*clusterv1alpha1.MachineDeployment)) error { dCopy := d.DeepCopy() modify(dCopy) diff --git a/pkg/controller/machineset/delete_policy.go b/pkg/controller/machineset/delete_policy.go index 2c861ce24..44fee2cd3 100644 --- a/pkg/controller/machineset/delete_policy.go +++ b/pkg/controller/machineset/delete_policy.go @@ -46,7 +46,7 @@ const ( secondsPerTenDays float64 = 864000 ) -// maps the creation timestamp onto the 0-100 priority range +// maps the creation timestamp onto the 0-100 priority range. func oldestDeletePriority(machine *v1alpha1.Machine) deletePriority { if machine.DeletionTimestamp != nil && !machine.DeletionTimestamp.IsZero() { return mustDelete diff --git a/pkg/controller/machineset/machineset_controller.go b/pkg/controller/machineset/machineset_controller.go index 6a430ac7c..e43fe6def 100644 --- a/pkg/controller/machineset/machineset_controller.go +++ b/pkg/controller/machineset/machineset_controller.go @@ -43,7 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" ) -// controllerName is the name of this controller +// controllerName is the name of this controller. const controllerName = "machineset-controller" var ( @@ -105,7 +105,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) err ) } -// ReconcileMachineSet reconciles a MachineSet object +// ReconcileMachineSet reconciles a MachineSet object. type ReconcileMachineSet struct { client.Client scheme *runtime.Scheme @@ -233,10 +233,8 @@ func (r *ReconcileMachineSet) reconcile(ctx context.Context, machineSet *cluster if updatedMS.Spec.MinReadySeconds > 0 && updatedMS.Status.ReadyReplicas == replicas && updatedMS.Status.AvailableReplicas != replicas { - return reconcile.Result{RequeueAfter: time.Duration(updatedMS.Spec.MinReadySeconds) * time.Second}, nil } - return reconcile.Result{}, nil } diff --git a/pkg/controller/nodecsrapprover/node_csr_approver.go b/pkg/controller/nodecsrapprover/node_csr_approver.go index 9f0d60128..5997a105e 100644 --- a/pkg/controller/nodecsrapprover/node_csr_approver.go +++ b/pkg/controller/nodecsrapprover/node_csr_approver.go @@ -41,7 +41,7 @@ import ( ) const ( - // ControllerName is name of the NodeCSRApprover controller + // ControllerName is name of the NodeCSRApprover controller. ControllerName = "node_csr_autoapprover" nodeUser = "system:node" @@ -69,7 +69,7 @@ type reconciler struct { func Add(mgr manager.Manager) error { certClient, err := certificatesv1client.NewForConfig(mgr.GetConfig()) if err != nil { - return fmt.Errorf("failed to create certificate client: %v", err) + return fmt.Errorf("failed to create certificate client: %w", err) } rec := &reconciler{Client: mgr.GetClient(), certClient: certClient.CertificateSigningRequests()} @@ -77,7 +77,7 @@ func Add(mgr manager.Manager) error { cntrl, err := controller.New(ControllerName, mgr, controller.Options{Reconciler: rec}) if err != nil { - return fmt.Errorf("failed to construct controller: %v", err) + return fmt.Errorf("failed to construct controller: %w", err) } return cntrl.Watch(&source.Kind{Type: watchType}, &handler.EnqueueRequestForObject{}) @@ -120,7 +120,7 @@ func (r *reconciler) reconcile(ctx context.Context, request reconcile.Request) e // Get machine name for the appropriate node machine, found, err := r.getMachineForNode(ctx, nodeName) if err != nil { - return fmt.Errorf("failed to get machine for node '%s': %v", nodeName, err) + return fmt.Errorf("failed to get machine for node '%s': %w", nodeName, err) } if !found { return fmt.Errorf("no machine found for given node '%s'", nodeName) @@ -141,7 +141,7 @@ func (r *reconciler) reconcile(ctx context.Context, request reconcile.Request) e // Validate the certificate request if err := r.validateX509CSR(csr, certRequest, machine); err != nil { - return fmt.Errorf("error validating the x509 certificate request: %v", err) + return fmt.Errorf("error validating the x509 certificate request: %w", err) } // Approve CSR @@ -154,16 +154,16 @@ func (r *reconciler) reconcile(ctx context.Context, request reconcile.Request) e csr.Status.Conditions = append(csr.Status.Conditions, approvalCondition) if _, err := r.certClient.UpdateApproval(ctx, csr.Name, csr, metav1.UpdateOptions{}); err != nil { - return fmt.Errorf("failed to approve CSR %q: %v", csr.Name, err) + return fmt.Errorf("failed to approve CSR %q: %w", csr.Name, err) } klog.Infof("Successfully approved CSR %s", csr.ObjectMeta.Name) return nil } -// validateCSRObject valides the CSR object and returns name of the node that requested the certificate +// validateCSRObject valides the CSR object and returns name of the node that requested the certificate. func (r *reconciler) validateCSRObject(csr *certificatesv1.CertificateSigningRequest) (string, error) { - // Get and validate the node name + // Get and validate the node name. if !strings.HasPrefix(csr.Spec.Username, nodeUserPrefix) { return "", fmt.Errorf("username must have the '%s' prefix", nodeUserPrefix) } @@ -172,7 +172,7 @@ func (r *reconciler) validateCSRObject(csr *certificatesv1.CertificateSigningReq return "", fmt.Errorf("node name is empty") } - // Ensure system:nodes and system:authenticated are in groups + // Ensure system:nodes and system:authenticated are in groups. if len(csr.Spec.Groups) < 2 { return "", fmt.Errorf("there are less than 2 groups") } @@ -196,12 +196,12 @@ func (r *reconciler) validateCSRObject(csr *certificatesv1.CertificateSigningReq // validateX509CSR validates the certificate request by comparing CN with username, // and organization with groups. func (r *reconciler) validateX509CSR(csr *certificatesv1.CertificateSigningRequest, certReq *x509.CertificateRequest, machine v1alpha1.Machine) error { - // Validate Subject CommonName + // Validate Subject CommonName. if certReq.Subject.CommonName != csr.Spec.Username { return fmt.Errorf("commonName '%s' is different then CSR username '%s'", certReq.Subject.CommonName, csr.Spec.Username) } - // Validate Subject Organization + // Validate Subject Organization. if len(certReq.Subject.Organization) != 1 { return fmt.Errorf("expected only one organization but got %d instead", len(certReq.Subject.Organization)) } @@ -214,7 +214,7 @@ func (r *reconciler) validateX509CSR(csr *certificatesv1.CertificateSigningReque machineAddressSet.Insert(addr.Address) } - // Validate SAN DNS names + // Validate SAN DNS names. for _, dns := range certReq.DNSNames { if len(dns) == 0 { continue @@ -238,10 +238,10 @@ func (r *reconciler) validateX509CSR(csr *certificatesv1.CertificateSigningReque } func (r *reconciler) getMachineForNode(ctx context.Context, nodeName string) (v1alpha1.Machine, bool, error) { - // List all Machines in all namespaces + // List all Machines in all namespaces. machines := &v1alpha1.MachineList{} if err := r.Client.List(ctx, machines); err != nil { - return v1alpha1.Machine{}, false, fmt.Errorf("failed to list all machine objects: %v", err) + return v1alpha1.Machine{}, false, fmt.Errorf("failed to list all machine objects: %w", err) } for _, machine := range machines.Items { diff --git a/pkg/controller/util/machine_deployment.go b/pkg/controller/util/machine_deployment.go index 48074279c..e24fdfbbf 100644 --- a/pkg/controller/util/machine_deployment.go +++ b/pkg/controller/util/machine_deployment.go @@ -29,7 +29,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -42,7 +42,7 @@ import ( const ( DefaultMachineDeploymentUniqueLabelKey = "machine-template-hash" - // RevisionAnnotation is the revision annotation of a machine deployment's machine sets which records its rollout sequence + // RevisionAnnotation is the revision annotation of a machine deployment's machine sets which records its rollout sequence. RevisionAnnotation = "machinedeployment.clusters.k8s.io/revision" // RevisionHistoryAnnotation maintains the history of all old revisions that a machine set has served for a machine deployment. RevisionHistoryAnnotation = "machinedeployment.clusters.k8s.io/revision-history" @@ -129,7 +129,7 @@ func SetDeploymentRevision(deployment *v1alpha1.MachineDeployment, revision stri return updated } -// MaxRevision finds the highest revision in the machine sets +// MaxRevision finds the highest revision in the machine sets. func MaxRevision(allMSs []*v1alpha1.MachineSet) int64 { max := int64(0) for _, ms := range allMSs { @@ -192,7 +192,7 @@ func copyDeploymentAnnotationsToMachineSet(deployment *v1alpha1.MachineDeploymen return msAnnotationsChanged } -// GetDesiredReplicasAnnotation returns the number of desired replicas +// GetDesiredReplicasAnnotation returns the number of desired replicas. func GetDesiredReplicasAnnotation(ms *v1alpha1.MachineSet) (int32, bool) { return getIntFromAnnotation(ms, DesiredReplicasAnnotation) } @@ -292,7 +292,7 @@ func FindOneActiveOrLatest(newMS *v1alpha1.MachineSet, oldMSs []*v1alpha1.Machin } } -// SetReplicasAnnotations sets the desiredReplicas and maxReplicas into the annotations +// SetReplicasAnnotations sets the desiredReplicas and maxReplicas into the annotations. func SetReplicasAnnotations(ms *v1alpha1.MachineSet, desiredReplicas, maxReplicas int32) bool { updated := false if ms.Annotations == nil { @@ -311,7 +311,7 @@ func SetReplicasAnnotations(ms *v1alpha1.MachineSet, desiredReplicas, maxReplica return updated } -// AnnotationsNeedUpdate return true if ReplicasAnnotations need to be updated +// AnnotationsNeedUpdate return true if ReplicasAnnotations need to be updated. func ReplicasAnnotationsNeedUpdate(ms *v1alpha1.MachineSet, desiredReplicas, maxReplicas int32) bool { if ms.Annotations == nil { return true @@ -401,11 +401,11 @@ func getMachineSetFraction(ms v1alpha1.MachineSet, d v1alpha1.MachineDeployment) // We ignore machine-template-hash because: // 1. The hash result would be different upon machineTemplateSpec API changes // (e.g. the addition of a new field will cause the hash code to change) -// 2. The deployment template won't have hash labels +// 2. The deployment template won't have hash labels. func EqualIgnoreHash(template1, template2 *v1alpha1.MachineTemplateSpec) bool { t1Copy := template1.DeepCopy() t2Copy := template2.DeepCopy() - // Remove hash labels from template.Labels before comparing + // Remove hash labels from template.Labels before comparing. delete(t1Copy.Labels, DefaultMachineDeploymentUniqueLabelKey) delete(t2Copy.Labels, DefaultMachineDeploymentUniqueLabelKey) return apiequality.Semantic.DeepEqual(t1Copy, t2Copy) @@ -509,7 +509,7 @@ func DeploymentComplete(deployment *v1alpha1.MachineDeployment, newStatus *v1alp // NewMSNewReplicas calculates the number of replicas a deployment's new MS should have. // When one of the following is true, we're rolling out the deployment; otherwise, we're scaling it. // 1) The new MS is saturated: newMS's replicas == deployment's replicas -// 2) Max number of machines allowed is reached: deployment's replicas + maxSurge == all MSs' replicas +// 2) Max number of machines allowed is reached: deployment's replicas + maxSurge == all MSs' replicas. func NewMSNewReplicas(deployment *v1alpha1.MachineDeployment, allMSs []*v1alpha1.MachineSet, newMS *v1alpha1.MachineSet) (int32, error) { switch deployment.Spec.Strategy.Type { case common.RollingUpdateMachineDeploymentStrategyType: @@ -578,7 +578,7 @@ func IsSaturated(deployment *v1alpha1.MachineDeployment, ms *v1alpha1.MachineSet // 2 desired, max unavailable 25%, surge 1% - should scale new(+1), then old(-1), then new(+1), then old(-1) // 1 desired, max unavailable 25%, surge 1% - should scale new(+1), then old(-1) // 2 desired, max unavailable 0%, surge 1% - should scale new(+1), then old(-1), then new(+1), then old(-1) -// 1 desired, max unavailable 0%, surge 1% - should scale new(+1), then old(-1) +// 1 desired, max unavailable 0%, surge 1% - should scale new(+1), then old(-1). func ResolveFenceposts(maxSurge, maxUnavailable *intstrutil.IntOrString, desired int32) (int32, int32, error) { surge, err := intstrutil.GetValueFromIntOrPercent(maxSurge, int(desired), true) if err != nil { diff --git a/pkg/health/readiness.go b/pkg/health/readiness.go index 1f3cf9a47..d79a21c6c 100644 --- a/pkg/health/readiness.go +++ b/pkg/health/readiness.go @@ -32,7 +32,7 @@ func ApiserverReachable(client kubernetes.Interface) healthz.Checker { return func(req *http.Request) error { _, err := client.CoreV1().Nodes().List(req.Context(), metav1.ListOptions{}) if err != nil { - return fmt.Errorf("unable to list nodes check: %v", err) + return fmt.Errorf("unable to list nodes check: %w", err) } return nil @@ -43,7 +43,7 @@ func KubeconfigAvailable(kubeconfigProvider machinecontroller.KubeconfigProvider return func(req *http.Request) error { cm, err := kubeconfigProvider.GetKubeconfig(req.Context()) if err != nil { - return fmt.Errorf("unable to get kubeconfig: %v", err) + return fmt.Errorf("unable to get kubeconfig: %w", err) } if len(cm.Clusters) != 1 { diff --git a/pkg/ini/duration.go b/pkg/ini/duration.go index 7baba13aa..0f3af1f83 100644 --- a/pkg/ini/duration.go +++ b/pkg/ini/duration.go @@ -20,12 +20,12 @@ import ( "time" ) -// Duration is the encoding.TextUnmarshaler interface for time.Duration +// Duration is the encoding.TextUnmarshaler interface for time.Duration. type Duration struct { time.Duration } -// UnmarshalText is used to convert from text to Duration +// UnmarshalText is used to convert from text to Duration. func (d *Duration) UnmarshalText(text []byte) error { res, err := time.ParseDuration(string(text)) if err != nil { @@ -35,7 +35,7 @@ func (d *Duration) UnmarshalText(text []byte) error { return nil } -// MarshalText is used to convert from Duration to text +// MarshalText is used to convert from Duration to text. func (d *Duration) MarshalText() []byte { return []byte(d.Duration.String()) } diff --git a/pkg/ini/escape.go b/pkg/ini/escape.go index 84fd79a21..fa45b5854 100644 --- a/pkg/ini/escape.go +++ b/pkg/ini/escape.go @@ -20,13 +20,13 @@ import ( "strings" ) -// Allowed escaping characters by gopkg.in/gcfg.v1 - the lib kubernetes uses +// Allowed escaping characters by gopkg.in/gcfg.v1 - the lib kubernetes uses. var escaper = strings.NewReplacer( `\`, `\\`, `"`, `\"`, ) -// Escape escapes values in ini files correctly according to gopkg.in/gcfg.v1 - the lib kubernetes uses +// Escape escapes values in ini files correctly according to gopkg.in/gcfg.v1 - the lib kubernetes uses. func Escape(s string) string { return `"` + escaper.Replace(s) + `"` } diff --git a/pkg/ini/escape_test.go b/pkg/ini/escape_test.go index dd55b4913..725b6b9b8 100644 --- a/pkg/ini/escape_test.go +++ b/pkg/ini/escape_test.go @@ -40,7 +40,7 @@ type testData struct { Global globalSection } -// TestINIEscape will ensure that we hopefully cover every case +// TestINIEscape will ensure that we hopefully cover every case. func TestINIEscape(t *testing.T) { // We'll simply generate 1000 times a password with special chars, // Put it into a OpenStack cloud config, diff --git a/pkg/kubernetes/helper.go b/pkg/kubernetes/helper.go index b76a5de31..df6272c47 100644 --- a/pkg/kubernetes/helper.go +++ b/pkg/kubernetes/helper.go @@ -21,12 +21,12 @@ import ( "k8s.io/apimachinery/pkg/util/sets" ) -// HasFinalizer tells if a object has the given finalizer +// HasFinalizer tells if a object has the given finalizer. func HasFinalizer(o metav1.Object, name string) bool { return sets.NewString(o.GetFinalizers()...).Has(name) } -// RemoveFinalizer removes the given finalizer and returns the cleaned list +// RemoveFinalizer removes the given finalizer and returns the cleaned list. func RemoveFinalizer(finalizers []string, toRemove string) []string { set := sets.NewString(finalizers...) set.Delete(toRemove) diff --git a/pkg/node/eviction/eviction.go b/pkg/node/eviction/eviction.go index e6d1a2024..d22679b84 100644 --- a/pkg/node/eviction/eviction.go +++ b/pkg/node/eviction/eviction.go @@ -41,7 +41,7 @@ type NodeEviction struct { kubeClient kubernetes.Interface } -// New returns a new NodeEviction +// New returns a new NodeEviction. func New(ctx context.Context, nodeName string, client ctrlruntimeclient.Client, kubeClient kubernetes.Interface) *NodeEviction { return &NodeEviction{ nodeManager: nodemanager.New(ctx, client, nodeName), @@ -51,11 +51,11 @@ func New(ctx context.Context, nodeName string, client ctrlruntimeclient.Client, } } -// Run executes the eviction +// Run executes the eviction. func (ne *NodeEviction) Run() (bool, error) { node, err := ne.nodeManager.GetNode() if err != nil { - return false, fmt.Errorf("failed to get node from lister: %v", err) + return false, fmt.Errorf("failed to get node from lister: %w", err) } if _, exists := node.Annotations[evictiontypes.SkipEvictionAnnotationKey]; exists { klog.V(3).Infof("Skipping eviction for node %s as it has a %s annotation", ne.nodeName, evictiontypes.SkipEvictionAnnotationKey) @@ -64,13 +64,13 @@ func (ne *NodeEviction) Run() (bool, error) { klog.V(3).Infof("Starting to evict node %s", ne.nodeName) if err := ne.nodeManager.CordonNode(node); err != nil { - return false, fmt.Errorf("failed to cordon node %s: %v", ne.nodeName, err) + return false, fmt.Errorf("failed to cordon node %s: %w", ne.nodeName, err) } klog.V(6).Infof("Successfully cordoned node %s", ne.nodeName) podsToEvict, err := ne.getFilteredPods() if err != nil { - return false, fmt.Errorf("failed to get Pods to evict for node %s: %v", ne.nodeName, err) + return false, fmt.Errorf("failed to get Pods to evict for node %s: %w", ne.nodeName, err) } klog.V(6).Infof("Found %v pods to evict for node %s", len(podsToEvict), ne.nodeName) @@ -95,7 +95,7 @@ func (ne *NodeEviction) getFilteredPods() ([]corev1.Pod, error) { FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": ne.nodeName}).String(), }) if err != nil { - return nil, fmt.Errorf("failed to list pods: %v", err) + return nil, fmt.Errorf("failed to list pods: %w", err) } var filteredPods []corev1.Pod @@ -116,7 +116,6 @@ func (ne *NodeEviction) getFilteredPods() ([]corev1.Pod, error) { } func (ne *NodeEviction) evictPods(pods []corev1.Pod) []error { - errCh := make(chan error, len(pods)) retErrs := []error{} @@ -140,7 +139,7 @@ func (ne *NodeEviction) evictPods(pods []corev1.Pod) []error { // PDB prevents eviction, return and make the controller retry later return } else { - errCh <- fmt.Errorf("error evicting pod %s/%s on node %s: %v", p.Namespace, p.Name, ne.nodeName, err) + errCh <- fmt.Errorf("error evicting pod %s/%s on node %s: %w", p.Namespace, p.Name, ne.nodeName, err) return } } diff --git a/pkg/node/eviction/eviction_test.go b/pkg/node/eviction/eviction_test.go index 849f0ee0b..61ed90a5e 100644 --- a/pkg/node/eviction/eviction_test.go +++ b/pkg/node/eviction/eviction_test.go @@ -26,7 +26,7 @@ import ( ) // Unfortunately we can not directly test `EvictNode` as a List with a fieldSelector -// against a fake client returns nothing +// against a fake client returns nothing. func TestEvictPods(t *testing.T) { tests := []struct { Name string @@ -54,9 +54,7 @@ func TestEvictPods(t *testing.T) { literalPods = append(literalPods, *(pod.(*corev1.Pod))) } client := kubefake.NewSimpleClientset(test.Pods...) - t.Run(test.Name, func(t *testing.T) { - ne := &NodeEviction{kubeClient: client, nodeName: "node1"} if errs := ne.evictPods(literalPods); len(errs) > 0 { t.Fatalf("Got unexpected errors=%v when running evictPods", errs) diff --git a/pkg/node/nodemanager/node_manager.go b/pkg/node/nodemanager/node_manager.go index 342a2bf57..1c69b5d90 100644 --- a/pkg/node/nodemanager/node_manager.go +++ b/pkg/node/nodemanager/node_manager.go @@ -45,7 +45,7 @@ func New(ctx context.Context, client ctrlruntimeclient.Client, nodeName string) func (nm *NodeManager) GetNode() (*corev1.Node, error) { node := &corev1.Node{} if err := nm.client.Get(nm.ctx, types.NamespacedName{Name: nm.nodeName}, node); err != nil { - return nil, fmt.Errorf("failed to get node from lister: %v", err) + return nil, fmt.Errorf("failed to get node from lister: %w", err) } return node, nil } diff --git a/pkg/node/poddeletion/pod_deletion.go b/pkg/node/poddeletion/pod_deletion.go index 9781b8864..6a8ecad62 100644 --- a/pkg/node/poddeletion/pod_deletion.go +++ b/pkg/node/poddeletion/pod_deletion.go @@ -42,7 +42,7 @@ type NodeVolumeAttachmentsCleanup struct { kubeClient kubernetes.Interface } -// New returns a new NodeVolumeAttachmentsCleanup +// New returns a new NodeVolumeAttachmentsCleanup. func New(ctx context.Context, nodeName string, client ctrlruntimeclient.Client, kubeClient kubernetes.Interface) *NodeVolumeAttachmentsCleanup { return &NodeVolumeAttachmentsCleanup{ nodeManager: nodemanager.New(ctx, client, nodeName), @@ -52,33 +52,33 @@ func New(ctx context.Context, nodeName string, client ctrlruntimeclient.Client, } } -// Run executes the pod deletion +// Run executes the pod deletion. func (vc *NodeVolumeAttachmentsCleanup) Run() (bool, bool, error) { node, err := vc.nodeManager.GetNode() if err != nil { - return false, false, fmt.Errorf("failed to get node from lister: %v", err) + return false, false, fmt.Errorf("failed to get node from lister: %w", err) } klog.V(3).Infof("Starting to cleanup node %s", vc.nodeName) - // if there are no more volumeAttachments related to the node, then it can be deleted + // if there are no more volumeAttachments related to the node, then it can be deleted. volumeAttachmentsDeleted, err := vc.nodeCanBeDeleted() if err != nil { - return false, false, fmt.Errorf("failed to check volumeAttachments deletion: %v", err) + return false, false, fmt.Errorf("failed to check volumeAttachments deletion: %w", err) } if volumeAttachmentsDeleted { return false, true, nil } - // cordon the node to be sure that the deleted pods are re-scheduled in the same node + // cordon the node to be sure that the deleted pods are re-scheduled in the same node. if err := vc.nodeManager.CordonNode(node); err != nil { - return false, false, fmt.Errorf("failed to cordon node %s: %v", vc.nodeName, err) + return false, false, fmt.Errorf("failed to cordon node %s: %w", vc.nodeName, err) } klog.V(6).Infof("Successfully cordoned node %s", vc.nodeName) - // get all the pods that needs to be deleted (i.e. those mounting volumes attached to the node that is going to be deleted) + // get all the pods that needs to be deleted (i.e. those mounting volumes attached to the node that is going to be deleted). podsToDelete, errors := vc.getFilteredPods() if len(errors) > 0 { - return false, false, fmt.Errorf("failed to get Pods to delete for node %s, errors encountered: %v", vc.nodeName, err) + return false, false, fmt.Errorf("failed to get Pods to delete for node %s, errors encountered: %w", vc.nodeName, err) } klog.V(6).Infof("Found %v pods to delete for node %s", len(podsToDelete), vc.nodeName) @@ -86,7 +86,7 @@ func (vc *NodeVolumeAttachmentsCleanup) Run() (bool, bool, error) { return false, false, nil } - // delete the previously filtered pods, then tells the controller to retry later + // delete the previously filtered pods, then tells the controller to retry later. if errs := vc.deletePods(podsToDelete); len(errs) > 0 { return false, false, fmt.Errorf("failed to delete pods, errors encountered: %v", errs) } @@ -101,13 +101,13 @@ func (vc *NodeVolumeAttachmentsCleanup) getFilteredPods() ([]corev1.Pod, []error volumeAttachments, err := vc.kubeClient.StorageV1().VolumeAttachments().List(vc.ctx, metav1.ListOptions{}) if err != nil { - retErrs = append(retErrs, fmt.Errorf("failed to list pods: %v", err)) + retErrs = append(retErrs, fmt.Errorf("failed to list pods: %w", err)) return nil, retErrs } persistentVolumeClaims, err := vc.kubeClient.CoreV1().PersistentVolumeClaims(metav1.NamespaceAll).List(vc.ctx, metav1.ListOptions{}) if err != nil { - retErrs = append(retErrs, fmt.Errorf("failed to list persistent volumes: %v", err)) + retErrs = append(retErrs, fmt.Errorf("failed to list persistent volumes: %w", err)) return nil, retErrs } @@ -125,7 +125,7 @@ func (vc *NodeVolumeAttachmentsCleanup) getFilteredPods() ([]corev1.Pod, []error case kerrors.IsTooManyRequests(err): return case err != nil: - errCh <- fmt.Errorf("failed to list pod: %v", err) + errCh <- fmt.Errorf("failed to list pod: %w", err) default: for _, pod := range pods.Items { if doesPodClaimVolume(pod, pvc.Name) && pod.Spec.NodeName == vc.nodeName { @@ -150,11 +150,11 @@ func (vc *NodeVolumeAttachmentsCleanup) getFilteredPods() ([]corev1.Pod, []error return filteredPods, nil } -// nodeCanBeDeleted checks if all the volumeAttachments related to the node have already been collected by the external CSI driver +// nodeCanBeDeleted checks if all the volumeAttachments related to the node have already been collected by the external CSI driver. func (vc *NodeVolumeAttachmentsCleanup) nodeCanBeDeleted() (bool, error) { volumeAttachments, err := vc.kubeClient.StorageV1().VolumeAttachments().List(vc.ctx, metav1.ListOptions{}) if err != nil { - return false, fmt.Errorf("error while listing volumeAttachments: %v", err) + return false, fmt.Errorf("error while listing volumeAttachments: %w", err) } for _, va := range volumeAttachments.Items { if va.Spec.NodeName == vc.nodeName { @@ -166,7 +166,6 @@ func (vc *NodeVolumeAttachmentsCleanup) nodeCanBeDeleted() (bool, error) { } func (vc *NodeVolumeAttachmentsCleanup) deletePods(pods []corev1.Pod) []error { - errCh := make(chan error, len(pods)) retErrs := []error{} @@ -187,10 +186,10 @@ func (vc *NodeVolumeAttachmentsCleanup) deletePods(pods []corev1.Pod) []error { klog.V(6).Infof("Successfully deleted pod %s/%s on node %s", p.Namespace, p.Name, vc.nodeName) return } else if kerrors.IsTooManyRequests(err) { - // PDB prevents pod deletion, return and make the controller retry later + // PDB prevents pod deletion, return and make the controller retry later. return } else { - errCh <- fmt.Errorf("error deleting pod %s/%s on node %s: %v", p.Namespace, p.Name, vc.nodeName, err) + errCh <- fmt.Errorf("error deleting pod %s/%s on node %s: %w", p.Namespace, p.Name, vc.nodeName, err) return } } @@ -206,7 +205,7 @@ func (vc *NodeVolumeAttachmentsCleanup) deletePods(pods []corev1.Pod) []error { return retErrs } -// doesPodClaimVolume checks if the volume is mounted by the pod +// doesPodClaimVolume checks if the volume is mounted by the pod. func doesPodClaimVolume(pod corev1.Pod, pvcName string) bool { for _, volumeMount := range pod.Spec.Volumes { if volumeMount.PersistentVolumeClaim != nil && volumeMount.PersistentVolumeClaim.ClaimName == pvcName { diff --git a/pkg/providerconfig/types.go b/pkg/providerconfig/types.go index b4e168adf..27e73c6a5 100644 --- a/pkg/providerconfig/types.go +++ b/pkg/providerconfig/types.go @@ -81,7 +81,7 @@ func (cvr *ConfigVarResolver) GetConfigVarStringValue(configVar providerconfigty secret := &corev1.Secret{} name := types.NamespacedName{Namespace: configVar.SecretKeyRef.Namespace, Name: configVar.SecretKeyRef.Name} if err := cvr.client.Get(cvr.ctx, name, secret); err != nil { - return "", fmt.Errorf("error retrieving secret '%s' from namespace '%s': '%v'", configVar.SecretKeyRef.Name, configVar.SecretKeyRef.Namespace, err) + return "", fmt.Errorf("error retrieving secret '%s' from namespace '%s': '%w'", configVar.SecretKeyRef.Name, configVar.SecretKeyRef.Namespace, err) } if val, ok := secret.Data[configVar.SecretKeyRef.Key]; ok { return string(val), nil @@ -94,7 +94,7 @@ func (cvr *ConfigVarResolver) GetConfigVarStringValue(configVar providerconfigty configMap := &corev1.ConfigMap{} name := types.NamespacedName{Namespace: configVar.ConfigMapKeyRef.Namespace, Name: configVar.ConfigMapKeyRef.Name} if err := cvr.client.Get(cvr.ctx, name, configMap); err != nil { - return "", fmt.Errorf("error retrieving configmap '%s' from namespace '%s': '%v'", configVar.ConfigMapKeyRef.Name, configVar.ConfigMapKeyRef.Namespace, err) + return "", fmt.Errorf("error retrieving configmap '%s' from namespace '%s': '%w'", configVar.ConfigMapKeyRef.Name, configVar.ConfigMapKeyRef.Namespace, err) } if val, ok := configMap.Data[configVar.ConfigMapKeyRef.Key]; ok { return val, nil @@ -106,7 +106,7 @@ func (cvr *ConfigVarResolver) GetConfigVarStringValue(configVar providerconfigty } // GetConfigVarStringValueOrEnv tries to get the value from ConfigVarString, when it fails, it falls back to -// getting the value from an environment variable specified by envVarName parameter +// getting the value from an environment variable specified by envVarName parameter. func (cvr *ConfigVarResolver) GetConfigVarStringValueOrEnv(configVar providerconfigtypes.ConfigVarString, envVarName string) (string, error) { cfgVar, err := cvr.GetConfigVarStringValue(configVar) if err == nil && len(cfgVar) > 0 { @@ -118,14 +118,14 @@ func (cvr *ConfigVarResolver) GetConfigVarStringValueOrEnv(configVar providercon } // GetConfigVarBoolValue returns a boolean from a ConfigVarBool. If there is no valid source for the boolean, -// the second bool returned will be false (to be able to differentiate between "false" and "unset") +// the second bool returned will be false (to be able to differentiate between "false" and "unset"). func (cvr *ConfigVarResolver) GetConfigVarBoolValue(configVar providerconfigtypes.ConfigVarBool) (bool, bool, error) { // We need all three of these to fetch and use a secret if configVar.SecretKeyRef.Name != "" && configVar.SecretKeyRef.Namespace != "" && configVar.SecretKeyRef.Key != "" { secret := &corev1.Secret{} name := types.NamespacedName{Namespace: configVar.SecretKeyRef.Namespace, Name: configVar.SecretKeyRef.Name} if err := cvr.client.Get(cvr.ctx, name, secret); err != nil { - return false, false, fmt.Errorf("error retrieving secret '%s' from namespace '%s': '%v'", configVar.SecretKeyRef.Name, configVar.SecretKeyRef.Namespace, err) + return false, false, fmt.Errorf("error retrieving secret '%s' from namespace '%s': '%w'", configVar.SecretKeyRef.Name, configVar.SecretKeyRef.Namespace, err) } if val, ok := secret.Data[configVar.SecretKeyRef.Key]; ok { boolVal, err := strconv.ParseBool(string(val)) @@ -139,7 +139,7 @@ func (cvr *ConfigVarResolver) GetConfigVarBoolValue(configVar providerconfigtype configMap := &corev1.ConfigMap{} name := types.NamespacedName{Namespace: configVar.ConfigMapKeyRef.Namespace, Name: configVar.ConfigMapKeyRef.Name} if err := cvr.client.Get(cvr.ctx, name, configMap); err != nil { - return false, false, fmt.Errorf("error retrieving configmap '%s' from namespace '%s': '%v'", configVar.ConfigMapKeyRef.Name, configVar.ConfigMapKeyRef.Namespace, err) + return false, false, fmt.Errorf("error retrieving configmap '%s' from namespace '%s': '%w'", configVar.ConfigMapKeyRef.Name, configVar.ConfigMapKeyRef.Namespace, err) } if val, ok := configMap.Data[configVar.ConfigMapKeyRef.Key]; ok { boolVal, err := strconv.ParseBool(val) @@ -185,7 +185,6 @@ func DefaultOperatingSystemSpec( cloudProvider providerconfigtypes.CloudProvider, operatingSystemSpec runtime.RawExtension, ) (runtime.RawExtension, error) { - switch osys { case providerconfigtypes.OperatingSystemAmazonLinux2: return amzn2.DefaultConfig(operatingSystemSpec), nil diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 80b302a9a..8584746d6 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -102,12 +102,12 @@ var ( } ) -// DNSConfig contains a machine's DNS configuration +// DNSConfig contains a machine's DNS configuration. type DNSConfig struct { Servers []string `json:"servers"` } -// NetworkConfig contains a machine's static network configuration +// NetworkConfig contains a machine's static network configuration. type NetworkConfig struct { CIDR string `json:"cidr"` Gateway string `json:"gateway"` @@ -149,7 +149,7 @@ type Config struct { } // GlobalObjectKeySelector is needed as we can not use v1.SecretKeySelector -// because it is not cross namespace +// because it is not cross namespace. type GlobalObjectKeySelector struct { corev1.ObjectReference `json:",inline"` Key string `json:"key,omitempty"` @@ -166,13 +166,13 @@ type ConfigVarString struct { // This type only exists to have the same fields as ConfigVarString but // not its funcs, so it can be used as target for json.Unmarshal without -// causing a recursion +// causing a recursion. type configVarStringWithoutUnmarshaller ConfigVarString // MarshalJSON converts a configVarString to its JSON form, omitting empty strings. // This is done to not have the json object cluttered with empty strings // This will eventually hopefully be resolved within golang itself -// https://github.com/golang/go/issues/11939 +// https://github.com/golang/go/issues/11939. func (configVarString ConfigVarString) MarshalJSON() ([]byte, error) { var secretKeyRefEmpty, configMapKeyRefEmpty bool if configVarString.SecretKeyRef.ObjectReference.Namespace == "" && @@ -313,7 +313,7 @@ func (configVarBool *ConfigVarBool) UnmarshalJSON(b []byte) error { if !bytes.HasPrefix(b, []byte("{")) { var val *bool if err := json.Unmarshal(b, &val); err != nil { - return fmt.Errorf("Error parsing value: '%v'", err) + return fmt.Errorf("Error parsing value: '%w'", err) } configVarBool.Value = val diff --git a/pkg/providerconfig/types/types_test.go b/pkg/providerconfig/types/types_test.go index 1ee107966..7b0d8601a 100644 --- a/pkg/providerconfig/types/types_test.go +++ b/pkg/providerconfig/types/types_test.go @@ -94,7 +94,6 @@ func TestConfigVarBoolUnmarshalling(t *testing.T) { } func TestConfigVarStringMarshalling(t *testing.T) { - testCases := []struct { cvs ConfigVarString expected string @@ -121,7 +120,6 @@ func TestConfigVarStringMarshalling(t *testing.T) { } func TestConfigVarBoolMarshalling(t *testing.T) { - testCases := []struct { cvb ConfigVarBool expected string @@ -160,7 +158,6 @@ func TestConfigVarBoolMarshalling(t *testing.T) { } func TestConfigVarStringMarshallingAndUnmarshalling(t *testing.T) { - testCases := []ConfigVarString{ {Value: "val"}, {SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, @@ -195,7 +192,6 @@ func TestConfigVarStringMarshallingAndUnmarshalling(t *testing.T) { } func TestConfigVarBoolMarshallingAndUnmarshalling(t *testing.T) { - testCases := []ConfigVarBool{ {}, {Value: pointer.Bool(false)}, diff --git a/pkg/providerconfig/types_test.go b/pkg/providerconfig/types_test.go index 896593020..40930062b 100644 --- a/pkg/providerconfig/types_test.go +++ b/pkg/providerconfig/types_test.go @@ -39,7 +39,6 @@ func TestDefaultOperatingSystemSpec(t *testing.T) { if operatingSystemSpec.Raw == nil { t.Errorf("expected not nil") } - }) } } diff --git a/pkg/rhsm/satellite_subscription_manager.go b/pkg/rhsm/satellite_subscription_manager.go index 77818e0a8..b32deac10 100644 --- a/pkg/rhsm/satellite_subscription_manager.go +++ b/pkg/rhsm/satellite_subscription_manager.go @@ -28,7 +28,7 @@ import ( "k8s.io/klog" ) -// SatelliteSubscriptionManager manages the communications between machine-controller and redhat satellite server +// SatelliteSubscriptionManager manages the communications between machine-controller and redhat satellite server. type SatelliteSubscriptionManager interface { DeleteSatelliteHost(machineName, username, password, serverURL string) error } @@ -93,14 +93,14 @@ func (s *DefaultSatelliteSubscriptionManager) executeDeleteRequest(machineName, deleteHostRequest, err := http.NewRequest(http.MethodDelete, requestURL.String(), nil) if err != nil { - return fmt.Errorf("failed to create a delete host request: %v", err) + return fmt.Errorf("failed to create a delete host request: %w", err) } deleteHostRequest.SetBasicAuth(username, password) response, err := s.client.Do(deleteHostRequest) if err != nil { - return fmt.Errorf("failed executing delete host request: %v", err) + return fmt.Errorf("failed executing delete host request: %w", err) } defer response.Body.Close() diff --git a/pkg/rhsm/subscription_manager.go b/pkg/rhsm/subscription_manager.go index c9da007e3..93a367535 100644 --- a/pkg/rhsm/subscription_manager.go +++ b/pkg/rhsm/subscription_manager.go @@ -103,7 +103,7 @@ func (d *defaultRedHatSubscriptionManager) UnregisterInstance(offlineToken, mach for retries < maxRetries { machineUUID, err := d.findSystemsProfile(ctx, offlineToken, machineName) if err != nil { - return fmt.Errorf("failed to find system profile: %v", err) + return fmt.Errorf("failed to find system profile: %w", err) } if machineUUID == "" { @@ -130,7 +130,7 @@ func (d *defaultRedHatSubscriptionManager) findSystemsProfile(ctx context.Contex for { systemsInfo, err := d.executeFindSystemsRequest(ctx, offlineToken, offset) if err != nil { - return "", fmt.Errorf("failed to retrieve systems: %v", err) + return "", fmt.Errorf("failed to retrieve systems: %w", err) } for _, system := range systemsInfo.Body { @@ -154,19 +154,18 @@ func (d *defaultRedHatSubscriptionManager) deleteSubscription(ctx context.Contex client := newOAuthClientWithRefreshToken(offlineToken, d.authURL) req, err := http.NewRequest("DELETE", fmt.Sprintf("%s/%s", d.apiURL, uuid), nil) if err != nil { - return fmt.Errorf("failed to create delete system request: %v", err) + return fmt.Errorf("failed to create delete system request: %w", err) } - req.WithContext(ctx) - res, err := client.Do(req) + res, err := client.Do(req.WithContext(ctx)) if err != nil { - return fmt.Errorf("failed to delete system profile: %v", err) + return fmt.Errorf("failed to delete system profile: %w", err) } defer res.Body.Close() data, err := ioutil.ReadAll(res.Body) if err != nil { - return fmt.Errorf("failed while reading response: %v", err) + return fmt.Errorf("failed while reading response: %w", err) } if res.StatusCode != http.StatusNoContent { @@ -184,19 +183,17 @@ func (d *defaultRedHatSubscriptionManager) executeFindSystemsRequest(ctx context client := newOAuthClientWithRefreshToken(offlineToken, d.authURL) req, err := http.NewRequest("GET", fmt.Sprintf(d.apiURL+"?limit=%v&offset=%v", d.requestsLimiter, offset), nil) if err != nil { - return nil, fmt.Errorf("failed to create fetch systems request: %v", err) + return nil, fmt.Errorf("failed to create fetch systems request: %w", err) } - req.WithContext(ctx) - - res, err := client.Do(req) + res, err := client.Do(req.WithContext(ctx)) if err != nil { - return nil, fmt.Errorf("failed executing fetch systems request: %v", err) + return nil, fmt.Errorf("failed executing fetch systems request: %w", err) } defer res.Body.Close() data, err := ioutil.ReadAll(res.Body) if err != nil { - return nil, fmt.Errorf("failed while reading response: %v", err) + return nil, fmt.Errorf("failed while reading response: %w", err) } if res.StatusCode != http.StatusOK { @@ -208,7 +205,7 @@ func (d *defaultRedHatSubscriptionManager) executeFindSystemsRequest(ctx context var fetchedSystems = &systemsResponse{} if err := json.Unmarshal(data, fetchedSystems); err != nil { - return nil, fmt.Errorf("failed while unmarshalling data: %v", err) + return nil, fmt.Errorf("failed while unmarshalling data: %w", err) } return fetchedSystems, nil diff --git a/pkg/userdata/convert/ignition-converter.go b/pkg/userdata/convert/ignition-converter.go index da8f5ef75..bb9d4c865 100644 --- a/pkg/userdata/convert/ignition-converter.go +++ b/pkg/userdata/convert/ignition-converter.go @@ -57,7 +57,7 @@ func ToIgnition(s string) (string, error) { out, err := json.Marshal(ignCfg) if err != nil { - return "", fmt.Errorf("failed to marshal ignition config: %v", err) + return "", fmt.Errorf("failed to marshal ignition config: %w", err) } return string(out), nil diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index 6047db7e9..9d5819536 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -40,7 +40,7 @@ type Provider struct{} func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) if err != nil { - return "", fmt.Errorf("failed to get provider config: %v", err) + return "", fmt.Errorf("failed to get provider config: %w", err) } if pconfig.OverwriteCloudConfig != nil { @@ -49,22 +49,22 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { flatcarConfig, err := LoadConfig(pconfig.OperatingSystemSpec) if err != nil { - return "", fmt.Errorf("failed to get flatcar config from provider config: %v", err) + return "", fmt.Errorf("failed to get flatcar config from provider config: %w", err) } userDataTemplate, err := getUserDataTemplate(flatcarConfig.ProvisioningUtility) if err != nil { - return "", fmt.Errorf("failed to get an appropriate user-data template: %v", err) + return "", fmt.Errorf("failed to get an appropriate user-data template: %w", err) } tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(userDataTemplate) if err != nil { - return "", fmt.Errorf("failed to parse user-data template: %v", err) + return "", fmt.Errorf("failed to parse user-data template: %w", err) } kubeletVersion, err := semver.NewVersion(req.MachineSpec.Versions.Kubelet) if err != nil { - return "", fmt.Errorf("invalid kubelet version: %v", err) + return "", fmt.Errorf("invalid kubelet version: %w", err) } kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) @@ -74,7 +74,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { kubernetesCACert, err := userdatahelper.GetCACert(req.Kubeconfig) if err != nil { - return "", fmt.Errorf("error extracting cacert: %v", err) + return "", fmt.Errorf("error extracting cacert: %w", err) } if flatcarConfig.DisableAutoUpdate { @@ -124,12 +124,12 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { b := &bytes.Buffer{} err = tmpl.Execute(b, data) if err != nil { - return "", fmt.Errorf("failed to execute user-data template: %v", err) + return "", fmt.Errorf("failed to execute user-data template: %w", err) } out, err := userdatahelper.CleanupTemplateOutput(b.String()) if err != nil { - return "", fmt.Errorf("failed to cleanup user-data template: %v", err) + return "", fmt.Errorf("failed to cleanup user-data template: %w", err) } if flatcarConfig.ProvisioningUtility == CloudInit { @@ -501,7 +501,7 @@ storage: runtime-endpoint: unix:///run/containerd/containerd.sock ` -// Coreos cloud-config template +// Coreos cloud-config template. const userDataCloudInitTemplate = `#cloud-config users: diff --git a/pkg/userdata/helper/download_binaries_script.go b/pkg/userdata/helper/download_binaries_script.go index 358ceb3e1..6ff2df3a1 100644 --- a/pkg/userdata/helper/download_binaries_script.go +++ b/pkg/userdata/helper/download_binaries_script.go @@ -160,7 +160,7 @@ fi func SafeDownloadBinariesScript(kubeVersion string) (string, error) { tmpl, err := template.New("download-binaries").Funcs(TxtFuncMap()).Parse(safeDownloadBinariesTpl) if err != nil { - return "", fmt.Errorf("failed to parse download-binaries template: %v", err) + return "", fmt.Errorf("failed to parse download-binaries template: %w", err) } const ( @@ -186,7 +186,7 @@ func SafeDownloadBinariesScript(kubeVersion string) (string, error) { b := &bytes.Buffer{} err = tmpl.Execute(b, data) if err != nil { - return "", fmt.Errorf("failed to execute download-binaries template: %v", err) + return "", fmt.Errorf("failed to execute download-binaries template: %w", err) } return b.String(), nil @@ -197,7 +197,7 @@ func SafeDownloadBinariesScript(kubeVersion string) (string, error) { func DownloadBinariesScript(kubeletVersion string, downloadKubelet bool) (string, error) { tmpl, err := template.New("download-binaries").Funcs(TxtFuncMap()).Parse(downloadBinariesTpl) if err != nil { - return "", fmt.Errorf("failed to parse download-binaries template: %v", err) + return "", fmt.Errorf("failed to parse download-binaries template: %w", err) } data := struct { @@ -210,7 +210,7 @@ func DownloadBinariesScript(kubeletVersion string, downloadKubelet bool) (string b := &bytes.Buffer{} err = tmpl.Execute(b, data) if err != nil { - return "", fmt.Errorf("failed to execute download-binaries template: %v", err) + return "", fmt.Errorf("failed to execute download-binaries template: %w", err) } return b.String(), nil diff --git a/pkg/userdata/helper/helper.go b/pkg/userdata/helper/helper.go index 32069503c..9ee7ab926 100644 --- a/pkg/userdata/helper/helper.go +++ b/pkg/userdata/helper/helper.go @@ -38,16 +38,14 @@ func GetServerAddressFromKubeconfig(kubeconfig *clientcmdapi.Config) (string, er for _, clusterConfig := range kubeconfig.Clusters { return strings.Replace(clusterConfig.Server, "https://", "", -1), nil } - return "", fmt.Errorf("no server address found") - } func GetCACert(kubeconfig *clientcmdapi.Config) (string, error) { if len(kubeconfig.Clusters) != 1 { return "", fmt.Errorf("kubeconfig does not contain exactly one cluster, can not extract server address") } - // Clusters is a map so we have to use range here + // Clusters is a map so we have to use range here. for _, clusterConfig := range kubeconfig.Clusters { return string(clusterConfig.CertificateAuthorityData), nil } @@ -55,18 +53,18 @@ func GetCACert(kubeconfig *clientcmdapi.Config) (string, error) { return "", fmt.Errorf("no CACert found") } -// StringifyKubeconfig marshals a kubeconfig to its text form +// StringifyKubeconfig marshals a kubeconfig to its text form. func StringifyKubeconfig(kubeconfig *clientcmdapi.Config) (string, error) { kubeconfigBytes, err := clientcmd.Write(*kubeconfig) if err != nil { - return "", fmt.Errorf("error writing kubeconfig: %v", err) + return "", fmt.Errorf("error writing kubeconfig: %w", err) } return string(kubeconfigBytes), nil } // LoadKernelModules returns a script which is responsible for loading all required kernel modules -// The nf_conntrack_ipv4 module get removed in newer kernel versions +// The nf_conntrack_ipv4 module get removed in newer kernel versions. func LoadKernelModulesScript() string { return `#!/usr/bin/env bash set -euo pipefail @@ -85,7 +83,7 @@ fi } // KernelSettings returns the list of kernel settings required for a kubernetes worker node -// inotify changes according to https://github.com/kubernetes/kubernetes/issues/10421 - better than letting the kubelet die +// inotify changes according to https://github.com/kubernetes/kubernetes/issues/10421 - better than letting the kubelet die. func KernelSettings() string { return `net.bridge.bridge-nf-call-ip6tables = 1 net.bridge.bridge-nf-call-iptables = 1 @@ -98,7 +96,7 @@ fs.inotify.max_user_instances = 8192 ` } -// JournalDConfig returns the journal config preferable on every node +// JournalDConfig returns the journal config preferable on every node. func JournalDConfig() string { // JournaldMaxUse defines the maximum space that journalD logs can occupy. // https://www.freedesktop.org/software/systemd/man/journald.conf.html#SystemMaxUse= diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index 5bc264998..759ea4d46 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -107,7 +107,7 @@ WantedBy=multi-user.target` const cpFlags = `--cloud-provider=%s \ --cloud-config=/etc/kubernetes/cloud-config` -// List of allowed TLS cipher suites for kubelet +// List of allowed TLS cipher suites for kubelet. var kubeletTLSCipherSuites = []string{ // TLS 1.3 cipher suites "TLS_AES_128_GCM_SHA256", @@ -122,7 +122,7 @@ var kubeletTLSCipherSuites = []string{ "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", } -// CloudProviderFlags returns --cloud-provider and --cloud-config flags +// CloudProviderFlags returns --cloud-provider and --cloud-config flags. func CloudProviderFlags(cpName string, external bool) (string, error) { if cpName == "" && !external { return "", nil @@ -134,11 +134,11 @@ func CloudProviderFlags(cpName string, external bool) (string, error) { return fmt.Sprintf(cpFlags, cpName), nil } -// KubeletSystemdUnit returns the systemd unit for the kubelet +// KubeletSystemdUnit returns the systemd unit for the kubelet. func KubeletSystemdUnit(containerRuntime, kubeletVersion, cloudProvider, hostname string, dnsIPs []net.IP, external bool, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string, disableSwap bool) (string, error) { tmpl, err := template.New("kubelet-systemd-unit").Funcs(TxtFuncMap()).Parse(kubeletSystemdUnitTpl) if err != nil { - return "", fmt.Errorf("failed to parse kubelet-systemd-unit template: %v", err) + return "", fmt.Errorf("failed to parse kubelet-systemd-unit template: %w", err) } data := struct { @@ -173,7 +173,7 @@ func KubeletSystemdUnit(containerRuntime, kubeletVersion, cloudProvider, hostnam return buf.String(), nil } -// kubeletConfiguration returns marshaled kubelet.config.k8s.io/v1beta1 KubeletConfiguration +// kubeletConfiguration returns marshaled kubelet.config.k8s.io/v1beta1 KubeletConfiguration. func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGates map[string]bool, kubeletConfigs map[string]string, containerRuntime string) (string, error) { clusterDNSstr := make([]string, 0, len(clusterDNS)) for _, ip := range clusterDNS { @@ -267,11 +267,11 @@ func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGate return string(buf), err } -// KubeletFlags returns the kubelet flags +// KubeletFlags returns the kubelet flags. func KubeletFlags(version, cloudProvider, hostname string, dnsIPs []net.IP, external bool, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string) (string, error) { tmpl, err := template.New("kubelet-flags").Funcs(TxtFuncMap()).Parse(kubeletFlagsTpl) if err != nil { - return "", fmt.Errorf("failed to parse kubelet-flags template: %v", err) + return "", fmt.Errorf("failed to parse kubelet-flags template: %w", err) } initialTaintsArgs := []string{} @@ -333,13 +333,13 @@ func KubeletFlags(version, cloudProvider, hostname string, dnsIPs []net.IP, exte var buf strings.Builder if err = tmpl.Execute(&buf, data); err != nil { - return "", fmt.Errorf("failed to execute kubelet-flags template: %v", err) + return "", fmt.Errorf("failed to execute kubelet-flags template: %w", err) } return buf.String(), nil } -// KubeletHealthCheckSystemdUnit kubelet health checking systemd unit +// KubeletHealthCheckSystemdUnit kubelet health checking systemd unit. func KubeletHealthCheckSystemdUnit() string { return `[Unit] Requires=kubelet.service @@ -353,11 +353,11 @@ WantedBy=multi-user.target ` } -// ContainerRuntimeHealthCheckSystemdUnit container-runtime health checking systemd unit +// ContainerRuntimeHealthCheckSystemdUnit container-runtime health checking systemd unit. func ContainerRuntimeHealthCheckSystemdUnit(containerRuntime string) (string, error) { tmpl, err := template.New("container-runtime-healthcheck-systemd-unit").Funcs(TxtFuncMap()).Parse(containerRuntimeHealthCheckSystemdUnitTpl) if err != nil { - return "", fmt.Errorf("failed to parse container-runtime-healthcheck-systemd-unit template: %v", err) + return "", fmt.Errorf("failed to parse container-runtime-healthcheck-systemd-unit template: %w", err) } data := struct { diff --git a/pkg/userdata/helper/template_functions.go b/pkg/userdata/helper/template_functions.go index 62ecbade8..faeb1ea00 100644 --- a/pkg/userdata/helper/template_functions.go +++ b/pkg/userdata/helper/template_functions.go @@ -23,7 +23,7 @@ import ( "github.com/Masterminds/sprig/v3" ) -// TxtFuncMap returns an aggregated template function map. Currently (custom functions + sprig) +// TxtFuncMap returns an aggregated template function map. Currently (custom functions + sprig). func TxtFuncMap() template.FuncMap { funcMap := sprig.TxtFuncMap() diff --git a/pkg/userdata/manager/plugin.go b/pkg/userdata/manager/plugin.go index 0e87502e2..69fdc7199 100644 --- a/pkg/userdata/manager/plugin.go +++ b/pkg/userdata/manager/plugin.go @@ -76,7 +76,7 @@ func (p *Plugin) UserData(req plugin.UserDataRequest) (string, error) { // Execute command. out, err := cmd.CombinedOutput() if err != nil { - return "", fmt.Errorf("failed to execute command %q: output: %q error: %q", p.command, string(out), err) + return "", fmt.Errorf("failed to execute command %q: output: %q error: %w", p.command, string(out), err) } var resp plugin.UserDataResponse err = json.Unmarshal(out, &resp) @@ -126,7 +126,7 @@ func (p *Plugin) findPlugin(name string) error { if os.IsNotExist(err) { continue } - return fmt.Errorf("error when looking for %q: %v", command, err) + return fmt.Errorf("error when looking for %q: %w", command, err) } if fi.IsDir() || (fi.Mode()&0111 == 0) { klog.Infof("found '%s', but is no executable", command) diff --git a/pkg/userdata/sles/provider.go b/pkg/userdata/sles/provider.go index c58fb4da8..44a5d839b 100644 --- a/pkg/userdata/sles/provider.go +++ b/pkg/userdata/sles/provider.go @@ -38,20 +38,19 @@ type Provider struct{} // UserData renders user-data template to string. func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { - tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(userDataTemplate) if err != nil { - return "", fmt.Errorf("failed to parse user-data template: %v", err) + return "", fmt.Errorf("failed to parse user-data template: %w", err) } kubeletVersion, err := semver.NewVersion(req.MachineSpec.Versions.Kubelet) if err != nil { - return "", fmt.Errorf("invalid kubelet version: %v", err) + return "", fmt.Errorf("invalid kubelet version: %w", err) } pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) if err != nil { - return "", fmt.Errorf("failed to get providerSpec: %v", err) + return "", fmt.Errorf("failed to get providerSpec: %w", err) } if pconfig.OverwriteCloudConfig != nil { @@ -64,12 +63,12 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { slesConfig, err := LoadConfig(pconfig.OperatingSystemSpec) if err != nil { - return "", fmt.Errorf("failed to get sles config from provider config: %v", err) + return "", fmt.Errorf("failed to get sles config from provider config: %w", err) } serverAddr, err := userdatahelper.GetServerAddressFromKubeconfig(req.Kubeconfig) if err != nil { - return "", fmt.Errorf("error extracting server address from kubeconfig: %v", err) + return "", fmt.Errorf("error extracting server address from kubeconfig: %w", err) } kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) @@ -79,7 +78,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { kubernetesCACert, err := userdatahelper.GetCACert(req.Kubeconfig) if err != nil { - return "", fmt.Errorf("error extracting cacert: %v", err) + return "", fmt.Errorf("error extracting cacert: %w", err) } crEngine := req.ContainerRuntime.Engine(kubeletVersion) @@ -118,7 +117,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { b := &bytes.Buffer{} err = tmpl.Execute(b, data) if err != nil { - return "", fmt.Errorf("failed to execute user-data template: %v", err) + return "", fmt.Errorf("failed to execute user-data template: %w", err) } return userdatahelper.CleanupTemplateOutput(b.String()) } diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index f5c4cd2ce..ab7b547b4 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -40,17 +40,17 @@ type Provider struct{} func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(userDataTemplate) if err != nil { - return "", fmt.Errorf("failed to parse user-data template: %v", err) + return "", fmt.Errorf("failed to parse user-data template: %w", err) } kubeletVersion, err := semver.NewVersion(req.MachineSpec.Versions.Kubelet) if err != nil { - return "", fmt.Errorf("invalid kubelet version: %v", err) + return "", fmt.Errorf("invalid kubelet version: %w", err) } pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) if err != nil { - return "", fmt.Errorf("failed to get providerSpec: %v", err) + return "", fmt.Errorf("failed to get providerSpec: %w", err) } if pconfig.OverwriteCloudConfig != nil { @@ -63,12 +63,12 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ubuntuConfig, err := LoadConfig(pconfig.OperatingSystemSpec) if err != nil { - return "", fmt.Errorf("failed to get ubuntu config from provider config: %v", err) + return "", fmt.Errorf("failed to get ubuntu config from provider config: %w", err) } serverAddr, err := userdatahelper.GetServerAddressFromKubeconfig(req.Kubeconfig) if err != nil { - return "", fmt.Errorf("error extracting server address from kubeconfig: %v", err) + return "", fmt.Errorf("error extracting server address from kubeconfig: %w", err) } kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) @@ -78,7 +78,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { kubernetesCACert, err := userdatahelper.GetCACert(req.Kubeconfig) if err != nil { - return "", fmt.Errorf("error extracting cacert: %v", err) + return "", fmt.Errorf("error extracting cacert: %w", err) } crEngine := req.ContainerRuntime.Engine(kubeletVersion) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index fcadb91bc..d8ae02e6b 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -137,7 +137,7 @@ func TestCustomCAsAreApplied(t *testing.T) { executor: func(kubeConfig, manifestPath string, parameters []string, d time.Duration) error { if err := updateMachineControllerForCustomCA(kubeConfig); err != nil { - return fmt.Errorf("failed to add CA: %v", err) + return fmt.Errorf("failed to add CA: %w", err) } return verifyCreateMachineFails(kubeConfig, manifestPath, parameters, d) @@ -153,12 +153,12 @@ func TestCustomCAsAreApplied(t *testing.T) { func updateMachineControllerForCustomCA(kubeconfig string) error { cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig) if err != nil { - return fmt.Errorf("Error building kubeconfig: %v", err) + return fmt.Errorf("Error building kubeconfig: %w", err) } client, err := ctrlruntimeclient.New(cfg, ctrlruntimeclient.Options{}) if err != nil { - return fmt.Errorf("failed to create Client: %v", err) + return fmt.Errorf("failed to create Client: %w", err) } ctx := context.Background() @@ -210,14 +210,14 @@ C8QmzsMaZhk+mVFr1sGy } if err := client.Create(ctx, caBundle); err != nil { - return fmt.Errorf("failed to create ca-bundle ConfigMap: %v", err) + return fmt.Errorf("failed to create ca-bundle ConfigMap: %w", err) } // add CA to deployments deployments := []string{"machine-controller", "machine-controller-webhook"} for _, deployment := range deployments { if err := addCAToDeployment(ctx, client, deployment, ns); err != nil { - return fmt.Errorf("failed to add CA to %s Deployment: %v", deployment, err) + return fmt.Errorf("failed to add CA to %s Deployment: %w", deployment, err) } } @@ -228,12 +228,12 @@ C8QmzsMaZhk+mVFr1sGy key := types.NamespacedName{Namespace: ns, Name: deployment} if err := client.Get(ctx, key, d); err != nil { - return false, fmt.Errorf("failed to get Deployment: %v", err) + return false, fmt.Errorf("failed to get Deployment: %w", err) } return d.Status.AvailableReplicas > 0, nil }); err != nil { - return fmt.Errorf("%s Deployment never became ready: %v", deployment, err) + return fmt.Errorf("%s Deployment never became ready: %w", deployment, err) } } @@ -245,7 +245,7 @@ func addCAToDeployment(ctx context.Context, client ctrlruntimeclient.Client, nam key := types.NamespacedName{Namespace: namespace, Name: name} if err := client.Get(ctx, key, deployment); err != nil { - return fmt.Errorf("failed to get Deployment: %v", err) + return fmt.Errorf("failed to get Deployment: %w", err) } caVolume := corev1.Volume{ @@ -855,7 +855,7 @@ func TestVsphereResourcePoolProvisioningE2E(t *testing.T) { // note that tests require the following environment variable: // - SCW_ACCESS_KEY -> the Scaleway Access Key // - SCW_SECRET_KEY -> the Scaleway Secret Key -// - SCW_DEFAULT_PROJECT_ID -> the Scaleway Project ID +// - SCW_DEFAULT_PROJECT_ID -> the Scaleway Project ID. func TestScalewayProvisioningE2E(t *testing.T) { t.Parallel() @@ -916,19 +916,19 @@ func getNutanixTestParams(t *testing.T) []string { return params } -// TestNutanixProvisioningE2E tests provisioning on Nutanix as cloud provider +// TestNutanixProvisioningE2E tests provisioning on Nutanix as cloud provider. func TestNutanixProvisioningE2E(t *testing.T) { t.Parallel() // exclude migrateUID test case because it's a no-op for Nutanix and runs from a different - // location, thus possibly blocking access a HTTP proxy if it is configured + // location, thus possibly blocking access a HTTP proxy if it is configured. selector := And(OsSelector("ubuntu", "centos"), Not(NameSelector("migrateUID"))) params := getNutanixTestParams(t) runScenarios(t, selector, params, nutanixManifest, fmt.Sprintf("nx-%s", *testRunIdentifier)) } // TestUbuntuProvisioningWithUpgradeE2E will create an instance from an old Ubuntu 1604 -// image and upgrade it prior to joining the cluster +// image and upgrade it prior to joining the cluster. func TestUbuntuProvisioningWithUpgradeE2E(t *testing.T) { t.Parallel() @@ -966,7 +966,7 @@ func TestUbuntuProvisioningWithUpgradeE2E(t *testing.T) { } // TestDeploymentControllerUpgradesMachineE2E verifies the machineDeployment controller correctly -// rolls over machines on changes in the machineDeployment +// rolls over machines on changes in the machineDeployment. func TestDeploymentControllerUpgradesMachineE2E(t *testing.T) { t.Parallel() diff --git a/test/e2e/provisioning/deploymentscenario.go b/test/e2e/provisioning/deploymentscenario.go index 2423efa43..0ccec451e 100644 --- a/test/e2e/provisioning/deploymentscenario.go +++ b/test/e2e/provisioning/deploymentscenario.go @@ -30,7 +30,6 @@ import ( ) func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []string, timeout time.Duration) error { - client, machineDeployment, err := prepareMachineDeployment(kubeConfig, manifestPath, parameters) if err != nil { return err @@ -40,13 +39,13 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s machineDeployment, err = createAndAssure(machineDeployment, client, timeout) if err != nil { - return fmt.Errorf("failed to verify creation of node for MachineDeployment: %v", err) + return fmt.Errorf("failed to verify creation of node for MachineDeployment: %w", err) } if err := updateMachineDeployment(machineDeployment, client, func(md *clusterv1alpha1.MachineDeployment) { md.Spec.Template.Labels["testUpdate"] = "true" }); err != nil { - return fmt.Errorf("failed to update MachineDeployment %s after modifying it: %v", machineDeployment.Name, err) + return fmt.Errorf("failed to update MachineDeployment %s after modifying it: %w", machineDeployment.Name, err) } klog.Infof("Waiting for second MachineSet to appear after updating MachineDeployment %s", machineDeployment.Name) @@ -126,7 +125,7 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s if err := updateMachineDeployment(machineDeployment, client, func(md *clusterv1alpha1.MachineDeployment) { md.Spec.Replicas = getInt32Ptr(0) }); err != nil { - return fmt.Errorf("failed to update replicas of MachineDeployment %s: %v", machineDeployment.Name, err) + return fmt.Errorf("failed to update replicas of MachineDeployment %s: %w", machineDeployment.Name, err) } klog.Infof("Successfully set replicas of MachineDeployment %s to 0", machineDeployment.Name) @@ -141,7 +140,7 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s klog.Infof("Deleting MachineDeployment %s and waiting for it to disappear", machineDeployment.Name) if err := client.Delete(context.Background(), machineDeployment); err != nil { - return fmt.Errorf("failed to delete MachineDeployment %s: %v", machineDeployment.Name, err) + return fmt.Errorf("failed to delete MachineDeployment %s: %w", machineDeployment.Name, err) } if err := wait.Poll(5*time.Second, timeout, func() (bool, error) { err := client.Get(context.Background(), types.NamespacedName{Namespace: machineDeployment.Namespace, Name: machineDeployment.Name}, &clusterv1alpha1.MachineDeployment{}) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 8fc4f362a..aaaf9cf9a 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -127,7 +127,7 @@ func (a *and) Match(tc scenario) bool { return a.s1.Match(tc) && a.s2.Match(tc) } -// NameSelector is used to match against a test case name +// NameSelector is used to match against a test case name. func NameSelector(tcName string) Selector { return &name{tcName} } @@ -159,7 +159,6 @@ func runScenarios(st *testing.T, selector Selector, testParams []string, manifes type scenarioExecutor func(string, string, []string, time.Duration) error func testScenario(t *testing.T, testCase scenario, cloudProvider string, testParams []string, manifestPath string, parallelize bool) { - if parallelize { t.Parallel() } diff --git a/test/e2e/provisioning/migrateuidscenario.go b/test/e2e/provisioning/migrateuidscenario.go index 8f1e20f60..96579ff0a 100644 --- a/test/e2e/provisioning/migrateuidscenario.go +++ b/test/e2e/provisioning/migrateuidscenario.go @@ -18,6 +18,7 @@ package provisioning import ( "context" + "errors" "fmt" "strings" "time" @@ -32,6 +33,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/klog" fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -40,14 +42,14 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time // prepare the manifest manifest, err := readAndModifyManifest(manifestPath, parameters) if err != nil { - return fmt.Errorf("failed to prepare the manifest, due to: %v", err) + return fmt.Errorf("failed to prepare the manifest, due to: %w", err) } machineDeployment := &v1alpha1.MachineDeployment{} manifestReader := strings.NewReader(manifest) manifestDecoder := yaml.NewYAMLToJSONDecoder(manifestReader) if err := manifestDecoder.Decode(machineDeployment); err != nil { - return fmt.Errorf("failed to decode manifest into MachineDeployment: %v", err) + return fmt.Errorf("failed to decode manifest into MachineDeployment: %w", err) } machine := &v1alpha1.Machine{ ObjectMeta: machineDeployment.Spec.Template.ObjectMeta, @@ -60,7 +62,10 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time machine.Name = machineDeployment.Name machine.Namespace = metav1.NamespaceSystem machine.Spec.Name = machine.Name - fakeClient := fakectrlruntimeclient.NewFakeClient(machine) + fakeClient := fakectrlruntimeclient.NewClientBuilder(). + WithScheme(scheme.Scheme). + WithObjects(machine). + Build() providerData := &cloudprovidertypes.ProviderData{ Update: cloudprovidertypes.GetMachineUpdater(context.Background(), fakeClient), @@ -69,17 +74,16 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time providerSpec, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to get provideSpec: %v", err) + return fmt.Errorf("failed to get provideSpec: %w", err) } skg := providerconfig.NewConfigVarResolver(context.Background(), fakeClient) prov, err := cloudprovider.ForProvider(providerSpec.CloudProvider, skg) if err != nil { - return fmt.Errorf("failed to get cloud provider %q: %v", providerSpec.CloudProvider, err) - + return fmt.Errorf("failed to get cloud provider %q: %w", providerSpec.CloudProvider, err) } defaultedSpec, err := prov.AddDefaults(machine.Spec) if err != nil { - return fmt.Errorf("failed to add defaults: %v", err) + return fmt.Errorf("failed to add defaults: %w", err) } machine.Spec = defaultedSpec @@ -88,13 +92,13 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time for i := 0; i < maxTries; i++ { _, err := prov.Get(machine, providerData) if err != nil { - if err != cloudprovidererrors.ErrInstanceNotFound { + if !errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { if i < maxTries-1 { time.Sleep(10 * time.Second) klog.V(4).Infof("failed to get machine %s before creating it on try %v with err=%v, will retry", machine.Name, i, err) continue } - return fmt.Errorf("failed to get machine %s before creating it: %v", machine.Name, err) + return fmt.Errorf("failed to get machine %s before creating it: %w", machine.Name, err) } _, err := prov.Create(machine, providerData, "#cloud-config\n") if err != nil { @@ -103,7 +107,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time klog.V(4).Infof("failed to create machine %s on try %v with err=%v, will retry", machine.Name, i, err) continue } - return fmt.Errorf("failed to create machine %s: %v", machine.Name, err) + return fmt.Errorf("failed to create machine %s: %w", machine.Name, err) } } break @@ -117,7 +121,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time time.Sleep(10 * time.Second) continue } - return fmt.Errorf("failed to get machine %s after creating it: %v", machine.Name, err) + return fmt.Errorf("failed to get machine %s after creating it: %w", machine.Name, err) } break } @@ -130,7 +134,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time klog.V(4).Infof("failed to migrate UID for machine %s on try %v with err=%v, will retry", machine.Name, i, err) continue } - return fmt.Errorf("failed to migrate UID for machine %s: %v", machine.Name, err) + return fmt.Errorf("failed to migrate UID for machine %s: %w", machine.Name, err) } break } @@ -144,14 +148,13 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time klog.V(4).Infof("failed to get instance for machine %s after migrating on try %v with err=%v, will retry", machine.Name, i, err) continue } - return fmt.Errorf("failed to get machine %s after migrating UID: %v", machine.Name, err) + return fmt.Errorf("failed to get machine %s after migrating UID: %w", machine.Name, err) } break } // Step 4: Delete the instance and then verify instance is gone for i := 0; i < maxTries; i++ { - // Deletion part 0: Delete and continue on err if there are tries left done, err := prov.Cleanup(machine, providerData) if err != nil { @@ -160,7 +163,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time time.Sleep(10 * time.Second) continue } - return fmt.Errorf("failed to delete machine %s: %v", machine.Name, err) + return fmt.Errorf("failed to delete machine %s: %w", machine.Name, err) } if !done { // The deletion is async, thus we wait 10 seconds to recheck if its done @@ -170,7 +173,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time // Deletion part 1: Get and continue if err != cloudprovidererrors.ErrInstanceNotFound if there are tries left _, err = prov.Get(machine, providerData) - if err != nil && err == cloudprovidererrors.ErrInstanceNotFound { + if err != nil && errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { break } if i < maxTries-1 { @@ -179,7 +182,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time time.Sleep(10 * time.Second) continue } - return fmt.Errorf("expected ErrInstanceNotFound after deleting instance for machine %s, but got err=%v", machine.Name, err) + return fmt.Errorf("expected ErrInstanceNotFound after deleting instance for machine %s, but got err=%w", machine.Name, err) } return nil diff --git a/test/e2e/provisioning/verify.go b/test/e2e/provisioning/verify.go index 89a236b4b..6a2f323b2 100644 --- a/test/e2e/provisioning/verify.go +++ b/test/e2e/provisioning/verify.go @@ -56,7 +56,6 @@ func verifyCreateMachineFails(kubeConfig, manifestPath string, parameters []stri } func verifyCreateAndDelete(kubeConfig, manifestPath string, parameters []string, timeout time.Duration) error { - client, machineDeployment, err := prepareMachineDeployment(kubeConfig, manifestPath, parameters) if err != nil { return err @@ -64,11 +63,11 @@ func verifyCreateAndDelete(kubeConfig, manifestPath string, parameters []string, machineDeployment, err = createAndAssure(machineDeployment, client, timeout) if err != nil { - return fmt.Errorf("failed to verify creation of node for MachineDeployment: %v", err) + return fmt.Errorf("failed to verify creation of node for MachineDeployment: %w", err) } if err := deleteAndAssure(machineDeployment, client, timeout); err != nil { - return fmt.Errorf("Failed to verify if a machine/node has been created/deleted, due to: \n%v", err) + return fmt.Errorf("Failed to verify if a machine/node has been created/deleted, due to: \n%w", err) } klog.Infof("Successfully finished test for MachineDeployment %s", machineDeployment.Name) @@ -76,7 +75,6 @@ func verifyCreateAndDelete(kubeConfig, manifestPath string, parameters []string, } func prepareMachineDeployment(kubeConfig, manifestPath string, parameters []string) (ctrlruntimeclient.Client, *clusterv1alpha1.MachineDeployment, error) { - client, manifest, err := prepare(kubeConfig, manifestPath, parameters) if err != nil { return nil, nil, err @@ -97,7 +95,6 @@ func prepareMachineDeployment(kubeConfig, manifestPath string, parameters []stri } func prepareMachine(kubeConfig, manifestPath string, parameters []string) (ctrlruntimeclient.Client, *clusterv1alpha1.Machine, error) { - client, manifest, err := prepare(kubeConfig, manifestPath, parameters) if err != nil { return nil, nil, err @@ -125,17 +122,17 @@ func prepare(kubeConfig, manifestPath string, parameters []string) (ctrlruntimec // init kube related stuff cfg, err := clientcmd.BuildConfigFromFlags("", kubeConfig) if err != nil { - return nil, "", fmt.Errorf("Error building kubeconfig: %v", err) + return nil, "", fmt.Errorf("Error building kubeconfig: %w", err) } client, err := ctrlruntimeclient.New(cfg, ctrlruntimeclient.Options{}) if err != nil { - return nil, "", fmt.Errorf("failed to create Client: %v", err) + return nil, "", fmt.Errorf("failed to create Client: %w", err) } // prepare the manifest manifest, err := readAndModifyManifest(manifestPath, parameters) if err != nil { - return nil, "", fmt.Errorf("failed to prepare the manifest, due to: %v", err) + return nil, "", fmt.Errorf("failed to prepare the manifest, due to: %w", err) } return client, manifest, nil @@ -145,7 +142,7 @@ func createAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien // we expect that no node for machine exists in the cluster err := assureNodeForMachineDeployment(machineDeployment, client, false) if err != nil { - return nil, fmt.Errorf("unable to perform the verification, incorrect cluster state detected %v", err) + return nil, fmt.Errorf("unable to perform the verification, incorrect cluster state detected %w", err) } klog.Infof("Creating a new %q MachineDeployment", machineDeployment.Name) @@ -164,7 +161,7 @@ func createAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien return true, nil }) if err != nil { - return nil, fmt.Errorf("failed to create MachineDeployment %q: %v", machineDeployment.Name, err) + return nil, fmt.Errorf("failed to create MachineDeployment %q: %w", machineDeployment.Name, err) } klog.Infof("MachineDeployment %q created", machineDeployment.Name) @@ -178,7 +175,7 @@ func createAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien return false, nil }) if err != nil { - return nil, fmt.Errorf("failed waiting for MachineDeployment %s to get a node: %v (%v)", machineDeployment.Name, err, pollErr) + return nil, fmt.Errorf("failed waiting for MachineDeployment %s to get a node: %w (%v)", machineDeployment.Name, err, pollErr) } klog.Infof("Found a node for MachineDeployment %s", machineDeployment.Name) @@ -200,7 +197,7 @@ func createAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien return false, nil }) if err != nil { - return nil, fmt.Errorf("failed waiting for MachineDeployment %s to get a node in ready state: %v", machineDeployment.Name, err) + return nil, fmt.Errorf("failed waiting for MachineDeployment %s to get a node in ready state: %w", machineDeployment.Name, err) } return machineDeployment, nil } @@ -208,7 +205,7 @@ func createAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien func hasMachineReadyNode(machine *clusterv1alpha1.Machine, client ctrlruntimeclient.Client) (bool, error) { nodes := &corev1.NodeList{} if err := client.List(context.Background(), nodes); err != nil { - return false, fmt.Errorf("failed to list nodes: %v", err) + return false, fmt.Errorf("failed to list nodes: %w", err) } for _, node := range nodes.Items { if isNodeForMachine(&node, machine) { @@ -231,7 +228,7 @@ func deleteAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien if err := updateMachineDeployment(machineDeployment, client, func(md *clusterv1alpha1.MachineDeployment) { md.Spec.Replicas = getInt32Ptr(0) }); err != nil { - return fmt.Errorf("failed to update replicas of MachineDeployment %s: %v", machineDeployment.Name, err) + return fmt.Errorf("failed to update replicas of MachineDeployment %s: %w", machineDeployment.Name, err) } // Ensure machines are gone @@ -245,12 +242,12 @@ func deleteAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien } return true, nil }); err != nil { - return fmt.Errorf("failed to wait for machines of MachineDeployment %s to be deleted: %v", machineDeployment.Name, err) + return fmt.Errorf("failed to wait for machines of MachineDeployment %s to be deleted: %w", machineDeployment.Name, err) } klog.V(2).Infof("Deleting MachineDeployment %s", machineDeployment.Name) if err := client.Delete(context.Background(), machineDeployment); err != nil { - return fmt.Errorf("unable to remove MachineDeployment %s, due to %v", machineDeployment.Name, err) + return fmt.Errorf("unable to remove MachineDeployment %s, due to %w", machineDeployment.Name, err) } return wait.Poll(machineReadyCheckPeriod, timeout, func() (bool, error) { err := client.Get(context.Background(), types.NamespacedName{Namespace: machineDeployment.Namespace, Name: machineDeployment.Name}, &clusterv1alpha1.MachineDeployment{}) @@ -264,10 +261,9 @@ func deleteAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien // assureNodeForMachineDeployment according to shouldExists parameter check if a node for machine exists in the system or not // this method examines OwnerReference of each node. func assureNodeForMachineDeployment(machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Client, shouldExist bool) error { - machines, err := getMatchingMachines(machineDeployment, client) if err != nil { - return fmt.Errorf("failed to list Machines: %v", err) + return fmt.Errorf("failed to list Machines: %w", err) } if shouldExist { @@ -279,7 +275,7 @@ func assureNodeForMachineDeployment(machineDeployment *clusterv1alpha1.MachineDe // Azure doesn't seem to easely expose the private IP address, there is only a PublicIPAddressClient in the sdk providerConfig, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) if err != nil { - return fmt.Errorf("failed to get provider config: %v", err) + return fmt.Errorf("failed to get provider config: %w", err) } if providerConfig.CloudProvider == providerconfigtypes.CloudProviderAzure { continue @@ -289,12 +285,11 @@ func assureNodeForMachineDeployment(machineDeployment *clusterv1alpha1.MachineDe return fmt.Errorf("expected to find a node for MachineDeployment %q but Machine %q has no address yet, indicating instance creation at the provider failed", machineDeployment.Name, machine.Name) } } - } nodes := &corev1.NodeList{} if err := client.List(context.Background(), nodes); err != nil { - return fmt.Errorf("failed to list Nodes: %v", err) + return fmt.Errorf("failed to list Nodes: %w", err) } nodeForMachineExists := false @@ -341,7 +336,7 @@ func readAndModifyManifest(pathToManifest string, keyValuePairs []string) (strin return content, nil } -// getMatchingMachines returns all machines that are owned by the passed machineDeployment +// getMatchingMachines returns all machines that are owned by the passed machineDeployment. func getMatchingMachines(machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Client) ([]clusterv1alpha1.Machine, error) { matchingMachineSets, err := getMachingMachineSets(machineDeployment, client) if err != nil { @@ -352,7 +347,7 @@ func getMatchingMachines(machineDeployment *clusterv1alpha1.MachineDeployment, c for _, machineSet := range matchingMachineSets { machinesForMachineSet, err := getMatchingMachinesForMachineset(&machineSet, client) if err != nil { - return nil, fmt.Errorf("failed to get matching Machines for MachineSet %s: %v", machineSet.Name, err) + return nil, fmt.Errorf("failed to get matching Machines for MachineSet %s: %w", machineSet.Name, err) } matchingMachines = append(matchingMachines, machinesForMachineSet...) } @@ -363,7 +358,7 @@ func getMatchingMachines(machineDeployment *clusterv1alpha1.MachineDeployment, c func getMatchingMachinesForMachineset(machineSet *clusterv1alpha1.MachineSet, client ctrlruntimeclient.Client) ([]clusterv1alpha1.Machine, error) { allMachines := &clusterv1alpha1.MachineList{} if err := client.List(context.Background(), allMachines, &ctrlruntimeclient.ListOptions{Namespace: machineSet.Namespace}); err != nil { - return nil, fmt.Errorf("failed to list Machines: %v", err) + return nil, fmt.Errorf("failed to list Machines: %w", err) } var matchingMachines []clusterv1alpha1.Machine for _, machine := range allMachines.Items { @@ -374,22 +369,22 @@ func getMatchingMachinesForMachineset(machineSet *clusterv1alpha1.MachineSet, cl return matchingMachines, nil } -// getMachingMachineSets returns all machineSets that are owned by the passed machineDeployment +// getMachingMachineSets returns all machineSets that are owned by the passed machineDeployment. func getMachingMachineSets(machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Reader) ([]clusterv1alpha1.MachineSet, error) { // Ensure we actually have an object from the KubeAPI and not just the result of the yaml parsing, as the latter - // can not be the owner of anything due to missing UID + // can not be the owner of anything due to missing UID. if machineDeployment.ResourceVersion == "" { nn := types.NamespacedName{Namespace: machineDeployment.Namespace, Name: machineDeployment.Name} if err := client.Get(context.Background(), nn, machineDeployment); err != nil { if !kerrors.IsNotFound(err) { - return nil, fmt.Errorf("failed to get MachineDeployment %s: %v", nn.Name, err) + return nil, fmt.Errorf("failed to get MachineDeployment %s: %w", nn.Name, err) } return nil, nil } } allMachineSets := &clusterv1alpha1.MachineSetList{} if err := client.List(context.Background(), allMachineSets, &ctrlruntimeclient.ListOptions{Namespace: machineDeployment.Namespace}); err != nil { - return nil, fmt.Errorf("failed to list MachineSets: %v", err) + return nil, fmt.Errorf("failed to list MachineSets: %w", err) } var matchingMachineSets []clusterv1alpha1.MachineSet for _, machineSet := range allMachineSets.Items { From 3d1687f81c6bd8a5ef74af066b568957866281ef Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Wed, 18 May 2022 11:16:10 +0200 Subject: [PATCH 147/489] Contexts, contexts everywhere. (#1294) * properly inject the context, get rid of background/TODO contexts * use upstream signals helper --- cmd/machine-controller/main.go | 9 +-- pkg/admission/machines.go | 2 +- .../cluster/v1alpha1/migrations/migrations.go | 4 +- .../provider/alibaba/provider.go | 13 ++-- pkg/cloudprovider/provider/anexia/provider.go | 26 ++++---- .../provider/anexia/provider_test.go | 11 ++-- .../provider/anexia/utils/utils.go | 4 +- pkg/cloudprovider/provider/aws/provider.go | 11 ++-- pkg/cloudprovider/provider/azure/provider.go | 59 +++++++++---------- .../provider/baremetal/provider.go | 20 +++---- .../provider/digitalocean/provider.go | 41 ++++++------- .../provider/equinixmetal/provider.go | 13 ++-- pkg/cloudprovider/provider/fake/provider.go | 15 ++--- pkg/cloudprovider/provider/gce/provider.go | 13 ++-- .../provider/gce/provider_test.go | 2 +- .../provider/hetzner/provider.go | 19 ++---- .../provider/kubevirt/provider.go | 17 +++--- pkg/cloudprovider/provider/linode/provider.go | 33 ++++------- .../provider/nutanix/provider.go | 13 ++-- .../provider/openstack/provider.go | 13 ++-- .../provider/openstack/provider_test.go | 2 +- .../provider/scaleway/provider.go | 12 ++-- pkg/cloudprovider/provider/vsphere/client.go | 4 +- .../provider/vsphere/helper_test.go | 4 +- .../provider/vsphere/provider.go | 33 ++++------- .../provider/vsphere/provider_test.go | 2 +- pkg/cloudprovider/types/types.go | 10 ++-- pkg/cloudprovider/validationwrapper.go | 21 +++---- pkg/controller/machine/machine_controller.go | 16 ++--- pkg/rhsm/subscription_manager.go | 14 ++--- pkg/rhsm/subscription_manager_test.go | 3 +- pkg/signals/signal.go | 47 --------------- test/e2e/provisioning/migrateuidscenario.go | 16 ++--- 33 files changed, 220 insertions(+), 302 deletions(-) delete mode 100644 pkg/signals/signal.go diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 20b6b2055..6b946b34a 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -41,7 +41,6 @@ import ( "github.com/kubermatic/machine-controller/pkg/health" machinesv1alpha1 "github.com/kubermatic/machine-controller/pkg/machines/v1alpha1" "github.com/kubermatic/machine-controller/pkg/node" - "github.com/kubermatic/machine-controller/pkg/signals" osmv1alpha1 "k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -53,6 +52,7 @@ import ( "k8s.io/klog" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" "sigs.k8s.io/controller-runtime/pkg/metrics" ) @@ -282,13 +282,10 @@ func main() { runOptions.bootstrapTokenServiceAccountName = &types.NamespacedName{Namespace: flagParts[0], Name: flagParts[1]} } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - signalCh := signals.SetupSignalHandler() + ctx := signals.SetupSignalHandler() go func() { - <-signalCh + <-ctx.Done() klog.Info("caught signal, shutting down...") - cancel() }() mgr, err := createManager(5*time.Minute, runOptions) diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index d317ef615..978dd5d98 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -165,7 +165,7 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec } spec = &defaultedSpec - if err := prov.Validate(*spec); err != nil { + if err := prov.Validate(ctx, *spec); err != nil { return fmt.Errorf("validation failed: %w", err) } diff --git a/pkg/apis/cluster/v1alpha1/migrations/migrations.go b/pkg/apis/cluster/v1alpha1/migrations/migrations.go index b2696588e..0d1f712fa 100644 --- a/pkg/apis/cluster/v1alpha1/migrations/migrations.go +++ b/pkg/apis/cluster/v1alpha1/migrations/migrations.go @@ -293,13 +293,13 @@ func migrateMachines(ctx context.Context, client ctrlruntimeclient.Client, kubeC klog.Infof("Attempting to update the UID at the cloud provider for machine.cluster.k8s.io/v1alpha1 %s", machinesV1Alpha1Machine.Name) newMachineWithOldUID := finalClusterV1Alpha1Machine.DeepCopy() newMachineWithOldUID.UID = machinesV1Alpha1Machine.UID - if err := prov.MigrateUID(newMachineWithOldUID, finalClusterV1Alpha1Machine.UID); err != nil { + if err := prov.MigrateUID(ctx, newMachineWithOldUID, finalClusterV1Alpha1Machine.UID); err != nil { return fmt.Errorf("running the provider migration for the UID failed: %w", err) } // Block until we can actually GET the instance with the new UID var isMigrated bool for i := 0; i < 100; i++ { - if _, err := prov.Get(finalClusterV1Alpha1Machine, providerData); err == nil { + if _, err := prov.Get(ctx, finalClusterV1Alpha1Machine, providerData); err == nil { isMigrated = true break } diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index 33201c3b4..64cd3339a 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -17,6 +17,7 @@ limitations under the License. package alibaba import ( + "context" "encoding/base64" "errors" "fmt" @@ -107,7 +108,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, nil } -func (p *provider) Validate(machineSpec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(_ context.Context, machineSpec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(machineSpec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -148,7 +149,7 @@ func (p *provider) Validate(machineSpec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -197,7 +198,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri return "", "", nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -257,8 +258,8 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return &alibabaInstance{instance: foundInstance}, nil } -func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - foundInstance, err := p.Get(machine, data) +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { + foundInstance, err := p.Get(ctx, machine, data) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return util.RemoveFinalizerOnInstanceNotFound(finalizerInstance, machine, data) @@ -302,7 +303,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to decode providerconfig: %w", err) diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index fca8ea5e4..337389fa0 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -58,7 +58,7 @@ type provider struct { configVarResolver *providerconfig.ConfigVarResolver } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance instance.Instance, retErr error) { +func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance instance.Instance, retErr error) { status := getProviderStatus(machine) klog.V(3).Infof(fmt.Sprintf("'%s' has status %#v", machine.Name, status)) @@ -70,7 +70,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return nil, fmt.Errorf("unable to get provider config: %w", err) } - ctx := utils.CreateReconcileContext(utils.ReconcileContext{ + ctx = utils.CreateReconcileContext(ctx, utils.ReconcileContext{ Status: &status, UserData: userdata, Config: config, @@ -96,7 +96,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert if err != nil { return nil, err } - return p.Get(machine, data) + return p.Get(ctx, machine, data) } // provision machine @@ -104,7 +104,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert if err != nil { return nil, err } - return p.Get(machine, data) + return p.Get(ctx, machine, data) } func waitForVM(ctx context.Context, client anxclient.Client) error { @@ -335,7 +335,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha } // Validate returns success or failure based according to its ProviderSpec. -func (p *provider) Validate(machinespec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(_ context.Context, machinespec clusterv1alpha1.MachineSpec) error { config, _, err := p.getConfig(machinespec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -372,7 +372,7 @@ func (p *provider) Validate(machinespec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, "failed to parse MachineSpec: %v", err) @@ -392,10 +392,10 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P return nil, cloudprovidererrors.ErrInstanceNotFound } - ctx, cancel := context.WithTimeout(context.Background(), anxtypes.GetRequestTimeout) + timeoutCtx, cancel := context.WithTimeout(ctx, anxtypes.GetRequestTimeout) defer cancel() - info, err := vsphereAPI.Info().Get(ctx, status.InstanceID) + info, err := vsphereAPI.Info().Get(timeoutCtx, status.InstanceID) if err != nil { return nil, fmt.Errorf("failed get machine info: %w", err) } @@ -409,7 +409,7 @@ func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string return "", "", nil } -func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (isDeleted bool, retErr error) { +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (isDeleted bool, retErr error) { status := getProviderStatus(machine) // make sure status is reflected in Machine Object defer func() { @@ -433,12 +433,12 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider return false, newError(common.InvalidConfigurationMachineError, "failed to get machine status: %v", err) } - ctx, cancel := context.WithTimeout(context.Background(), anxtypes.DeleteRequestTimeout) + deleteCtx, cancel := context.WithTimeout(ctx, anxtypes.DeleteRequestTimeout) defer cancel() // first check whether there is an provisioning ongoing if status.DeprovisioningID == "" { - response, err := vsphereAPI.Provisioning().VM().Deprovision(ctx, status.InstanceID, false) + response, err := vsphereAPI.Provisioning().VM().Deprovision(deleteCtx, status.InstanceID, false) if err != nil { var respErr *anxclient.ResponseError // Only error if the error was not "not found" @@ -449,7 +449,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider status.DeprovisioningID = response.Identifier } - return isTaskDone(ctx, cli, status.DeprovisioningID) + return isTaskDone(deleteCtx, cli, status.DeprovisioningID) } func isTaskDone(ctx context.Context, cli anxclient.Client, progressIdentifier string) (bool, error) { @@ -470,7 +470,7 @@ func isTaskDone(ctx context.Context, cli anxclient.Client, progressIdentifier st return false, nil } -func (p *provider) MigrateUID(_ *clusterv1alpha1.Machine, _ k8stypes.UID) error { +func (p *provider) MigrateUID(_ context.Context, _ *clusterv1alpha1.Machine, _ k8stypes.UID) error { return nil } diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index 5daaaf4af..9345f7450 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -17,6 +17,7 @@ limitations under the License. package anexia import ( + "context" "encoding/json" "errors" "net/http" @@ -57,7 +58,7 @@ func TestAnexiaProvider(t *testing.T) { testhelper.Mux.HandleFunc("/api/vsphere/v1/search/by_name.json", createSearchHandler(t, waitUntilVMIsFound)) providerStatus := anxtypes.ProviderStatus{} - ctx := utils.CreateReconcileContext(utils.ReconcileContext{ + ctx := utils.CreateReconcileContext(context.Background(), utils.ReconcileContext{ Machine: &v1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{Name: "TestMachine"}, }, @@ -149,7 +150,7 @@ func TestAnexiaProvider(t *testing.T) { }) providerStatus := anxtypes.ProviderStatus{} - ctx := utils.CreateReconcileContext(utils.ReconcileContext{ + ctx := utils.CreateReconcileContext(context.Background(), utils.ReconcileContext{ Machine: &v1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{Name: "TestMachine"}, }, @@ -185,7 +186,7 @@ func TestAnexiaProvider(t *testing.T) { }, }, } - ctx := utils.CreateReconcileContext(utils.ReconcileContext{ + ctx := utils.CreateReconcileContext(context.Background(), utils.ReconcileContext{ Status: &providerStatus, UserData: "", Config: nil, @@ -213,7 +214,7 @@ func TestAnexiaProvider(t *testing.T) { ReservedIP: "", IPState: "", } - ctx := utils.CreateReconcileContext(utils.ReconcileContext{Status: providerStatus}) + ctx := utils.CreateReconcileContext(context.Background(), utils.ReconcileContext{Status: providerStatus}) t.Run("with unbound reserved IP", func(t *testing.T) { expectedIP := "8.8.8.8" @@ -270,7 +271,7 @@ func TestValidate(t *testing.T) { provider := New(nil) for _, testCase := range getSpecsForValidationTest(t, configCases) { - err := provider.Validate(testCase.Spec) + err := provider.Validate(context.Background(), testCase.Spec) if testCase.ExpectedError != nil { testhelper.AssertEquals(t, testCase.ExpectedError.Error(), err.Error()) } else { diff --git a/pkg/cloudprovider/provider/anexia/utils/utils.go b/pkg/cloudprovider/provider/anexia/utils/utils.go index 97aa73d5e..51a2a549b 100644 --- a/pkg/cloudprovider/provider/anexia/utils/utils.go +++ b/pkg/cloudprovider/provider/anexia/utils/utils.go @@ -36,8 +36,8 @@ type ReconcileContext struct { ProviderData *cloudprovidertypes.ProviderData } -func CreateReconcileContext(cc ReconcileContext) context.Context { - return context.WithValue(context.Background(), MachineReconcileContextKey, cc) +func CreateReconcileContext(ctx context.Context, cc ReconcileContext) context.Context { + return context.WithValue(ctx, MachineReconcileContextKey, cc) } func GetReconcileContext(ctx context.Context) ReconcileContext { diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index fc2592c13..e9911e57b 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -17,6 +17,7 @@ limitations under the License. package aws import ( + "context" "encoding/base64" "encoding/json" "errors" @@ -572,7 +573,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, err } -func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { config, pc, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -684,7 +685,7 @@ func getVpc(client *ec2.EC2, id string) (*ec2.Vpc, error) { return vpcOut.Vpcs[0], nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -836,7 +837,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return &awsInstance{instance: runOut.Instances[0]}, nil } -func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(_ context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { ec2instance, err := p.get(machine) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { @@ -889,7 +890,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertyp return false, nil } -func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { return p.get(machine) } @@ -976,7 +977,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { machineInstance, err := p.get(machine) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index beaf5a9dc..3313d3ae5 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -548,7 +548,7 @@ func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) return sp, nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { config, providerCfg, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -589,13 +589,13 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert }); err != nil { return nil, err } - publicIP, err = createOrUpdatePublicIPAddress(context.TODO(), publicIPName(ifaceName(machine)), network.IPVersionIPv4, sku, network.IPAllocationMethodStatic, machine.UID, config) + publicIP, err = createOrUpdatePublicIPAddress(ctx, publicIPName(ifaceName(machine)), network.IPVersionIPv4, sku, network.IPAllocationMethodStatic, machine.UID, config) if err != nil { return nil, fmt.Errorf("failed to create public IP: %w", err) } if ipFamily == util.DualStack { - publicIPv6, err = createOrUpdatePublicIPAddress(context.TODO(), publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodStatic, machine.UID, config) + publicIPv6, err = createOrUpdatePublicIPAddress(ctx, publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodStatic, machine.UID, config) if err != nil { return nil, fmt.Errorf("failed to create public IP: %w", err) } @@ -610,7 +610,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return nil, err } - iface, err := createOrUpdateNetworkInterface(context.TODO(), ifaceName(machine), machine.UID, config, publicIP, publicIPv6, ipFamily) + iface, err := createOrUpdateNetworkInterface(ctx, ifaceName(machine), machine.UID, config, publicIP, publicIPv6, ipFamily) if err != nil { return nil, fmt.Errorf("failed to generate main network interface: %w", err) } @@ -686,12 +686,12 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return nil, err } - future, err := vmClient.CreateOrUpdate(context.TODO(), config.ResourceGroup, machine.Name, vmSpec) + future, err := vmClient.CreateOrUpdate(ctx, config.ResourceGroup, machine.Name, vmSpec) if err != nil { return nil, fmt.Errorf("trying to create a VM: %w", err) } - err = future.WaitForCompletionRef(context.TODO(), vmClient.Client) + err = future.WaitForCompletionRef(ctx, vmClient.Client) if err != nil { return nil, fmt.Errorf("waiting for operation returned: %w", err) } @@ -702,17 +702,17 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert } // get the actual VM object filled in with additional data - vm, err = vmClient.Get(context.TODO(), config.ResourceGroup, machine.Name, "") + vm, err = vmClient.Get(ctx, config.ResourceGroup, machine.Name, "") if err != nil { return nil, fmt.Errorf("failed to retrieve updated data for VM %q: %w", machine.Name, err) } - ipAddresses, err := getVMIPAddresses(context.TODO(), config, &vm) + ipAddresses, err := getVMIPAddresses(ctx, config, &vm) if err != nil { return nil, fmt.Errorf("failed to retrieve IP addresses for VM %q: %w", machine.Name, err) } - status, err := getVMStatus(context.TODO(), config, machine.Name) + status, err := getVMStatus(ctx, config, machine.Name) if err != nil { return nil, fmt.Errorf("failed to retrieve status for VM %q: %w", machine.Name, err) } @@ -720,7 +720,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return &azureVM{vm: &vm, ipAddresses: ipAddresses, status: status}, nil } -func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, fmt.Errorf("failed to parse MachineSpec: %w", err) @@ -728,7 +728,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider if kuberneteshelper.HasFinalizer(machine, finalizerVM) { klog.Infof("deleting VM %q", machine.Name) - if err = deleteVMsByMachineUID(context.TODO(), config, machine.UID); err != nil { + if err = deleteVMsByMachineUID(ctx, config, machine.UID); err != nil { return false, fmt.Errorf("failed to delete instance for machine %q: %w", machine.Name, err) } @@ -741,7 +741,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider if kuberneteshelper.HasFinalizer(machine, finalizerDisks) { klog.Infof("deleting disks of VM %q", machine.Name) - if err := deleteDisksByMachineUID(context.TODO(), config, machine.UID); err != nil { + if err := deleteDisksByMachineUID(ctx, config, machine.UID); err != nil { return false, fmt.Errorf("failed to remove disks of machine %q: %w", machine.Name, err) } if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { @@ -753,7 +753,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider if kuberneteshelper.HasFinalizer(machine, finalizerNIC) { klog.Infof("deleting network interfaces of VM %q", machine.Name) - if err := deleteInterfacesByMachineUID(context.TODO(), config, machine.UID); err != nil { + if err := deleteInterfacesByMachineUID(ctx, config, machine.UID); err != nil { return false, fmt.Errorf("failed to remove network interfaces of machine %q: %w", machine.Name, err) } if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { @@ -765,7 +765,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider if kuberneteshelper.HasFinalizer(machine, finalizerPublicIP) { klog.Infof("deleting public IP addresses of VM %q", machine.Name) - if err := deleteIPAddressesByMachineUID(context.TODO(), config, machine.UID); err != nil { + if err := deleteIPAddressesByMachineUID(ctx, config, machine.UID); err != nil { return false, fmt.Errorf("failed to remove public IP addresses of machine %q: %w", machine.Name, err) } if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { @@ -862,17 +862,17 @@ func getVMStatus(ctx context.Context, c *config, vmName string) (instance.Status } } -func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - return p.get(machine) +func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { + return p.get(ctx, machine) } -func (p *provider) get(machine *clusterv1alpha1.Machine) (*azureVM, error) { +func (p *provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (*azureVM, error) { config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("failed to parse MachineSpec: %w", err) } - vm, err := getVMByUID(context.TODO(), config, machine.UID) + vm, err := getVMByUID(ctx, config, machine.UID) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return nil, cloudprovidererrors.ErrInstanceNotFound @@ -881,12 +881,12 @@ func (p *provider) get(machine *clusterv1alpha1.Machine) (*azureVM, error) { return nil, fmt.Errorf("failed to find machine %q by its UID: %w", machine.UID, err) } - ipAddresses, err := getVMIPAddresses(context.TODO(), config, vm) + ipAddresses, err := getVMIPAddresses(ctx, config, vm) if err != nil { return nil, fmt.Errorf("failed to retrieve IP addresses for VM %v: %w", vm.Name, err) } - status, err := getVMStatus(context.TODO(), config, machine.Name) + status, err := getVMStatus(ctx, config, machine.Name) if err != nil { return nil, fmt.Errorf("failed to retrieve status for VM %v: %w", vm.Name, err) } @@ -932,9 +932,9 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri return s, "azure", nil } -func validateDiskSKUs(c *config) error { +func validateDiskSKUs(ctx context.Context, c *config) error { if c.OSDiskSKU != nil || c.DataDiskSKU != nil { - sku, err := getSKU(context.TODO(), c) + sku, err := getSKU(ctx, c) if err != nil { return fmt.Errorf("failed to get VM SKU: %w", err) } @@ -969,7 +969,7 @@ func validateDiskSKUs(c *config) error { return nil } -func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { c, providerConfig, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -1023,20 +1023,20 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return fmt.Errorf("failed to (create) vm client: %w", err) } - _, err = vmClient.List(context.TODO(), c.ResourceGroup, "") + _, err = vmClient.List(ctx, c.ResourceGroup, "") if err != nil { return fmt.Errorf("failed to list virtual machines: %w", err) } - if _, err := getVirtualNetwork(context.TODO(), c); err != nil { + if _, err := getVirtualNetwork(ctx, c); err != nil { return fmt.Errorf("failed to get virtual network: %w", err) } - if _, err := getSubnet(context.TODO(), c); err != nil { + if _, err := getSubnet(ctx, c); err != nil { return fmt.Errorf("failed to get subnet: %w", err) } - if err := validateDiskSKUs(c); err != nil { + if err := validateDiskSKUs(ctx, c); err != nil { return fmt.Errorf("failed to validate disk SKUs: %w", err) } @@ -1056,10 +1056,7 @@ func publicIPv6Name(ifaceName string) string { return ifaceName + "-pubipv6" } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - +func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/baremetal/provider.go b/pkg/cloudprovider/provider/baremetal/provider.go index 0ae28cffc..e62e757ff 100644 --- a/pkg/cloudprovider/provider/baremetal/provider.go +++ b/pkg/cloudprovider/provider/baremetal/provider.go @@ -166,7 +166,7 @@ func (p provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1 return spec, err } -func (p provider) Validate(spec clusterv1alpha1.MachineSpec) error { +func (p provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { c, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -183,7 +183,7 @@ func (p provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func (p provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -192,7 +192,7 @@ func (p provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.Pr } } - server, err := c.driver.GetServer(context.Background(), machine.UID, c.driverSpec) + server, err := c.driver.GetServer(ctx, machine.UID, c.driverSpec) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return nil, cloudprovidererrors.ErrInstanceNotFound @@ -206,11 +206,11 @@ func (p provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.Pr }, nil } -func (p provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { +func (p provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } -func (p provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -219,7 +219,6 @@ func (p provider) Create(machine *clusterv1alpha1.Machine, data *cloudproviderty } } - ctx := context.Background() if err := util.CreateMachineCloudInitSecret(ctx, userdata, machine.Name, data.Client); err != nil { return nil, fmt.Errorf("failed to create cloud-init secret for machine %s: %w", machine.Name, err) } @@ -246,7 +245,7 @@ func (p provider) Create(machine *clusterv1alpha1.Machine, data *cloudproviderty }, nil } -func (p provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, cloudprovidererrors.TerminalError{ @@ -255,7 +254,6 @@ func (p provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidert } } - ctx := context.Background() if err := c.driver.DeprovisionServer(ctx, machine.UID); err != nil { return false, fmt.Errorf("failed to de-provision server: %w", err) } @@ -276,14 +274,14 @@ func (p provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidert return true, nil } -func (p provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { +func (p provider) MachineMetricsLabels(_ *clusterv1alpha1.Machine) (map[string]string, error) { return nil, nil } -func (p provider) MigrateUID(machine *clusterv1alpha1.Machine, uid types.UID) error { +func (p provider) MigrateUID(_ context.Context, _ *clusterv1alpha1.Machine, _ types.UID) error { return nil } -func (p provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { +func (p provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index 031b14a34..fcbe3db09 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -93,12 +93,12 @@ func getSlugForOS(os providerconfigtypes.OperatingSystem) (string, error) { return "", providerconfigtypes.ErrOSNotSupported } -func getClient(token string) *godo.Client { +func getClient(ctx context.Context, token string) *godo.Client { tokenSource := &TokenSource{ AccessToken: token, } - oauthClient := oauth2.NewClient(context.Background(), tokenSource) + oauthClient := oauth2.NewClient(ctx, tokenSource) return godo.NewClient(oauthClient) } @@ -165,7 +165,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, nil } -func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -188,8 +188,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return fmt.Errorf("invalid operating system specified %q: %w", pc.OperatingSystem, err) } - ctx := context.TODO() - client := getClient(c.Token) + client := getClient(ctx, c.Token) regions, _, err := client.Regions.List(ctx, &godo.ListOptions{PerPage: 1000}) if err != nil { @@ -265,7 +264,7 @@ func uploadRandomSSHPublicKey(ctx context.Context, service godo.KeysService) (st return newDoKey.Fingerprint, nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -274,8 +273,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert } } - ctx := context.TODO() - client := getClient(c.Token) + client := getClient(ctx, c.Token) fingerprint, err := uploadRandomSSHPublicKey(ctx, client.Keys) if err != nil { @@ -337,8 +335,8 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return &doInstance{droplet: droplet}, err } -func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { - instance, err := p.get(machine) +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { + instance, err := p.get(ctx, machine) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return true, nil @@ -353,8 +351,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertyp Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), } } - ctx := context.TODO() - client := getClient(c.Token) + client := getClient(ctx, c.Token) doID, err := strconv.Atoi(instance.ID()) if err != nil { @@ -369,11 +366,11 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertyp return false, nil } -func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - return p.get(machine) +func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { + return p.get(ctx, machine) } -func (p *provider) get(machine *clusterv1alpha1.Machine) (*doInstance, error) { +func (p *provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (*doInstance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -382,7 +379,7 @@ func (p *provider) get(machine *clusterv1alpha1.Machine) (*doInstance, error) { } } - droplets, err := p.listDroplets(c.Token) + droplets, err := p.listDroplets(ctx, c.Token) if err != nil { return nil, err } @@ -396,9 +393,8 @@ func (p *provider) get(machine *clusterv1alpha1.Machine) (*doInstance, error) { return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) listDroplets(token string) ([]godo.Droplet, error) { - ctx := context.TODO() - client := getClient(token) +func (p *provider) listDroplets(ctx context.Context, token string) ([]godo.Droplet, error) { + client := getClient(ctx, token) result := make([]godo.Droplet, 0) opt := &godo.ListOptions{ @@ -428,15 +424,12 @@ func (p *provider) listDroplets(token string) ([]godo.Droplet, error) { return result, nil } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - +func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to decode providerconfig: %w", err) } - client := getClient(c.Token) + client := getClient(ctx, c.Token) droplets, _, err := client.Droplets.List(ctx, &godo.ListOptions{PerPage: 1000}) if err != nil { return fmt.Errorf("failed to list droplets: %w", err) diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index da076e57b..94ac01b82 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -17,6 +17,7 @@ limitations under the License. package equinixmetal import ( + "context" "encoding/json" "errors" "fmt" @@ -163,7 +164,7 @@ func (p *provider) getMetalDevice(machine *clusterv1alpha1.Machine) (*packngo.De return device, client, nil } -func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { c, _, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -214,7 +215,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, _, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -254,8 +255,8 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return &metalDevice{device: device}, nil } -func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - instance, err := p.Get(machine, data) +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { + instance, err := p.Get(ctx, machine, data) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return true, nil @@ -293,7 +294,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, nil } -func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { device, _, err := p.getMetalDevice(machine) if err != nil { return nil, err @@ -305,7 +306,7 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newID types.UID) error { +func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machine, newID types.UID) error { device, client, err := p.getMetalDevice(machine) if err != nil { return err diff --git a/pkg/cloudprovider/provider/fake/provider.go b/pkg/cloudprovider/provider/fake/provider.go index b25950518..b3d116d59 100644 --- a/pkg/cloudprovider/provider/fake/provider.go +++ b/pkg/cloudprovider/provider/fake/provider.go @@ -17,6 +17,7 @@ limitations under the License. package fake import ( + "context" "encoding/json" "fmt" @@ -65,7 +66,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha } // Validate returns success or failure based according to its FakeCloudProviderSpec. -func (p *provider) Validate(machinespec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(_ context.Context, machinespec clusterv1alpha1.MachineSpec) error { pconfig, err := providerconfigtypes.GetConfig(machinespec.ProviderSpec) if err != nil { return err @@ -85,28 +86,28 @@ func (p *provider) Validate(machinespec clusterv1alpha1.MachineSpec) error { return fmt.Errorf("failing validation as requested") } -func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(_ context.Context, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { return CloudProviderInstance{}, nil } -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (string, string, error) { +func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string, error) { return "", "", nil } // Create creates a cloud instance according to the given machine. -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(_ context.Context, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, _ string) (instance.Instance, error) { return CloudProviderInstance{}, nil } -func (p *provider) Cleanup(_ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(_ context.Context, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { return true, nil } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, uid types.UID) error { +func (p *provider) MigrateUID(_ context.Context, _ *clusterv1alpha1.Machine, _ types.UID) error { return nil } -func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { +func (p *provider) MachineMetricsLabels(_ *clusterv1alpha1.Machine) (map[string]string, error) { return map[string]string{}, nil } diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 4d4bd1764..54f63f9d4 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -21,6 +21,7 @@ limitations under the License. package gce import ( + "context" "errors" "fmt" "net/http" @@ -101,7 +102,7 @@ func (p *Provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha } // Validate checks the given machine's specification. -func (p *Provider) Validate(spec clusterv1alpha1.MachineSpec) error { +func (p *Provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { // Read configuration. cfg, err := newConfig(p.resolver, spec.ProviderSpec) if err != nil { @@ -142,7 +143,7 @@ func (p *Provider) Validate(spec clusterv1alpha1.MachineSpec) error { } // Get retrieves a node instance that is associated with the given machine. -func (p *Provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *Provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { return p.get(machine) } @@ -209,7 +210,7 @@ func (p *Provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri } // Create inserts a cloud instance according to the given machine. -func (p *Provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *Provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { // Read configuration. cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) if err != nil { @@ -285,11 +286,11 @@ func (p *Provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return nil, newError(common.InvalidConfigurationMachineError, errInsertInstance, err) } // Retrieve it to get a full qualified instance. - return p.Get(machine, data) + return p.Get(ctx, machine, data) } // Cleanup deletes the instance associated with the machine and all associated resources. -func (p *Provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *Provider) Cleanup(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { // Read configuration. cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) if err != nil { @@ -340,7 +341,7 @@ func (p *Provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s // MigrateUID updates the UID of an instance after the controller migrates types // and the UID of the machine object changed. -func (p *Provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *Provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { // Read configuration. cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) if err != nil { diff --git a/pkg/cloudprovider/provider/gce/provider_test.go b/pkg/cloudprovider/provider/gce/provider_test.go index f63ef2248..d8a36cc42 100644 --- a/pkg/cloudprovider/provider/gce/provider_test.go +++ b/pkg/cloudprovider/provider/gce/provider_test.go @@ -161,7 +161,7 @@ func TestValidate(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - err := p.Validate(test.mspec) + err := p.Validate(context.Background(), test.mspec) if (err != nil) != test.expectErr { t.Fatalf("expectedErr: %t, got: %v", test.expectErr, err) } diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 0db5fe0e5..076514949 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -184,7 +184,7 @@ func (p *provider) getServerPlacementGroup(ctx context.Context, client *hcloud.C return createdPg.PlacementGroup, nil } -func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -199,7 +199,6 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, err) } - ctx := context.TODO() client := getClient(c.Token) if c.Location != "" && c.Datacenter != "" { @@ -247,7 +246,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -256,7 +255,6 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert } } - ctx := context.TODO() client := getClient(c.Token) if c.Image == "" { @@ -388,8 +386,8 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return &hetznerServer{server: serverCreateRes.Server}, nil } -func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - instance, err := p.Get(machine, data) +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { + instance, err := p.Get(ctx, machine, data) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return true, nil @@ -405,7 +403,6 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider } } - ctx := context.TODO() client := getClient(c.Token) hzServer := instance.(*hetznerServer).server @@ -443,7 +440,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, nil } -func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -452,7 +449,6 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P } } - ctx := context.TODO() client := getClient(c.Token) servers, _, err := client.Server.List(ctx, hcloud.ServerListOpts{ListOpts: hcloud.ListOpts{ @@ -471,10 +467,7 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - +func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 8cd0fd4b6..b31ee13cf 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -327,7 +327,7 @@ func getNamespace() string { return ns } -func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -339,7 +339,6 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P if err != nil { return nil, fmt.Errorf("failed to get kubevirt client: %w", err) } - ctx := context.Background() virtualMachine := &kubevirtv1.VirtualMachine{} if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: machine.Name}, virtualMachine); err != nil { @@ -384,11 +383,11 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P // We don't use the UID for kubevirt because the name of a VMI must stay stable // in order for the node name to stay stable. The operator is responsible for ensuring // there are no conflicts, e.G. by using one Namespace per Kubevirt user cluster. -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, uid types.UID) error { +func (p *provider) MigrateUID(_ context.Context, _ *clusterv1alpha1.Machine, _ types.UID) error { return nil } -func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -414,7 +413,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { } // Check if we can reach the API of the target cluster. vmi := &kubevirtv1.VirtualMachineInstance{} - if err := sigClient.Get(context.Background(), types.NamespacedName{Namespace: c.Namespace, Name: "not-expected-to-exist"}, vmi); err != nil && !kerrors.IsNotFound(err) { + if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: "not-expected-to-exist"}, vmi); err != nil && !kerrors.IsNotFound(err) { return fmt.Errorf("failed to request VirtualMachineInstances: %w", err) } @@ -451,7 +450,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -469,7 +468,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert resourceRequirements := kubevirtv1.ResourceRequirements{} labels := map[string]string{"kubevirt.io/vm": machine.Name} // Add a common label to all VirtualMachines spawned by the same MachineDeployment (= MachineDeployment name). - if mdName, err := controllerutil.GetMachineDeploymentNameForMachine(context.Background(), machine, data.Client); err == nil { + if mdName, err := controllerutil.GetMachineDeploymentNameForMachine(ctx, machine, data.Client); err == nil { labels[machineDeploymentLabelKey] = mdName } @@ -477,7 +476,6 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert if err != nil { return nil, fmt.Errorf("failed to get kubevirt client: %w", err) } - ctx := context.Background() // Add VMIPreset label if specified if c.FlavorName != "" { @@ -572,7 +570,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return &kubeVirtServer{}, nil } -func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, cloudprovidererrors.TerminalError{ @@ -584,7 +582,6 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertyp if err != nil { return false, fmt.Errorf("failed to get kubevirt client: %w", err) } - ctx := context.Background() vm := &kubevirtv1.VirtualMachine{} if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: machine.Name}, vm); err != nil { diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index 7389756b2..2738c7d01 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -94,12 +94,12 @@ func getSlugForOS(os providerconfigtypes.OperatingSystem) (string, error) { return "", providerconfigtypes.ErrOSNotSupported } -func getClient(token string) linodego.Client { +func getClient(ctx context.Context, token string) linodego.Client { tokenSource := &TokenSource{ AccessToken: token, } - oauthClient := oauth2.NewClient(context.Background(), tokenSource) + oauthClient := oauth2.NewClient(ctx, tokenSource) client := linodego.NewClient(oauthClient) client.SetUserAgent("Kubermatic linodego") @@ -163,7 +163,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, nil } -func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -186,8 +186,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return fmt.Errorf("invalid operating system specified %q: %w", pc.OperatingSystem, err) } - ctx := context.TODO() - client := getClient(c.Token) + client := getClient(ctx, c.Token) _, err = client.GetRegion(ctx, c.Region) if err != nil { @@ -212,7 +211,7 @@ func createRandomPassword() (string, error) { return rootPass, nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -221,8 +220,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert } } - ctx := context.TODO() - client := getClient(c.Token) + client := getClient(ctx, c.Token) sshkey, err := ssh.NewKey() if err != nil { @@ -274,8 +272,8 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return &linodeInstance{linode: linode}, err } -func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - instance, err := p.Get(machine, data) +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { + instance, err := p.Get(ctx, machine, data) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return true, nil @@ -290,8 +288,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), } } - ctx := context.TODO() - client := getClient(c.Token) + client := getClient(ctx, c.Token) linodeID, err := strconv.Atoi(instance.ID()) if err != nil { @@ -315,7 +312,7 @@ func getListOptions(name string) *linodego.ListOptions { return listOptions } -func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -324,8 +321,7 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P } } - ctx := context.TODO() - client := getClient(c.Token) + client := getClient(ctx, c.Token) listOptions := getListOptions(machine.Spec.Name) linodes, err := client.ListInstances(ctx, listOptions) @@ -343,15 +339,12 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - +func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to decode providerconfig: %w", err) } - client := getClient(c.Token) + client := getClient(ctx, c.Token) listOptions := getListOptions(machine.Spec.Name) linodes, err := client.ListInstances(ctx, listOptions) if err != nil { diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 6a7cdfe68..5c6d7ea31 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -17,6 +17,7 @@ limitations under the License. package nutanix import ( + "context" "errors" "fmt" "strconv" @@ -195,7 +196,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, nil } -func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { config, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse machineSpec: %w", err) @@ -241,10 +242,10 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { vm, err := p.create(machine, userdata) if err != nil { - _, cleanupErr := p.Cleanup(machine, data) + _, cleanupErr := p.Cleanup(ctx, machine, data) if cleanupErr != nil { return nil, fmt.Errorf("cleaning up failed with err %v after creation failed with err %w", cleanupErr, err) } @@ -273,7 +274,7 @@ func (p *provider) create(machine *clusterv1alpha1.Machine, userdata string) (in return createVM(client, machine.Name, *config, pc.OperatingSystem, userdata) } -func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { return p.cleanup(machine, data) } @@ -338,7 +339,7 @@ func (p *provider) cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider return true, nil } -func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -403,7 +404,7 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertype }, nil } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, uid ktypes.UID) error { +func (p *provider) MigrateUID(_ context.Context, _ *clusterv1alpha1.Machine, _ ktypes.UID) error { return nil } diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 16deef23d..8b591c1c6 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -17,6 +17,7 @@ limitations under the License. package openstack import ( + "context" "encoding/json" "errors" "fmt" @@ -440,7 +441,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, nil } -func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { c, pc, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -556,7 +557,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { cfg, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -725,13 +726,13 @@ func deleteInstanceDueToFatalLogged(computeClient *gophercloud.ServiceClient, se klog.V(0).Infof("Instance %s got deleted", serverID) } -func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { var hasFloatingIPReleaseFinalizer bool if finalizers := sets.NewString(machine.Finalizers...); finalizers.Has(floatingIPReleaseFinalizer) { hasFloatingIPReleaseFinalizer = true } - instance, err := p.Get(machine, data) + instance, err := p.Get(ctx, machine, data) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { if hasFloatingIPReleaseFinalizer { @@ -773,7 +774,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider return false, nil } -func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -816,7 +817,7 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.P return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index 249815af6..672aa7635 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -295,7 +295,7 @@ func TestCreateServer(t *testing.T) { // It only verifies that the content of the create request matches // the expectation // TODO(irozzo) check the returned instance too - _, err := p.Create(m, tt.data, tt.userdata) + _, err := p.Create(context.Background(), m, tt.data, tt.userdata) if (err != nil) != tt.wantErr { t.Errorf("provider.Create() or = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go index 4e66e32a3..cb69420ce 100644 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ b/pkg/cloudprovider/provider/scaleway/provider.go @@ -139,7 +139,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, nil } -func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -172,7 +172,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { return nil } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (cloudInstance.Instance, error) { +func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (cloudInstance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -181,7 +181,6 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert } } - ctx := context.TODO() api, err := c.getInstanceAPI() if err != nil { return nil, err @@ -221,7 +220,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return &scwServer{server: serverResp.Server}, err } -func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { i, err := p.get(machine) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { @@ -237,7 +236,6 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertyp Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), } } - ctx := context.TODO() api, err := c.getInstanceAPI() if err != nil { return false, err @@ -254,7 +252,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, _ *cloudprovidertyp return false, nil } -func (p *provider) Get(machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (cloudInstance.Instance, error) { +func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (cloudInstance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -319,7 +317,7 @@ func (p *provider) get(machine *clusterv1alpha1.Machine) (*scwServer, error) { return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to decode providerconfig: %w", err) diff --git a/pkg/cloudprovider/provider/vsphere/client.go b/pkg/cloudprovider/provider/vsphere/client.go index 1d2d00b1d..f89f9c0e0 100644 --- a/pkg/cloudprovider/provider/vsphere/client.go +++ b/pkg/cloudprovider/provider/vsphere/client.go @@ -78,8 +78,8 @@ func NewSession(ctx context.Context, config *Config) (*Session, error) { } // Logout closes the idling vCenter connections. -func (s *Session) Logout() { - if err := s.Client.Logout(context.Background()); err != nil { +func (s *Session) Logout(ctx context.Context) { + if err := s.Client.Logout(ctx); err != nil { utilruntime.HandleError(fmt.Errorf("vsphere client failed to logout: %w", err)) } } diff --git a/pkg/cloudprovider/provider/vsphere/helper_test.go b/pkg/cloudprovider/provider/vsphere/helper_test.go index 8bdad74ad..359d867e7 100644 --- a/pkg/cloudprovider/provider/vsphere/helper_test.go +++ b/pkg/cloudprovider/provider/vsphere/helper_test.go @@ -98,7 +98,7 @@ func TestResolveDatastoreRef(t *testing.T) { tt.config.Datacenter = "DC0" session, err := NewSession(ctx, tt.config) - defer session.Logout() + defer session.Logout(ctx) if err != nil { t.Fatalf("error creating session: %v", err) } @@ -216,7 +216,7 @@ func TestResolveResourcePoolRef(t *testing.T) { tt.config.Datacenter = "DC0" session, err := NewSession(ctx, tt.config) - defer session.Logout() + defer session.Logout(ctx) if err != nil { t.Fatalf("error creating session: %v", err) } diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 09c49b2c1..db36990aa 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -199,9 +199,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return &c, pconfig, rawConfig, nil } -func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() +func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { config, pc, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to get config: %w", err) @@ -211,7 +209,7 @@ func (p *provider) Validate(spec clusterv1alpha1.MachineSpec) error { if err != nil { return fmt.Errorf("failed to create vCenter session: %w", err) } - defer session.Logout() + defer session.Logout(ctx) if config.Tags != nil { restAPISession, err := NewRESTSession(ctx, config) @@ -294,10 +292,10 @@ func machineInvalidConfigurationTerminalError(err error) error { } } -func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - vm, err := p.create(machine, userdata) +func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { + vm, err := p.create(ctx, machine, userdata) if err != nil { - _, cleanupErr := p.Cleanup(machine, data) + _, cleanupErr := p.Cleanup(ctx, machine, data) if cleanupErr != nil { return nil, fmt.Errorf("cleaning up failed with err %v after creation failed with err %w", cleanupErr, err) } @@ -306,9 +304,7 @@ func (p *provider) Create(machine *clusterv1alpha1.Machine, data *cloudprovidert return vm, nil } -func (p *provider) create(machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { - ctx := context.Background() - +func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("failed to parse config: %w", err) @@ -318,7 +314,7 @@ func (p *provider) create(machine *clusterv1alpha1.Machine, userdata string) (in if err != nil { return nil, fmt.Errorf("failed to create vCenter session: %w", err) } - defer session.Logout() + defer session.Logout(ctx) var containerLinuxUserdata string if pc.OperatingSystem == providerconfigtypes.OperatingSystemFlatcar { @@ -378,10 +374,7 @@ func (p *provider) create(machine *clusterv1alpha1.Machine, userdata string) (in return Server{name: virtualMachine.Name(), status: instance.StatusRunning, id: virtualMachine.Reference().Value}, nil } -func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, fmt.Errorf("failed to parse config: %w", err) @@ -391,7 +384,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider if err != nil { return false, fmt.Errorf("failed to create vCenter session: %w", err) } - defer session.Logout() + defer session.Logout(ctx) virtualMachine, err := p.get(ctx, config.Folder, machine.Spec, session.Finder) if err != nil { @@ -475,9 +468,7 @@ func (p *provider) Cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider return true, nil } -func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { - ctx := context.Background() - +func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("failed to parse config: %w", err) @@ -487,7 +478,7 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertype if err != nil { return nil, fmt.Errorf("failed to create vCenter session: %w", err) } - defer session.Logout() + defer session.Logout(ctx) virtualMachine, err := p.get(ctx, config.Folder, machine.Spec, session.Finder) if err != nil { @@ -541,7 +532,7 @@ func (p *provider) Get(machine *clusterv1alpha1.Machine, data *cloudprovidertype return Server{name: virtualMachine.Name(), status: instance.StatusRunning, addresses: addresses, id: virtualMachine.Reference().Value}, nil } -func (p *provider) MigrateUID(machine *clusterv1alpha1.Machine, uid ktypes.UID) error { +func (p *provider) MigrateUID(_ context.Context, _ *clusterv1alpha1.Machine, _ ktypes.UID) error { return nil } diff --git a/pkg/cloudprovider/provider/vsphere/provider_test.go b/pkg/cloudprovider/provider/vsphere/provider_test.go index 3987483e0..481d030e7 100644 --- a/pkg/cloudprovider/provider/vsphere/provider_test.go +++ b/pkg/cloudprovider/provider/vsphere/provider_test.go @@ -178,7 +178,7 @@ func TestValidate(t *testing.T) { tt.args.URL = vSphereURL m := cloudprovidertesting.Creator{Name: "test", Namespace: "vsphere", ProviderSpecGetter: tt.args.rawProviderSpec}. CreateMachine(t) - if err := p.Validate(m.Spec); (err != nil) != tt.wantErr { + if err := p.Validate(context.Background(), m.Spec); (err != nil) != tt.wantErr { t.Errorf("provider.Validate() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/pkg/cloudprovider/types/types.go b/pkg/cloudprovider/types/types.go index 5dc22fb89..adc9010b1 100644 --- a/pkg/cloudprovider/types/types.go +++ b/pkg/cloudprovider/types/types.go @@ -37,7 +37,7 @@ type Provider interface { // // In case of any error a "terminal" error should be set, // See v1alpha1.MachineStatus for more info - Validate(machinespec clusterv1alpha1.MachineSpec) error + Validate(ctx context.Context, machinespec clusterv1alpha1.MachineSpec) error // Get gets a node that is associated with the given machine. // @@ -46,19 +46,19 @@ type Provider interface { // See v1alpha1.MachineStatus for more info and TerminalError type // // In case the instance cannot be found, github.com/kubermatic/machine-controller/pkg/cloudprovider/errors/ErrInstanceNotFound will be returned - Get(machine *clusterv1alpha1.Machine, data *ProviderData) (instance.Instance, error) + Get(ctx context.Context, machine *clusterv1alpha1.Machine, data *ProviderData) (instance.Instance, error) // GetCloudConfig will return the cloud provider specific cloud-config, which gets consumed by the kubelet GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) // Create creates a cloud instance according to the given machine - Create(machine *clusterv1alpha1.Machine, data *ProviderData, userdata string) (instance.Instance, error) + Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *ProviderData, userdata string) (instance.Instance, error) // Cleanup will delete the instance associated with the machine and all associated resources. // If all resources have been cleaned up, true will be returned. // In case the cleanup involves asynchronous deletion of resources & those resources are not gone yet, // false should be returned. This is to indicate that the cleanup is not done, but needs to be called again at a later point - Cleanup(machine *clusterv1alpha1.Machine, data *ProviderData) (bool, error) + Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *ProviderData) (bool, error) // MachineMetricsLabels returns labels used for the Prometheus metrics // about created machines, e.g. instance type, instance size, region @@ -69,7 +69,7 @@ type Provider interface { // MigrateUID is called when the controller migrates types and the UID of the machine object changes // All cloud providers that use Machine.UID to uniquely identify resources must implement this - MigrateUID(machine *clusterv1alpha1.Machine, uid types.UID) error + MigrateUID(ctx context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error // SetMetricsForMachines allows providers to provide provider-specific metrics. This may be implemented // as no-op diff --git a/pkg/cloudprovider/validationwrapper.go b/pkg/cloudprovider/validationwrapper.go index 6cd7c45f5..965e2205b 100644 --- a/pkg/cloudprovider/validationwrapper.go +++ b/pkg/cloudprovider/validationwrapper.go @@ -17,6 +17,7 @@ limitations under the License. package cloudprovider import ( + "context" "fmt" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -43,7 +44,7 @@ func (w *cachingValidationWrapper) AddDefaults(spec v1alpha1.MachineSpec) (v1alp // Validate tries to get the validation result from the cache and if not found, calls the // cloudproviders Validate and saves that to the cache. -func (w *cachingValidationWrapper) Validate(spec v1alpha1.MachineSpec) error { +func (w *cachingValidationWrapper) Validate(ctx context.Context, spec v1alpha1.MachineSpec) error { result, exists, err := cache.Get(spec) if err != nil { return fmt.Errorf("error getting validation result from cache: %w", err) @@ -54,7 +55,7 @@ func (w *cachingValidationWrapper) Validate(spec v1alpha1.MachineSpec) error { } klog.V(6).Infof("Got cache miss for validation") - err = w.actualProvider.Validate(spec) + err = w.actualProvider.Validate(ctx, spec) if err := cache.Set(spec, err); err != nil { return fmt.Errorf("failed to set cache after validation: %w", err) } @@ -63,8 +64,8 @@ func (w *cachingValidationWrapper) Validate(spec v1alpha1.MachineSpec) error { } // Get just calls the underlying cloudproviders Get. -func (w *cachingValidationWrapper) Get(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { - return w.actualProvider.Get(machine, data) +func (w *cachingValidationWrapper) Get(ctx context.Context, machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { + return w.actualProvider.Get(ctx, machine, data) } // GetCloudConfig just calls the underlying cloudproviders GetCloudConfig. @@ -73,18 +74,18 @@ func (w *cachingValidationWrapper) GetCloudConfig(spec v1alpha1.MachineSpec) (st } // Create just calls the underlying cloudproviders Create. -func (w *cachingValidationWrapper) Create(machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - return w.actualProvider.Create(machine, data, userdata) +func (w *cachingValidationWrapper) Create(ctx context.Context, machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { + return w.actualProvider.Create(ctx, machine, data, userdata) } // Cleanup just calls the underlying cloudproviders Cleanup. -func (w *cachingValidationWrapper) Cleanup(m *v1alpha1.Machine, mcd *cloudprovidertypes.ProviderData) (bool, error) { - return w.actualProvider.Cleanup(m, mcd) +func (w *cachingValidationWrapper) Cleanup(ctx context.Context, m *v1alpha1.Machine, mcd *cloudprovidertypes.ProviderData) (bool, error) { + return w.actualProvider.Cleanup(ctx, m, mcd) } // MigrateUID just calls the underlying cloudproviders MigrateUID. -func (w *cachingValidationWrapper) MigrateUID(m *v1alpha1.Machine, newUID types.UID) error { - return w.actualProvider.MigrateUID(m, newUID) +func (w *cachingValidationWrapper) MigrateUID(ctx context.Context, m *v1alpha1.Machine, newUID types.UID) error { + return w.actualProvider.MigrateUID(ctx, m, newUID) } // MachineMetricsLabels just calls the underlying cloudproviders MachineMetricsLabels. diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index fda75629a..3eb909bcd 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -338,13 +338,13 @@ func (r *Reconciler) updateMachineErrorIfTerminalError(machine *clusterv1alpha1. return fmt.Errorf("%s, due to %w", errMsg, err) } -func (r *Reconciler) createProviderInstance(prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { +func (r *Reconciler) createProviderInstance(ctx context.Context, prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { // Ensure finalizer is there. _, err := r.ensureDeleteFinalizerExists(machine) if err != nil { return nil, fmt.Errorf("failed to add %q finalizer: %w", FinalizerDeleteInstance, err) } - i, err := prov.Create(machine, r.providerData, userdata) + i, err := prov.Create(ctx, machine, r.providerData, userdata) if err != nil { return nil, err } @@ -573,7 +573,7 @@ func (r *Reconciler) deleteMachine(ctx context.Context, prov cloudprovidertypes. return &reconcile.Result{RequeueAfter: 10 * time.Second}, nil } - if result, err := r.deleteCloudProviderInstance(prov, machine); result != nil || err != nil { + if result, err := r.deleteCloudProviderInstance(ctx, prov, machine); result != nil || err != nil { return result, err } @@ -634,14 +634,14 @@ func (r *Reconciler) retrieveNodesRelatedToMachine(ctx context.Context, machine return nodes, nil } -func (r *Reconciler) deleteCloudProviderInstance(prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine) (*reconcile.Result, error) { +func (r *Reconciler) deleteCloudProviderInstance(ctx context.Context, prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine) (*reconcile.Result, error) { finalizers := sets.NewString(machine.Finalizers...) if !finalizers.Has(FinalizerDeleteInstance) { return nil, nil } // Delete the instance - completelyGone, err := prov.Cleanup(machine, r.providerData) + completelyGone, err := prov.Cleanup(ctx, machine, r.providerData) if err != nil { message := fmt.Sprintf("%v. Please manually delete %s finalizer from the machine object.", err, FinalizerDeleteInstance) return nil, r.updateMachineErrorIfTerminalError(machine, common.DeleteMachineError, message, err, "failed to delete machine at cloud provider") @@ -673,7 +673,7 @@ func (r *Reconciler) deleteCloudProviderInstance(prov cloudprovidertypes.Provide } if rhelConfig.RHSMOfflineToken != "" { - if err := r.redhatSubscriptionManager.UnregisterInstance(rhelConfig.RHSMOfflineToken, machineName); err != nil { + if err := r.redhatSubscriptionManager.UnregisterInstance(ctx, rhelConfig.RHSMOfflineToken, machineName); err != nil { return nil, fmt.Errorf("failed to delete subscription for machine name %s: %w", machine.Name, err) } } @@ -733,7 +733,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( ) (*reconcile.Result, error) { klog.V(6).Infof("Requesting instance for machine '%s' from cloudprovider because no associated node with status ready found...", machine.Name) - providerInstance, err := prov.Get(machine, r.providerData) + providerInstance, err := prov.Get(ctx, machine, r.providerData) // case 2: retrieving instance from provider was not successful if err != nil { @@ -840,7 +840,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( } // Create the instance - if _, err = r.createProviderInstance(prov, machine, userdata); err != nil { + if _, err = r.createProviderInstance(ctx, prov, machine, userdata); err != nil { message := fmt.Sprintf("%v. Unable to create a machine.", err) return nil, r.updateMachineErrorIfTerminalError(machine, common.CreateMachineError, message, err, "failed to create machine at cloudprovider") } diff --git a/pkg/rhsm/subscription_manager.go b/pkg/rhsm/subscription_manager.go index 93a367535..90f6b9181 100644 --- a/pkg/rhsm/subscription_manager.go +++ b/pkg/rhsm/subscription_manager.go @@ -34,8 +34,7 @@ const defaultTimeout = 10 * time.Second // RedHatSubscriptionManager is responsible for removing redhat subscriptions. type RedHatSubscriptionManager interface { - //TODO(irozzo) add context in input to give more control to the caller - UnregisterInstance(offlineToken, machineName string) error + UnregisterInstance(ctx context.Context, offlineToken, machineName string) error } type pagination struct { @@ -70,8 +69,7 @@ func NewRedHatSubscriptionManager() RedHatSubscriptionManager { } } -func newOAuthClientWithRefreshToken(refreshToken string, tokenURL string) *http.Client { - ctx := context.Background() +func newOAuthClientWithRefreshToken(ctx context.Context, refreshToken string, tokenURL string) *http.Client { // Use the custom HTTP client when requesting an access token in order to // set a timeout value. // See: https://github.com/golang/oauth2/blob/c85d3e98c914e3a33234ad863dcbff5dbc425bb8/internal/token.go#L232 @@ -92,9 +90,7 @@ func newOAuthClientWithRefreshToken(refreshToken string, tokenURL string) *http. return c } -func (d *defaultRedHatSubscriptionManager) UnregisterInstance(offlineToken, machineName string) error { - ctx := context.Background() - +func (d *defaultRedHatSubscriptionManager) UnregisterInstance(ctx context.Context, offlineToken, machineName string) error { var ( retries = 0 maxRetries = 15 @@ -151,7 +147,7 @@ func (d *defaultRedHatSubscriptionManager) findSystemsProfile(ctx context.Contex } func (d *defaultRedHatSubscriptionManager) deleteSubscription(ctx context.Context, uuid, offlineToken string) error { - client := newOAuthClientWithRefreshToken(offlineToken, d.authURL) + client := newOAuthClientWithRefreshToken(ctx, offlineToken, d.authURL) req, err := http.NewRequest("DELETE", fmt.Sprintf("%s/%s", d.apiURL, uuid), nil) if err != nil { return fmt.Errorf("failed to create delete system request: %w", err) @@ -180,7 +176,7 @@ func (d *defaultRedHatSubscriptionManager) deleteSubscription(ctx context.Contex } func (d *defaultRedHatSubscriptionManager) executeFindSystemsRequest(ctx context.Context, offlineToken string, offset int) (*systemsResponse, error) { - client := newOAuthClientWithRefreshToken(offlineToken, d.authURL) + client := newOAuthClientWithRefreshToken(ctx, offlineToken, d.authURL) req, err := http.NewRequest("GET", fmt.Sprintf(d.apiURL+"?limit=%v&offset=%v", d.requestsLimiter, offset), nil) if err != nil { return nil, fmt.Errorf("failed to create fetch systems request: %w", err) diff --git a/pkg/rhsm/subscription_manager_test.go b/pkg/rhsm/subscription_manager_test.go index a95b4d57e..e081401fd 100644 --- a/pkg/rhsm/subscription_manager_test.go +++ b/pkg/rhsm/subscription_manager_test.go @@ -17,6 +17,7 @@ limitations under the License. package rhsm import ( + "context" "fmt" "net/http" "net/http/httptest" @@ -62,7 +63,7 @@ func TestDefaultRedHatSubscriptionManager_UnregisterInstance(t *testing.T) { manager.(*defaultRedHatSubscriptionManager).authURL = tt.testingServer.URL manager.(*defaultRedHatSubscriptionManager).requestsLimiter = tt.requestLimiter - if err := manager.UnregisterInstance(tt.offlineToken, tt.machineName); err != nil { + if err := manager.UnregisterInstance(context.Background(), tt.offlineToken, tt.machineName); err != nil { t.Fatalf("failed executing test: %v", err) } }) diff --git a/pkg/signals/signal.go b/pkg/signals/signal.go deleted file mode 100644 index 393a285fc..000000000 --- a/pkg/signals/signal.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package signals - -import ( - "os" - "os/signal" - "syscall" -) - -var ( - onlyOneSignalHandler = make(chan struct{}) - shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} -) - -// SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned -// which is closed on one of these signals. If a second signal is caught, the program -// is terminated with exit code 1. -func SetupSignalHandler() (stopCh <-chan struct{}) { - close(onlyOneSignalHandler) // panics when called twice - - stop := make(chan struct{}) - c := make(chan os.Signal, 2) - signal.Notify(c, shutdownSignals...) - go func() { - <-c - close(stop) - <-c - os.Exit(1) // second signal. Exit directly. - }() - - return stop -} diff --git a/test/e2e/provisioning/migrateuidscenario.go b/test/e2e/provisioning/migrateuidscenario.go index 96579ff0a..66e3537ad 100644 --- a/test/e2e/provisioning/migrateuidscenario.go +++ b/test/e2e/provisioning/migrateuidscenario.go @@ -87,10 +87,12 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time } machine.Spec = defaultedSpec + ctx := context.Background() + // Step 0: Create instance with old UID maxTries := 15 for i := 0; i < maxTries; i++ { - _, err := prov.Get(machine, providerData) + _, err := prov.Get(ctx, machine, providerData) if err != nil { if !errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { if i < maxTries-1 { @@ -100,7 +102,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time } return fmt.Errorf("failed to get machine %s before creating it: %w", machine.Name, err) } - _, err := prov.Create(machine, providerData, "#cloud-config\n") + _, err := prov.Create(ctx, machine, providerData, "#cloud-config\n") if err != nil { if i < maxTries-1 { time.Sleep(10 * time.Second) @@ -115,7 +117,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time // Step 1: Verify we can successfully get the instance for i := 0; i < maxTries; i++ { - if _, err := prov.Get(machine, providerData); err != nil { + if _, err := prov.Get(ctx, machine, providerData); err != nil { if i < maxTries-1 { klog.V(4).Infof("failed to get instance for machine %s before migrating on try %v with err=%v, will retry", machine.Name, i, err) time.Sleep(10 * time.Second) @@ -128,7 +130,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time // Step 2: Migrate UID for i := 0; i < maxTries; i++ { - if err := prov.MigrateUID(machine, newUID); err != nil { + if err := prov.MigrateUID(ctx, machine, newUID); err != nil { if i < maxTries-1 { time.Sleep(10 * time.Second) klog.V(4).Infof("failed to migrate UID for machine %s on try %v with err=%v, will retry", machine.Name, i, err) @@ -142,7 +144,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time // Step 3: Verify we can successfully get the instance with the new UID for i := 0; i < maxTries; i++ { - if _, err := prov.Get(machine, providerData); err != nil { + if _, err := prov.Get(ctx, machine, providerData); err != nil { if i < maxTries-1 { time.Sleep(10 * time.Second) klog.V(4).Infof("failed to get instance for machine %s after migrating on try %v with err=%v, will retry", machine.Name, i, err) @@ -156,7 +158,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time // Step 4: Delete the instance and then verify instance is gone for i := 0; i < maxTries; i++ { // Deletion part 0: Delete and continue on err if there are tries left - done, err := prov.Cleanup(machine, providerData) + done, err := prov.Cleanup(ctx, machine, providerData) if err != nil { if i < maxTries-1 { klog.V(4).Infof("Failed to delete machine %s on try %v with err=%v, will retry", machine.Name, i, err) @@ -172,7 +174,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time } // Deletion part 1: Get and continue if err != cloudprovidererrors.ErrInstanceNotFound if there are tries left - _, err = prov.Get(machine, providerData) + _, err = prov.Get(ctx, machine, providerData) if err != nil && errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { break } From b426994de086d0ee6be5cb26169e63a0d6f7ba05 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 18 May 2022 15:03:10 +0500 Subject: [PATCH 148/489] Disable e2e tests for Nutanix (#1295) Signed-off-by: Waleed Malik --- .prow/provider-nutanix.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 234dd205e..8e390ec77 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -16,7 +16,8 @@ presubmits: - name: pull-machine-controller-e2e-nutanix optional: true always_run: false - run_if_changed: "(pkg/cloudprovider/provider/nutanix/)" + # TODO uncomment this when Nutanix is in a working condition + #run_if_changed: "(pkg/cloudprovider/provider/nutanix/)" decorate: true error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" From 8a81fef9b77e8cd0d7b0b62b95e548dbba653e24 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 20 May 2022 14:10:57 +0500 Subject: [PATCH 149/489] Update package repo path for buildah (#1302) Signed-off-by: Waleed Malik --- test/tools/integration/master_install_script.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tools/integration/master_install_script.sh b/test/tools/integration/master_install_script.sh index 16205b047..3b295d7b3 100644 --- a/test/tools/integration/master_install_script.sh +++ b/test/tools/integration/master_install_script.sh @@ -27,8 +27,8 @@ systemctl mask swap.target swapoff -a if ! which buildah; then - sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list" - wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_20.04/Release.key -O Release.key + sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/testing/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:testing.list" + wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:testing/xUbuntu_20.04/Release.key -O Release.key apt-key add - < Release.key apt-get update apt-get -y install buildah From a04da9b51b7b470433f98c3a8649522fe8249ef2 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 23 May 2022 13:57:34 +0500 Subject: [PATCH 150/489] Refactor handling of kubeconfig for kubevirt (#1301) Signed-off-by: Waleed Malik --- examples/kubevirt-machinedeployment.yaml | 6 ++-- .../provider/kubevirt/provider.go | 35 +++++++++++++++---- test/e2e/provisioning/all_e2e_test.go | 2 +- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index 49bb904cd..a40b95703 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -28,7 +28,9 @@ spec: cloudProviderSpec: auth: kubeconfig: - value: '<< KUBECONFIG >>' + # Can also be set via the env var 'KUBEVIRT_KUBECONFIG' on the machine-controller + # If specified directly, this value should be a base64 encoded kubeconfig in either yaml or json format. + value: "<< KUBECONFIG >>" virtualMachine: template: cpus: "1" @@ -44,7 +46,7 @@ spec: type: "" # Allowed values: "", "soft", "hard" key: "foo" values: - - bar + - bar # Can also be `centos`, must align with he configured registryImage above operatingSystem: "ubuntu" operatingSystemSpec: diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index b31ee13cf..423a42909 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -18,6 +18,7 @@ package kubevirt import ( "context" + "encoding/base64" "errors" "fmt" "net/url" @@ -199,10 +200,36 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } config := Config{} - config.Kubeconfig, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Auth.Kubeconfig, "KUBEVIRT_KUBECONFIG") + + // Kubeconfig was specified directly in the Machine/MachineDeployment CR. In this case we need to ensure that the value is base64 encoded. + if rawConfig.Auth.Kubeconfig.Value != "" { + val, err := base64.StdEncoding.DecodeString(rawConfig.Auth.Kubeconfig.Value) + if err != nil { + // An error here means that this is not a valid base64 string + // We can be more explicit here with the error for visibility. Webhook will return this error if we hit this scenario. + return nil, nil, fmt.Errorf("failed to decode base64 encoded kubeconfig. Expected value is a base64 encoded Kubeconfig in JSON or YAML format: %w", err) + } + config.Kubeconfig = string(val) + } else { + // Environment variable or secret reference was used for providing the value of kubeconfig + // We have to be lenient in this case and allow unencoded values as well. + config.Kubeconfig, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Auth.Kubeconfig, "KUBEVIRT_KUBECONFIG") + if err != nil { + return nil, nil, fmt.Errorf(`failed to get value of "kubeconfig" field: %w`, err) + } + val, err := base64.StdEncoding.DecodeString(config.Kubeconfig) + // We intentionally ignore errors here with an assumption that an unencoded YAML or JSON must have been passed on + // in this case. + if err == nil { + config.Kubeconfig = string(val) + } + } + + config.RestConfig, err = clientcmd.RESTConfigFromKubeConfig([]byte(config.Kubeconfig)) if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "kubeconfig" field: %w`, err) + return nil, nil, fmt.Errorf("failed to decode kubeconfig: %w", err) } + config.CPUs, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.CPUs) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "cpus" field: %w`, err) @@ -232,10 +259,6 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "storageClassName" field: %w`, err) } - config.RestConfig, err = clientcmd.RESTConfigFromKubeConfig([]byte(config.Kubeconfig)) - if err != nil { - return nil, nil, fmt.Errorf("failed to decode kubeconfig: %w", err) - } config.FlavorName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Flavor.Name) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "flavor.name" field: %w`, err) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index d8ae02e6b..e725f54dd 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -281,7 +281,7 @@ func addCAToDeployment(ctx context.Context, client ctrlruntimeclient.Client, nam func TestKubevirtProvisioningE2E(t *testing.T) { t.Parallel() - kubevirtKubeconfig := os.Getenv("KUBEVIRT_E2E_TESTS_KUBECONFIG_JSON") + kubevirtKubeconfig := os.Getenv("KUBEVIRT_E2E_TESTS_KUBECONFIG") if kubevirtKubeconfig == "" { t.Fatalf("Unable to run kubevirt tests, KUBEVIRT_E2E_TESTS_KUBECONFIG must be set") From 62fa4bb9018635491c47a2cfb7af0a39fe92c4ea Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 23 May 2022 16:05:31 +0500 Subject: [PATCH 151/489] docs: include rocky linux in compatibility matrix (#1297) Signed-off-by: Waleed Malik --- docs/operating-system.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/operating-system.md b/docs/operating-system.md index df44f62f8..c9cbe7cda 100644 --- a/docs/operating-system.md +++ b/docs/operating-system.md @@ -4,15 +4,15 @@ ### Cloud provider -| | Ubuntu | CentOS | Flatcar | RHEL | SLES | Amazon Linux 2 | -|---|---|---|---|---|---|---| -| AWS | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Azure | ✓ | ✓ | ✓ | ✓ | x | x | -| Digitalocean | ✓ | ✓ | x | x | x | x | -| Google Cloud Platform | ✓ | x | x | x | x | x | -| Hetzner | ✓ | ✓ | x | x | x | x | -| Equinix Metal | ✓ | ✓ | x | x | x | x | -| Openstack | ✓ | ✓ | ✓ | ✓ | x | x | +| | Ubuntu | CentOS | Flatcar | RHEL | SLES | Amazon Linux 2 | Rocky Linux | +|---|---|---|---|---|---|---|---| +| AWS | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Azure | ✓ | ✓ | ✓ | ✓ | x | x | ✓ | +| Digitalocean | ✓ | ✓ | x | x | x | x | ✓ | +| Google Cloud Platform | ✓ | x | x | x | x | x | x | +| Hetzner | ✓ | ✓ | x | x | x | x | ✓ | +| Equinix Metal | ✓ | ✓ | x | x | x | x | x | +| Openstack | ✓ | ✓ | ✓ | ✓ | x | x | ✓ | ## Configuring a operating system @@ -21,6 +21,7 @@ Allowed values: - `amzn2` - `centos` - `rhel` +- `rockylinux` - `sles` - `ubuntu` @@ -36,6 +37,7 @@ Machine controller may work with other OS versions that are not listed in the ta | AmazonLinux2 | 2.x | | CentOS | 7.4.x, 7.6.x, 7.7.x | | RHEL | 8.0, 8.1 | +| Rocky Linux | 8.5 | | SLES | SLES 15 SP1 | | Ubuntu | 18.04 LTS | From ebbd117869da8a35867e74193510ef6241adb4e1 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 24 May 2022 14:55:19 +0500 Subject: [PATCH 152/489] Switch to Ubuntu 22.04 LTS for e2e tests (#1307) * Switch to Ubuntu 22.04 LTS for e2e tests Signed-off-by: Waleed Malik * Use containerd 1.5 for e2e tests Signed-off-by: Waleed Malik * Fix short-name resolution Signed-off-by: Waleed Malik * Fix debian frontend issue Signed-off-by: Waleed Malik --- Dockerfile | 2 +- test/tools/integration/hetzner.tf | 2 +- test/tools/integration/master_install_script.sh | 7 +++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index a059d4b38..9485e86db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,7 @@ # limitations under the License. ARG GO_VERSION=1.18.2 -FROM golang:${GO_VERSION} AS builder +FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . RUN make all diff --git a/test/tools/integration/hetzner.tf b/test/tools/integration/hetzner.tf index c87525e8d..d12a907a2 100644 --- a/test/tools/integration/hetzner.tf +++ b/test/tools/integration/hetzner.tf @@ -14,7 +14,7 @@ resource "hcloud_network" "net" { resource "hcloud_server" "machine-controller-test" { name = var.hcloud_test_server_name - image = "ubuntu-20.04" + image = "ubuntu-22.04" server_type = "cx21" ssh_keys = [hcloud_ssh_key.default.id] location = "nbg1" diff --git a/test/tools/integration/master_install_script.sh b/test/tools/integration/master_install_script.sh index 3b295d7b3..50b0e4cd9 100644 --- a/test/tools/integration/master_install_script.sh +++ b/test/tools/integration/master_install_script.sh @@ -22,14 +22,13 @@ echo "$LC_E2E_SSH_PUBKEY" >> .ssh/authorized_keys echo "GatewayPorts clientspecified" >> /etc/ssh/sshd_config systemctl restart sshd.service +export DEBIAN_FRONTEND=noninteractive + # Hetzner's Ubuntu Bionic comes with swap pre-configured, so we force it off. systemctl mask swap.target swapoff -a if ! which buildah; then - sh -c "echo 'deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/testing/xUbuntu_20.04/ /' > /etc/apt/sources.list.d/devel:kubic:libcontainers:testing.list" - wget -nv https://download.opensuse.org/repositories/devel:kubic:libcontainers:testing/xUbuntu_20.04/Release.key -O Release.key - apt-key add - < Release.key apt-get update apt-get -y install buildah fi @@ -54,7 +53,7 @@ EOF EnvironmentFile=-/etc/environment EOF - DEBIAN_FRONTEND=noninteractive apt-get install -y containerd.io=1.4* + DEBIAN_FRONTEND=noninteractive apt-get install -y containerd.io=1.5* apt-mark hold containerd.io mkdir -p /etc/containerd/ && touch /etc/containerd/config.toml From 10d35a0d7e247ba1796c8290c0d2be793984dd16 Mon Sep 17 00:00:00 2001 From: Moritz Bracht <682686+dermorz@users.noreply.github.com> Date: Tue, 24 May 2022 14:20:24 +0200 Subject: [PATCH 153/489] Support node registry credentials with docker (#1298) * Explicitly set root user in kubelet systemd unit Without setting this (at least on debian) $HOME is not set when running kubelet. Without $HOME being set the search paths for docker credentials are not as they would be expected. See: https://github.com/kubernetes/kubernetes/issues/45487#issuecomment-322217521 * Adjust testdata Adjust userdata kubelet test data More test data adjustments Adjust flatcar iginition testdata * Add docker credentials support for ubuntu Go template trimming Only write docker auth config if CR is docker and credentials are given * Add docker credential support to remaining distributions * Fix typo * Add support for SecretTypeDockerConfigJson * Add documentation for additional supported secret type * AuthConfig to return emptystring if registryCredentials is not set --- cmd/machine-controller/main.go | 2 +- docs/registry-authentication.md | 16 +- pkg/containerruntime/config.go | 17 +- pkg/containerruntime/containerd.go | 23 +- pkg/containerruntime/containerruntime.go | 18 + pkg/containerruntime/docker.go | 23 + pkg/userdata/amzn2/provider.go | 67 ++- .../testdata/kubelet-v1.21-aws-external.yaml | 1 + .../amzn2/testdata/kubelet-v1.21-aws.yaml | 1 + .../kubelet-v1.21-vsphere-mirrors.yaml | 1 + .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 1 + .../amzn2/testdata/kubelet-v1.21-vsphere.yaml | 1 + .../amzn2/testdata/kubelet-v1.22-aws.yaml | 1 + .../amzn2/testdata/kubelet-v1.23-aws.yaml | 1 + .../amzn2/testdata/kubelet-v1.24-aws.yaml | 1 + pkg/userdata/centos/provider.go | 67 ++- .../testdata/kubelet-v1.21-aws-external.yaml | 1 + .../centos/testdata/kubelet-v1.21-aws.yaml | 1 + .../kubelet-v1.21-vsphere-mirrors.yaml | 1 + .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 1 + .../testdata/kubelet-v1.21-vsphere.yaml | 1 + .../centos/testdata/kubelet-v1.22-aws.yaml | 1 + .../centos/testdata/kubelet-v1.23-aws.yaml | 1 + .../testdata/kubelet-v1.23-nutanix.yaml | 1 + .../centos/testdata/kubelet-v1.24-aws.yaml | 1 + pkg/userdata/flatcar/provider.go | 73 ++- .../flatcar/testdata/cloud-init_v1.21.10.yaml | 1 + .../flatcar/testdata/cloud-init_v1.22.7.yaml | 1 + .../flatcar/testdata/cloud-init_v1.23.5.yaml | 1 + .../flatcar/testdata/cloud-init_v1.24.0.yaml | 1 + pkg/userdata/flatcar/testdata/containerd.yaml | 1 + .../flatcar/testdata/ignition_v1.21.10.json | 2 +- .../flatcar/testdata/ignition_v1.22.7.json | 2 +- .../flatcar/testdata/ignition_v1.23.5.json | 2 +- .../flatcar/testdata/ignition_v1.24.0.json | 2 +- pkg/userdata/helper/kubelet.go | 1 + ...let_systemd_unit_cloud-provider-set.golden | 1 + ...t_systemd_unit_multiple-dns-servers.golden | 1 + ...kublet_systemd_unit_pause-image-set.golden | 1 + .../kublet_systemd_unit_taints-set.golden | 1 + ...temd_unit_version-v1.21.10-external.golden | 1 + ...ublet_systemd_unit_version-v1.21.10.golden | 1 + ...stemd_unit_version-v1.22.7-external.golden | 1 + ...kublet_systemd_unit_version-v1.22.7.golden | 1 + ...stemd_unit_version-v1.23.5-external.golden | 1 + ...kublet_systemd_unit_version-v1.23.5.golden | 1 + ...stemd_unit_version-v1.24.0-external.golden | 1 + ...kublet_systemd_unit_version-v1.24.0.golden | 1 + pkg/userdata/rhel/provider.go | 67 ++- .../rhel/testdata/kubelet-v1.21-aws.yaml | 1 + .../rhel/testdata/kubelet-v1.22-aws.yaml | 1 + .../rhel/testdata/kubelet-v1.22-nutanix.yaml | 1 + .../testdata/kubelet-v1.23-aws-external.yaml | 1 + .../rhel/testdata/kubelet-v1.23-aws.yaml | 1 + .../kubelet-v1.23-vsphere-mirrors.yaml | 1 + .../testdata/kubelet-v1.23-vsphere-proxy.yaml | 1 + .../rhel/testdata/kubelet-v1.23-vsphere.yaml | 1 + .../testdata/kubelet-v1.24-aws-external.yaml | 1 + .../rhel/testdata/kubelet-v1.24-aws.yaml | 1 + .../rhel/testdata/pod-cidr-azure-rhel.yaml | 1 + pkg/userdata/rockylinux/provider.go | 67 ++- .../testdata/kubelet-v1.21-aws-external.yaml | 1 + .../testdata/kubelet-v1.21-aws.yaml | 1 + .../kubelet-v1.21-vsphere-mirrors.yaml | 1 + .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 1 + .../testdata/kubelet-v1.21-vsphere.yaml | 1 + .../testdata/kubelet-v1.22-aws.yaml | 1 + .../testdata/kubelet-v1.23-aws.yaml | 1 + .../testdata/kubelet-v1.23-nutanix.yaml | 1 + .../testdata/kubelet-v1.24-aws.yaml | 1 + pkg/userdata/sles/provider.go | 63 ++- .../sles/testdata/dist-upgrade-on-boot.yaml | 1 + .../kubelet-version-without-v-prefix.yaml | 1 + .../sles/testdata/multiple-dns-servers.yaml | 1 + .../sles/testdata/multiple-ssh-keys.yaml | 1 + .../openstack-overwrite-cloud-config.yaml | 1 + pkg/userdata/sles/testdata/openstack.yaml | 1 + .../sles/testdata/version-1.21.10.yaml | 1 + .../sles/testdata/version-1.22.7.yaml | 1 + .../sles/testdata/version-1.23.5.yaml | 1 + .../sles/testdata/version-1.24.0.yaml | 1 + .../sles/testdata/vsphere-mirrors.yaml | 1 + pkg/userdata/sles/testdata/vsphere-proxy.yaml | 1 + pkg/userdata/sles/testdata/vsphere.yaml | 1 + pkg/userdata/ubuntu/provider.go | 68 ++- pkg/userdata/ubuntu/provider_test.go | 32 ++ pkg/userdata/ubuntu/testdata/containerd.yaml | 1 + .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 1 + pkg/userdata/ubuntu/testdata/docker.yaml | 461 ++++++++++++++++++ .../kubelet-version-without-v-prefix.yaml | 1 + .../ubuntu/testdata/multiple-dns-servers.yaml | 1 + .../ubuntu/testdata/multiple-ssh-keys.yaml | 1 + pkg/userdata/ubuntu/testdata/nutanix.yaml | 1 + .../openstack-overwrite-cloud-config.yaml | 1 + pkg/userdata/ubuntu/testdata/openstack.yaml | 1 + .../ubuntu/testdata/version-1.21.10.yaml | 1 + .../ubuntu/testdata/version-1.22.7.yaml | 1 + .../ubuntu/testdata/version-1.23.5.yaml | 1 + .../ubuntu/testdata/version-1.24.0.yaml | 1 + .../ubuntu/testdata/vsphere-mirrors.yaml | 1 + .../ubuntu/testdata/vsphere-proxy.yaml | 1 + pkg/userdata/ubuntu/testdata/vsphere.yaml | 1 + 102 files changed, 958 insertions(+), 197 deletions(-) create mode 100644 pkg/userdata/ubuntu/testdata/docker.yaml diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 6b946b34a..0410ce203 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -165,7 +165,7 @@ func main() { flag.BoolVar(&nodeCSRApprover, "node-csr-approver", true, "Enable NodeCSRApprover controller to automatically approve node serving certificate requests") flag.StringVar(&podCIDR, "pod-cidr", "172.25.0.0/16", "WARNING: flag is unused, kept only for backwards compatibility") flag.StringVar(&nodePortRange, "node-port-range", "30000-32767", "A port range to reserve for services with NodePort visibility") - flag.StringVar(&nodeRegistryCredentialsSecret, "node-registry-credentials-secret", "", "A Secret object reference, that containt auth info for image registry in namespace/secret-name form, example: kube-system/registry-credentials. See doc at https://github.com/kubermaric/machine-controller/blob/master/docs/registry-authentication.md") + flag.StringVar(&nodeRegistryCredentialsSecret, "node-registry-credentials-secret", "", "A Secret object reference, that contains auth info for image registry in namespace/secret-name form, example: kube-system/registry-credentials. See doc at https://github.com/kubermaric/machine-controller/blob/master/docs/registry-authentication.md") flag.BoolVar(&useOSM, "use-osm", false, "use osm controller for node bootstrap") flag.Parse() diff --git a/docs/registry-authentication.md b/docs/registry-authentication.md index c96d5f293..0cfbc152a 100644 --- a/docs/registry-authentication.md +++ b/docs/registry-authentication.md @@ -6,6 +6,11 @@ reference in form `namespace/secret-name` where authentication info will be stored. During the VM creation this info will be used to configure container runtime. +There are two options for the type of the secret that can be passed on this +flag. + +## Custom secret + Secret format is serialized `map[string]github.com/containerd/containerd/pkg/cri/config.AuthConfig`, where `AuthConfig` is defined as @@ -27,9 +32,7 @@ type AuthConfig struct { Original source: https://github.com/containerd/containerd/blob/v1.5.9/pkg/cri/config/config.go#L126-L137 - Example: - ```yaml apiVersion: v1 kind: Secret @@ -45,3 +48,12 @@ data: Now having this saved in the Kubernetes API, launch machine-controller with `-node-registry-credentials-secret=kube-system/my-registries` flag. + +## `kubernetes.io/dockerconfigjson` + +This type stores a serialized `~/.docker/config.json` and can directly be +created via `kubectl` by either passing such file directly or by providing +the necessary data. + +See also: +https://kubernetes.io/docs/concepts/configuration/secret/#docker-config-secrets diff --git a/pkg/containerruntime/config.go b/pkg/containerruntime/config.go index 13faf255f..551befbb6 100644 --- a/pkg/containerruntime/config.go +++ b/pkg/containerruntime/config.go @@ -93,12 +93,21 @@ func GetContainerdAuthConfig(ctx context.Context, client ctrlruntimeclient.Clien return nil, fmt.Errorf("failed to retrieve registry credentials secret object: %w", err) } - for registry, data := range credsSecret.Data { - var regCred AuthConfig - if err := json.Unmarshal(data, ®Cred); err != nil { + switch credsSecret.Type { + case corev1.SecretTypeDockerConfigJson: + var regCred DockerCfgJSON + if err := json.Unmarshal(credsSecret.Data[".dockerconfigjson"], ®Cred); err != nil { return nil, fmt.Errorf("failed to unmarshal registry credentials: %w", err) } - registryCredentials[registry] = regCred + registryCredentials = regCred.Auths + default: + for registry, data := range credsSecret.Data { + var regCred AuthConfig + if err := json.Unmarshal(data, ®Cred); err != nil { + return nil, fmt.Errorf("failed to unmarshal registry credentials: %w", err) + } + registryCredentials[registry] = regCred + } } } return registryCredentials, nil diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index ebf7a8c5a..413b0297f 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -43,6 +43,14 @@ func (eng *Containerd) ConfigFileName() string { return "/etc/containerd/config.toml" } +func (eng *Containerd) AuthConfig() (string, error) { + return "", nil +} + +func (eng *Containerd) AuthConfigFileName() string { + return "" +} + func (eng *Containerd) KubeletFlags() []string { return []string{ "--container-runtime=remote", @@ -226,21 +234,6 @@ type containerdRegistryTLSConfig struct { InsecureSkipVerify bool `toml:"insecure_skip_verify"` } -// AuthConfig is a COPY of github.com/containerd/containerd/pkg/cri/config.AuthConfig. -// AuthConfig contains the config related to authentication to a specific registry. -type AuthConfig struct { - // Username is the username to login the registry. - Username string `toml:"username,omitempty" json:"username,omitempty"` - // Password is the password to login the registry. - Password string `toml:"password,omitempty" json:"password,omitempty"` - // Auth is a base64 encoded string from the concatenation of the username, - // a colon, and the password. - Auth string `toml:"auth,omitempty" json:"auth,omitempty"` - // IdentityToken is used to authenticate the user and get - // an access token for the registry. - IdentityToken string `toml:"identitytoken,omitempty" json:"identitytoken,omitempty"` -} - func (eng *Containerd) Config() (string, error) { criPlugin := containerdCRIPlugin{ SandboxImage: eng.sandboxImage, diff --git a/pkg/containerruntime/containerruntime.go b/pkg/containerruntime/containerruntime.go index f8c23ca26..95eb55076 100644 --- a/pkg/containerruntime/containerruntime.go +++ b/pkg/containerruntime/containerruntime.go @@ -32,6 +32,8 @@ type Engine interface { ScriptFor(os types.OperatingSystem) (string, error) ConfigFileName() string Config() (string, error) + AuthConfigFileName() string + AuthConfig() (string, error) String() string } @@ -88,6 +90,21 @@ type Config struct { ContainerLogMaxSize string `json:",omitempty"` } +// AuthConfig is a COPY of github.com/containerd/containerd/pkg/cri/config.AuthConfig. +// AuthConfig contains the config related to authentication to a specific registry. +type AuthConfig struct { + // Username is the username to login the registry. + Username string `toml:"username,omitempty" json:"username,omitempty"` + // Password is the password to login the registry. + Password string `toml:"password,omitempty" json:"password,omitempty"` + // Auth is a base64 encoded string from the concatenation of the username, + // a colon, and the password. + Auth string `toml:"auth,omitempty" json:"auth,omitempty"` + // IdentityToken is used to authenticate the user and get + // an access token for the registry. + IdentityToken string `toml:"identitytoken,omitempty" json:"identitytoken,omitempty"` +} + func (cfg Config) String() string { switch { case cfg.Containerd != nil: @@ -105,6 +122,7 @@ func (cfg Config) Engine(kubeletVersion *semver.Version) Engine { registryMirrors: cfg.RegistryMirrors["docker.io"], containerLogMaxFiles: cfg.ContainerLogMaxFiles, containerLogMaxSize: cfg.ContainerLogMaxSize, + registryCredentials: cfg.RegistryCredentials, } containerd := &Containerd{ diff --git a/pkg/containerruntime/docker.go b/pkg/containerruntime/docker.go index d12dfa11e..dbbc1e58d 100644 --- a/pkg/containerruntime/docker.go +++ b/pkg/containerruntime/docker.go @@ -17,6 +17,7 @@ limitations under the License. package containerruntime import ( + "encoding/json" "fmt" "strings" "text/template" @@ -36,6 +37,11 @@ type Docker struct { registryMirrors []string containerLogMaxFiles string containerLogMaxSize string + registryCredentials map[string]AuthConfig +} + +type DockerCfgJSON struct { + Auths map[string]AuthConfig `json:"auths,omitempty"` } func (eng *Docker) Config() (string, error) { @@ -46,6 +52,23 @@ func (eng *Docker) ConfigFileName() string { return "/etc/docker/daemon.json" } +func (eng *Docker) AuthConfig() (string, error) { + if eng.registryCredentials == nil { + return "", nil + } + + cfg := DockerCfgJSON{ + Auths: eng.registryCredentials, + } + b, err := json.MarshalIndent(cfg, "", " ") + + return string(b), err +} + +func (eng *Docker) AuthConfigFileName() string { + return "/root/.docker/config.json" +} + func (eng *Docker) KubeletFlags() []string { return []string{ "--container-runtime=docker", diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index 70df9daac..b3e778aae 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -92,34 +92,43 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { return "", fmt.Errorf("failed to generate container runtime config: %w", err) } + crAuthConfig, err := crEngine.AuthConfig() + if err != nil { + return "", fmt.Errorf("failed to generate container runtime auth config: %w", err) + } + data := struct { plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - OSConfig *Config - KubeletVersion string - ServerAddr string - Kubeconfig string - KubernetesCACert string - NodeIPScript string - ExtraKubeletFlags []string - ContainerRuntimeScript string - ContainerRuntimeConfigFileName string - ContainerRuntimeConfig string - ContainerRuntimeName string + ProviderSpec *providerconfigtypes.Config + OSConfig *Config + KubeletVersion string + ServerAddr string + Kubeconfig string + KubernetesCACert string + NodeIPScript string + ExtraKubeletFlags []string + ContainerRuntimeScript string + ContainerRuntimeConfigFileName string + ContainerRuntimeConfig string + ContainerRuntimeAuthConfigFileName string + ContainerRuntimeAuthConfig string + ContainerRuntimeName string }{ - UserDataRequest: req, - ProviderSpec: pconfig, - OSConfig: amznConfig, - KubeletVersion: kubeletVersion.String(), - ServerAddr: serverAddr, - Kubeconfig: kubeconfigString, - KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), - ExtraKubeletFlags: crEngine.KubeletFlags(), - ContainerRuntimeScript: crScript, - ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), - ContainerRuntimeConfig: crConfig, - ContainerRuntimeName: crEngine.String(), + UserDataRequest: req, + ProviderSpec: pconfig, + OSConfig: amznConfig, + KubeletVersion: kubeletVersion.String(), + ServerAddr: serverAddr, + Kubeconfig: kubeconfigString, + KubernetesCACert: kubernetesCACert, + NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), + ExtraKubeletFlags: crEngine.KubeletFlags(), + ContainerRuntimeScript: crScript, + ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), + ContainerRuntimeConfig: crConfig, + ContainerRuntimeAuthConfigFileName: crEngine.AuthConfigFileName(), + ContainerRuntimeAuthConfig: crAuthConfig, + ContainerRuntimeName: crEngine.String(), } buf := strings.Builder{} @@ -302,6 +311,14 @@ write_files: content: | {{ .ContainerRuntimeConfig | indent 4 }} +{{- if and (eq .ContainerRuntimeName "docker") .ContainerRuntimeAuthConfig }} + +- path: {{ .ContainerRuntimeAuthConfigFileName }} + permissions: "0600" + content: | +{{ .ContainerRuntimeAuthConfig | indent 4 }} +{{- end }} + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" content: | diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml index 27d9eb15b..7e0f6bc5b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml @@ -198,6 +198,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml index b2b7f81ad..29ea96787 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml @@ -198,6 +198,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 91b61d1c2..223c174e2 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -213,6 +213,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml index 8a71ab415..5c30291d4 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -213,6 +213,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml index 004b4504f..b8e74651c 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml @@ -205,6 +205,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml index 5f0b13b30..3313f2065 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml @@ -198,6 +198,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml index d957ee4c3..fe320b2a6 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml @@ -198,6 +198,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml index 6c2ae7203..cc323a27b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml @@ -201,6 +201,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index 40490338e..17b497a43 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -92,34 +92,43 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { return "", fmt.Errorf("failed to generate container runtime config: %w", err) } + crAuthConfig, err := crEngine.AuthConfig() + if err != nil { + return "", fmt.Errorf("failed to generate container runtime auth config: %w", err) + } + data := struct { plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - OSConfig *Config - KubeletVersion string - ServerAddr string - Kubeconfig string - KubernetesCACert string - NodeIPScript string - ExtraKubeletFlags []string - ContainerRuntimeScript string - ContainerRuntimeConfigFileName string - ContainerRuntimeConfig string - ContainerRuntimeName string + ProviderSpec *providerconfigtypes.Config + OSConfig *Config + KubeletVersion string + ServerAddr string + Kubeconfig string + KubernetesCACert string + NodeIPScript string + ExtraKubeletFlags []string + ContainerRuntimeScript string + ContainerRuntimeConfigFileName string + ContainerRuntimeConfig string + ContainerRuntimeAuthConfigFileName string + ContainerRuntimeAuthConfig string + ContainerRuntimeName string }{ - UserDataRequest: req, - ProviderSpec: pconfig, - OSConfig: centosConfig, - KubeletVersion: kubeletVersion.String(), - ServerAddr: serverAddr, - Kubeconfig: kubeconfigString, - KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), - ExtraKubeletFlags: crEngine.KubeletFlags(), - ContainerRuntimeScript: crScript, - ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), - ContainerRuntimeConfig: crConfig, - ContainerRuntimeName: crEngine.String(), + UserDataRequest: req, + ProviderSpec: pconfig, + OSConfig: centosConfig, + KubeletVersion: kubeletVersion.String(), + ServerAddr: serverAddr, + Kubeconfig: kubeconfigString, + KubernetesCACert: kubernetesCACert, + NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), + ExtraKubeletFlags: crEngine.KubeletFlags(), + ContainerRuntimeScript: crScript, + ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), + ContainerRuntimeConfig: crConfig, + ContainerRuntimeAuthConfigFileName: crEngine.AuthConfigFileName(), + ContainerRuntimeAuthConfig: crAuthConfig, + ContainerRuntimeName: crEngine.String(), } buf := strings.Builder{} @@ -320,6 +329,14 @@ write_files: content: | {{ .ContainerRuntimeConfig | indent 4 }} +{{- if and (eq .ContainerRuntimeName "docker") .ContainerRuntimeAuthConfig }} + +- path: {{ .ContainerRuntimeAuthConfigFileName }} + permissions: "0600" + content: | +{{ .ContainerRuntimeAuthConfig | indent 4 }} +{{- end }} + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" content: | diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml index 8084e5f05..1b378e43b 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml @@ -208,6 +208,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml index 55a72e4fc..2c75bdb1f 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml @@ -208,6 +208,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 24a680856..4a9fde5dd 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -223,6 +223,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml index 513051ec2..a8c507167 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -223,6 +223,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml index 9bda409e9..28888a8a2 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml @@ -215,6 +215,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index 2f0466221..fe3619423 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -208,6 +208,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index bb6785c09..b09de8e64 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -208,6 +208,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml index a5119df13..3087464bc 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml @@ -215,6 +215,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml index fb7f7adfa..72068f2f9 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml @@ -207,6 +207,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index 9d5819536..471ce53ec 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -93,32 +93,41 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { return "", fmt.Errorf("failed to generate container runtime config: %w", err) } + crAuthConfig, err := crEngine.AuthConfig() + if err != nil { + return "", fmt.Errorf("failed to generate container runtime auth config: %w", err) + } + data := struct { plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - FlatcarConfig *Config - Kubeconfig string - KubernetesCACert string - KubeletVersion string - NodeIPScript string - ExtraKubeletFlags []string - ContainerRuntimeScript string - ContainerRuntimeConfigFileName string - ContainerRuntimeConfig string - ContainerRuntimeName string + ProviderSpec *providerconfigtypes.Config + FlatcarConfig *Config + KubeletVersion string + Kubeconfig string + KubernetesCACert string + NodeIPScript string + ExtraKubeletFlags []string + ContainerRuntimeScript string + ContainerRuntimeConfigFileName string + ContainerRuntimeConfig string + ContainerRuntimeAuthConfigFileName string + ContainerRuntimeAuthConfig string + ContainerRuntimeName string }{ - UserDataRequest: req, - ProviderSpec: pconfig, - FlatcarConfig: flatcarConfig, - Kubeconfig: kubeconfigString, - KubernetesCACert: kubernetesCACert, - KubeletVersion: kubeletVersion.String(), - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), - ExtraKubeletFlags: crEngine.KubeletFlags(), - ContainerRuntimeScript: crScript, - ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), - ContainerRuntimeConfig: crConfig, - ContainerRuntimeName: crEngine.String(), + UserDataRequest: req, + ProviderSpec: pconfig, + FlatcarConfig: flatcarConfig, + KubeletVersion: kubeletVersion.String(), + Kubeconfig: kubeconfigString, + KubernetesCACert: kubernetesCACert, + NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), + ExtraKubeletFlags: crEngine.KubeletFlags(), + ContainerRuntimeScript: crScript, + ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), + ContainerRuntimeConfig: crConfig, + ContainerRuntimeAuthConfigFileName: crEngine.AuthConfigFileName(), + ContainerRuntimeAuthConfig: crAuthConfig, + ContainerRuntimeName: crEngine.String(), } b := &bytes.Buffer{} @@ -493,6 +502,16 @@ storage: inline: | {{ .ContainerRuntimeConfig | indent 10 }} +{{- if and (eq .ContainerRuntimeName "docker") .ContainerRuntimeAuthConfig }} + + - path: {{ .ContainerRuntimeAuthConfigFileName }} + filesystem: root + permissions: 0600 + content: + inline: | +{{ .ContainerRuntimeAuthConfig | indent 10 }} +{{- end }} + - path: /etc/crictl.yaml filesystem: root mode: 0644 @@ -758,6 +777,14 @@ write_files: content: | {{ .ContainerRuntimeConfig | indent 4 }} +{{- if and (eq .ContainerRuntimeName "docker") .ContainerRuntimeAuthConfig }} + +- path: {{ .ContainerRuntimeAuthConfigFileName }} + permissions: "0600" + content: | +{{ .ContainerRuntimeAuthConfig | indent 4 }} +{{- end }} + - path: /etc/crictl.yaml permissions: "0644" user: root diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.10.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.10.yaml index 1c01066bb..a1b39a561 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.10.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.10.yaml @@ -102,6 +102,7 @@ coreos: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml index 61378ccea..33c02cf25 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml @@ -102,6 +102,7 @@ coreos: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml index e78ec28c6..fabfc11fc 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml @@ -102,6 +102,7 @@ coreos: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml index 91fc82387..941d78668 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml @@ -102,6 +102,7 @@ coreos: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index d7ea9916f..3f98342f8 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -87,6 +87,7 @@ coreos: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.21.10.json b/pkg/userdata/flatcar/testdata/ignition_v1.21.10.json index 9422cbf51..b40d43f74 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.21.10.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.21.10.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.10%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.10%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json b/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json index 86790330c..a7d6cc773 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.7%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.7%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json b/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json index f470beb95..80f86b6a2 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json index 9c669c995..c678da203 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index 759ea4d46..74697a5d2 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -73,6 +73,7 @@ Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ [Service] +User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden index 4c54d80cc..4d758832f 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden @@ -6,6 +6,7 @@ Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ [Service] +User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden index be191df0e..26ed4f3ed 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden @@ -6,6 +6,7 @@ Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ [Service] +User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_pause-image-set.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_pause-image-set.golden index 712ddf88c..662c257dd 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_pause-image-set.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_pause-image-set.golden @@ -6,6 +6,7 @@ Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ [Service] +User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_taints-set.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_taints-set.golden index dbc3bf80c..d0d0c5ccf 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_taints-set.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_taints-set.golden @@ -6,6 +6,7 @@ Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ [Service] +User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10-external.golden index 9a73edd63..435cefdf8 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10-external.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10-external.golden @@ -6,6 +6,7 @@ Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ [Service] +User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10.golden index 4cbdf3e13..e1c0b6225 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10.golden @@ -6,6 +6,7 @@ Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ [Service] +User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.7-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.7-external.golden index 9a73edd63..435cefdf8 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.7-external.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.7-external.golden @@ -6,6 +6,7 @@ Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ [Service] +User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.7.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.7.golden index 4cbdf3e13..e1c0b6225 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.7.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.22.7.golden @@ -6,6 +6,7 @@ Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ [Service] +User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5-external.golden index 04ba38dbf..bfb133228 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5-external.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5-external.golden @@ -6,6 +6,7 @@ Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ [Service] +User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5.golden index be191df0e..26ed4f3ed 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5.golden @@ -6,6 +6,7 @@ Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ [Service] +User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.0-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.0-external.golden index 2682ca454..50f4f5138 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.0-external.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.0-external.golden @@ -6,6 +6,7 @@ Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ [Service] +User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.0.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.0.golden index e5fc6c60b..e70567560 100644 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.0.golden +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.0.golden @@ -6,6 +6,7 @@ Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ [Service] +User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 1d3821376..4d9d123b7 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -92,34 +92,43 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { return "", fmt.Errorf("failed to generate container runtime config: %w", err) } + crAuthConfig, err := crEngine.AuthConfig() + if err != nil { + return "", fmt.Errorf("failed to generate container runtime auth config: %w", err) + } + data := struct { plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - OSConfig *Config - KubeletVersion string - ServerAddr string - Kubeconfig string - KubernetesCACert string - NodeIPScript string - ExtraKubeletFlags []string - ContainerRuntimeScript string - ContainerRuntimeConfigFileName string - ContainerRuntimeConfig string - ContainerRuntimeName string + ProviderSpec *providerconfigtypes.Config + OSConfig *Config + KubeletVersion string + ServerAddr string + Kubeconfig string + KubernetesCACert string + NodeIPScript string + ExtraKubeletFlags []string + ContainerRuntimeScript string + ContainerRuntimeConfigFileName string + ContainerRuntimeConfig string + ContainerRuntimeAuthConfigFileName string + ContainerRuntimeAuthConfig string + ContainerRuntimeName string }{ - UserDataRequest: req, - ProviderSpec: pconfig, - OSConfig: rhelConfig, - KubeletVersion: kubeletVersion.String(), - ServerAddr: serverAddr, - Kubeconfig: kubeconfigString, - KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), - ExtraKubeletFlags: crEngine.KubeletFlags(), - ContainerRuntimeScript: crScript, - ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), - ContainerRuntimeConfig: crConfig, - ContainerRuntimeName: crEngine.String(), + UserDataRequest: req, + ProviderSpec: pconfig, + OSConfig: rhelConfig, + KubeletVersion: kubeletVersion.String(), + ServerAddr: serverAddr, + Kubeconfig: kubeconfigString, + KubernetesCACert: kubernetesCACert, + NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), + ExtraKubeletFlags: crEngine.KubeletFlags(), + ContainerRuntimeScript: crScript, + ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), + ContainerRuntimeConfig: crConfig, + ContainerRuntimeAuthConfigFileName: crEngine.AuthConfigFileName(), + ContainerRuntimeAuthConfig: crAuthConfig, + ContainerRuntimeName: crEngine.String(), } var buf strings.Builder @@ -320,6 +329,14 @@ write_files: content: | {{ .ContainerRuntimeConfig | indent 4 }} +{{- if and (eq .ContainerRuntimeName "docker") .ContainerRuntimeAuthConfig }} + +- path: {{ .ContainerRuntimeAuthConfigFileName }} + permissions: "0600" + content: | +{{ .ContainerRuntimeAuthConfig | indent 4 }} +{{- end }} + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" content: | diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml index db6c4a381..2081ec509 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml @@ -208,6 +208,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index 7f25ec49d..7f53086b2 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -208,6 +208,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml index 6ba2ce6e5..3e88d15a4 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml @@ -216,6 +216,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index 2ab3000dd..883f450c6 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -208,6 +208,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index c43fa2702..b9707e3a9 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -208,6 +208,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 01e0d1df5..6f9490298 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -224,6 +224,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index 38e98cb96..7b9d5b9f1 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -224,6 +224,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index 693e29a41..666cfccac 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -216,6 +216,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml index b1e1e8845..25dc1e6cb 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml @@ -207,6 +207,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml index b91e171b8..021ff6a99 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml @@ -207,6 +207,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index f804908c5..74743a1b9 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -213,6 +213,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rockylinux/provider.go b/pkg/userdata/rockylinux/provider.go index 96de00df3..44a406094 100644 --- a/pkg/userdata/rockylinux/provider.go +++ b/pkg/userdata/rockylinux/provider.go @@ -92,34 +92,43 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { return "", fmt.Errorf("failed to generate container runtime config: %w", err) } + crAuthConfig, err := crEngine.AuthConfig() + if err != nil { + return "", fmt.Errorf("failed to generate container runtime auth config: %w", err) + } + data := struct { plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - OSConfig *Config - KubeletVersion string - ServerAddr string - Kubeconfig string - KubernetesCACert string - NodeIPScript string - ExtraKubeletFlags []string - ContainerRuntimeScript string - ContainerRuntimeConfigFileName string - ContainerRuntimeConfig string - ContainerRuntimeName string + ProviderSpec *providerconfigtypes.Config + OSConfig *Config + KubeletVersion string + ServerAddr string + Kubeconfig string + KubernetesCACert string + NodeIPScript string + ExtraKubeletFlags []string + ContainerRuntimeScript string + ContainerRuntimeConfigFileName string + ContainerRuntimeConfig string + ContainerRuntimeAuthConfigFileName string + ContainerRuntimeAuthConfig string + ContainerRuntimeName string }{ - UserDataRequest: req, - ProviderSpec: pconfig, - OSConfig: rockyLinuxConfig, - KubeletVersion: kubeletVersion.String(), - ServerAddr: serverAddr, - Kubeconfig: kubeconfigString, - KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), - ExtraKubeletFlags: crEngine.KubeletFlags(), - ContainerRuntimeScript: crScript, - ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), - ContainerRuntimeConfig: crConfig, - ContainerRuntimeName: crEngine.String(), + UserDataRequest: req, + ProviderSpec: pconfig, + OSConfig: rockyLinuxConfig, + KubeletVersion: kubeletVersion.String(), + ServerAddr: serverAddr, + Kubeconfig: kubeconfigString, + KubernetesCACert: kubernetesCACert, + NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), + ExtraKubeletFlags: crEngine.KubeletFlags(), + ContainerRuntimeScript: crScript, + ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), + ContainerRuntimeConfig: crConfig, + ContainerRuntimeAuthConfigFileName: crEngine.AuthConfigFileName(), + ContainerRuntimeAuthConfig: crAuthConfig, + ContainerRuntimeName: crEngine.String(), } buf := strings.Builder{} @@ -312,6 +321,14 @@ write_files: content: | {{ .ContainerRuntimeConfig | indent 4 }} +{{- if and (eq .ContainerRuntimeName "docker") .ContainerRuntimeAuthConfig }} + +- path: {{ .ContainerRuntimeAuthConfigFileName }} + permissions: "0600" + content: | +{{ .ContainerRuntimeAuthConfig | indent 4 }} +{{- end }} + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" content: | diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml index 937197439..8e6bfb30a 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml @@ -203,6 +203,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml index b5f566c62..e21b297d9 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml @@ -203,6 +203,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml index a9f97caef..5eee9ee20 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -218,6 +218,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml index 43bc2130e..c6625b5f3 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -218,6 +218,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml index 551beb181..781c40f8e 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml @@ -210,6 +210,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml index dadbb77a5..c3dd772a0 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml @@ -203,6 +203,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml index 0c78587d7..397e92608 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml @@ -203,6 +203,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml index 49fb9bf58..07bc438ef 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml @@ -210,6 +210,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml index 73060d423..fcb39d701 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml @@ -202,6 +202,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/sles/provider.go b/pkg/userdata/sles/provider.go index 44a5d839b..6299c672c 100644 --- a/pkg/userdata/sles/provider.go +++ b/pkg/userdata/sles/provider.go @@ -87,32 +87,41 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { return "", fmt.Errorf("failed to generate container runtime config: %w", err) } + crAuthConfig, err := crEngine.AuthConfig() + if err != nil { + return "", fmt.Errorf("failed to generate container runtime auth config: %w", err) + } + data := struct { plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - OSConfig *Config - ServerAddr string - KubeletVersion string - Kubeconfig string - KubernetesCACert string - NodeIPScript string - ExtraKubeletFlags []string - ContainerRuntimeConfigFileName string - ContainerRuntimeConfig string - ContainerRuntimeName string + ProviderSpec *providerconfigtypes.Config + OSConfig *Config + ServerAddr string + KubeletVersion string + Kubeconfig string + KubernetesCACert string + NodeIPScript string + ExtraKubeletFlags []string + ContainerRuntimeConfigFileName string + ContainerRuntimeConfig string + ContainerRuntimeAuthConfigFileName string + ContainerRuntimeAuthConfig string + ContainerRuntimeName string }{ - UserDataRequest: req, - ProviderSpec: pconfig, - OSConfig: slesConfig, - ServerAddr: serverAddr, - KubeletVersion: kubeletVersion.String(), - Kubeconfig: kubeconfigString, - KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), - ExtraKubeletFlags: crEngine.KubeletFlags(), - ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), - ContainerRuntimeConfig: crConfig, - ContainerRuntimeName: crEngine.String(), + UserDataRequest: req, + ProviderSpec: pconfig, + OSConfig: slesConfig, + ServerAddr: serverAddr, + KubeletVersion: kubeletVersion.String(), + Kubeconfig: kubeconfigString, + KubernetesCACert: kubernetesCACert, + NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), + ExtraKubeletFlags: crEngine.KubeletFlags(), + ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), + ContainerRuntimeConfig: crConfig, + ContainerRuntimeAuthConfigFileName: crEngine.AuthConfigFileName(), + ContainerRuntimeAuthConfig: crAuthConfig, + ContainerRuntimeName: crEngine.String(), } b := &bytes.Buffer{} err = tmpl.Execute(b, data) @@ -270,6 +279,14 @@ write_files: content: | {{ .ContainerRuntimeConfig | indent 4 }} +{{- if and (eq .ContainerRuntimeName "docker") .ContainerRuntimeAuthConfig }} + +- path: {{ .ContainerRuntimeAuthConfigFileName }} + permissions: "0600" + content: | +{{ .ContainerRuntimeAuthConfig | indent 4 }} +{{- end }} + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" content: | diff --git a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml index 97b60ae7c..9009e4bb3 100644 --- a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml @@ -165,6 +165,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml index de832d974..7e1c146ea 100644 --- a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml @@ -163,6 +163,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml index affbbf41e..3cdfba5f6 100644 --- a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml @@ -163,6 +163,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml index 9d4ed5312..6643d04e4 100644 --- a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml @@ -165,6 +165,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml index 096835dbf..32a04e68b 100644 --- a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml @@ -163,6 +163,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/sles/testdata/openstack.yaml b/pkg/userdata/sles/testdata/openstack.yaml index 02f52bef2..1aa862a45 100644 --- a/pkg/userdata/sles/testdata/openstack.yaml +++ b/pkg/userdata/sles/testdata/openstack.yaml @@ -163,6 +163,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/sles/testdata/version-1.21.10.yaml b/pkg/userdata/sles/testdata/version-1.21.10.yaml index fb99d31fe..2c781afee 100644 --- a/pkg/userdata/sles/testdata/version-1.21.10.yaml +++ b/pkg/userdata/sles/testdata/version-1.21.10.yaml @@ -163,6 +163,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/sles/testdata/version-1.22.7.yaml b/pkg/userdata/sles/testdata/version-1.22.7.yaml index 5020276b6..c2d9687d1 100644 --- a/pkg/userdata/sles/testdata/version-1.22.7.yaml +++ b/pkg/userdata/sles/testdata/version-1.22.7.yaml @@ -163,6 +163,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/sles/testdata/version-1.23.5.yaml b/pkg/userdata/sles/testdata/version-1.23.5.yaml index 3cef64351..d42a82bc1 100644 --- a/pkg/userdata/sles/testdata/version-1.23.5.yaml +++ b/pkg/userdata/sles/testdata/version-1.23.5.yaml @@ -163,6 +163,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/sles/testdata/version-1.24.0.yaml b/pkg/userdata/sles/testdata/version-1.24.0.yaml index afbb0faef..179ecc5cb 100644 --- a/pkg/userdata/sles/testdata/version-1.24.0.yaml +++ b/pkg/userdata/sles/testdata/version-1.24.0.yaml @@ -163,6 +163,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml index 092a051c4..7d732ee9b 100644 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml @@ -173,6 +173,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml index 2a706d7ee..9576ec66d 100644 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/sles/testdata/vsphere-proxy.yaml @@ -173,6 +173,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml index 2c7a9d155..76a9a138b 100644 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ b/pkg/userdata/sles/testdata/vsphere.yaml @@ -164,6 +164,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index ab7b547b4..cccb7c7da 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -91,34 +91,44 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { if err != nil { return "", fmt.Errorf("failed to generate container runtime config: %w", err) } + + crAuthConfig, err := crEngine.AuthConfig() + if err != nil { + return "", fmt.Errorf("failed to generate container runtime auth config: %w", err) + } + data := struct { plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - OSConfig *Config - ServerAddr string - KubeletVersion string - Kubeconfig string - KubernetesCACert string - NodeIPScript string - ExtraKubeletFlags []string - ContainerRuntimeScript string - ContainerRuntimeConfigFileName string - ContainerRuntimeConfig string - ContainerRuntimeName string + ProviderSpec *providerconfigtypes.Config + OSConfig *Config + ServerAddr string + KubeletVersion string + Kubeconfig string + KubernetesCACert string + NodeIPScript string + ExtraKubeletFlags []string + ContainerRuntimeScript string + ContainerRuntimeConfigFileName string + ContainerRuntimeConfig string + ContainerRuntimeAuthConfigFileName string + ContainerRuntimeAuthConfig string + ContainerRuntimeName string }{ - UserDataRequest: req, - ProviderSpec: pconfig, - OSConfig: ubuntuConfig, - ServerAddr: serverAddr, - KubeletVersion: kubeletVersion.String(), - Kubeconfig: kubeconfigString, - KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), - ExtraKubeletFlags: crEngine.KubeletFlags(), - ContainerRuntimeScript: crScript, - ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), - ContainerRuntimeConfig: crConfig, - ContainerRuntimeName: crEngine.String(), + UserDataRequest: req, + ProviderSpec: pconfig, + OSConfig: ubuntuConfig, + ServerAddr: serverAddr, + KubeletVersion: kubeletVersion.String(), + Kubeconfig: kubeconfigString, + KubernetesCACert: kubernetesCACert, + NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), + ExtraKubeletFlags: crEngine.KubeletFlags(), + ContainerRuntimeScript: crScript, + ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), + ContainerRuntimeConfig: crConfig, + ContainerRuntimeAuthConfigFileName: crEngine.AuthConfigFileName(), + ContainerRuntimeAuthConfig: crAuthConfig, + ContainerRuntimeName: crEngine.String(), } var buf strings.Builder @@ -311,6 +321,14 @@ write_files: content: | {{ .ContainerRuntimeConfig | indent 4 }} +{{- if and (eq .ContainerRuntimeName "docker") .ContainerRuntimeAuthConfig }} + +- path: {{ .ContainerRuntimeAuthConfigFileName }} + permissions: "0600" + content: | +{{ .ContainerRuntimeAuthConfig | indent 4 }} +{{- end }} + - path: "/etc/kubernetes/kubelet.conf" content: | {{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }} diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 8f564b845..47914c90b 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -445,6 +445,38 @@ func TestUserDataGeneration(t *testing.T) { DistUpgradeOnBoot: true, }, }, + { + name: "docker", + containerruntime: "docker", + registryCredentials: map[string]containerruntime.AuthConfig{ + "docker.io": { + Username: "login1", + Password: "passwd1", + }, + }, + providerSpec: &providerconfigtypes.Config{ + CloudProvider: "", + SSHPublicKeys: []string{"ssh-rsa AAABBB"}, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: defaultVersion, + }, + }, + ccProvider: &fakeCloudConfigProvider{ + name: "", + config: "", + err: nil, + }, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, + kubernetesCACert: "CACert", + osConfig: &Config{ + DistUpgradeOnBoot: true, + }, + }, { name: "nutanix", providerSpec: &providerconfigtypes.Config{ diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index b3ccf4b3b..5837b5589 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -206,6 +206,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 848ed03bb..611fa8072 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -206,6 +206,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml new file mode 100644 index 000000000..5e32ac8c5 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/docker.yaml @@ -0,0 +1,461 @@ +#cloud-config + +hostname: node1 + +package_upgrade: true +package_reboot_if_required: true + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: /root/.docker/config.json + permissions: "0600" + content: | + { + "auths": { + "docker.io": { + "username": "login1", + "password": "passwd1" + } + } + } + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index 036278ac9..99fbdd863 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -204,6 +204,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index c9ac3fc14..95e57ea40 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -204,6 +204,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 2600324f9..fc6034a29 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -206,6 +206,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 0255b4787..52107be9c 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -207,6 +207,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index 3a87ca5cb..2ddb7214b 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -204,6 +204,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index b2e9a633a..48183d1b4 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -204,6 +204,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml b/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml index 59f3a7727..aea35e1d3 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml @@ -204,6 +204,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml b/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml index 036278ac9..99fbdd863 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml @@ -204,6 +204,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml index a161862a5..7b669de76 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml @@ -204,6 +204,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml index d825aba93..21dc8aebe 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml @@ -204,6 +204,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index 3019f8c48..7c3c03cf5 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -214,6 +214,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index f8ffd4b8e..cd69189d6 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -214,6 +214,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 16eb24573..5fadc4b76 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -205,6 +205,7 @@ write_files: Documentation=https://kubernetes.io/docs/home/ [Service] + User=root Restart=always StartLimitInterval=0 RestartSec=10 From 11060f55dc95dc6cd67e96ccc5a7637f3dc55512 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 24 May 2022 19:46:18 +0500 Subject: [PATCH 154/489] Add support for VMware Cloud Director as cloud provider (#1300) * Add support for VMware Cloud Director as cloud provider Signed-off-by: Waleed Malik * Refactor sample MD for VCD Signed-off-by: Waleed Malik * Refactored code Signed-off-by: Waleed Malik * Refactored code Signed-off-by: Waleed Malik --- .gitignore | 3 +- .prow/provider-vcloud-director.yaml | 38 ++ .wwhrd.yml | 1 + README.md | 109 ++-- cmd/provision/README.md | 15 +- docs/operating-system.md | 2 + docs/vmware-cloud-director.md | 14 + ...ware-cloud-director-machinedeployment.yaml | 79 +++ go.mod | 5 + go.sum | 7 + pkg/cloudprovider/provider.go | 4 + .../provider/vmware-cloud-director/client.go | 172 ++++++ .../provider/vmware-cloud-director/helper.go | 287 +++++++++ .../vmware-cloud-director/provider.go | 554 ++++++++++++++++++ .../vmware-cloud-director/types/types.go | 71 +++ .../provider/vsphere/provider.go | 7 +- pkg/providerconfig/types/types.go | 38 +- pkg/userdata/amzn2/provider.go | 2 +- pkg/userdata/centos/provider.go | 2 +- pkg/userdata/rhel/provider.go | 2 +- pkg/userdata/rockylinux/provider.go | 2 +- pkg/userdata/sles/provider.go | 2 +- pkg/userdata/ubuntu/provider.go | 2 +- test/e2e/provisioning/all_e2e_test.go | 33 ++ ...chinedeployment-vmware-cloud-director.yaml | 58 ++ 25 files changed, 1431 insertions(+), 78 deletions(-) create mode 100644 .prow/provider-vcloud-director.yaml create mode 100644 docs/vmware-cloud-director.md create mode 100644 examples/vmware-cloud-director-machinedeployment.yaml create mode 100644 pkg/cloudprovider/provider/vmware-cloud-director/client.go create mode 100644 pkg/cloudprovider/provider/vmware-cloud-director/helper.go create mode 100644 pkg/cloudprovider/provider/vmware-cloud-director/provider.go create mode 100644 pkg/cloudprovider/provider/vmware-cloud-director/types/types.go create mode 100644 test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml diff --git a/.gitignore b/.gitignore index b5a1dac67..8cc8d65e8 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,5 @@ examples/*.srl /webhook /vendor .vscode -.gitpod.yml \ No newline at end of file +.gitpod.yml +cmd/machine-controller/__debug_bin diff --git a/.prow/provider-vcloud-director.yaml b/.prow/provider-vcloud-director.yaml new file mode 100644 index 000000000..331bdabd1 --- /dev/null +++ b/.prow/provider-vcloud-director.yaml @@ -0,0 +1,38 @@ +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +presubmits: + - name: pull-machine-controller-e2e-vmware-cloud-director + always_run: false + run_if_changed: "(pkg/cloudprovider/provider/vcloud-director/|pkg/userdata)" + decorate: true + error_on_eviction: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-vcloud-director: "true" + preset-rhel: "true" + preset-goproxy: "true" + spec: + containers: + - image: golang:1.18.2 + command: + - "./hack/ci-e2e-test.sh" + args: + - "TestVMwareCloudDirectorProvisioningE2E" + resources: + requests: + memory: 1Gi + cpu: 500m diff --git a/.wwhrd.yml b/.wwhrd.yml index d3c17d045..239a0dc31 100644 --- a/.wwhrd.yml +++ b/.wwhrd.yml @@ -31,3 +31,4 @@ exceptions: - github.com/embik/nutanix-client-go/pkg/client/v3 # MPL-2.0 - github.com/embik/nutanix-client-go/internal/utils # MPL-2.0 - github.com/ajeddeloh/go-json # Since it's a fork, https://github.com/golang/go/blob/master/LICENSE + - github.com/hashicorp/go-version # MPL-2.0 diff --git a/README.md b/README.md index 370e76a23..e6634b8d5 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,40 @@ # Kubermatic machine-controller -# Table of Contents - -- [Features](#features) -- [Quickstart](#Quickstart) - - [Deployment](#Deploy-the-machine-controller) - - [Creating a machineDeployment](#Creating-a-machineDeployment) - - [Special network restrictions](/docs/network-restrictions.md) -- [Cloud provider](/docs/cloud-provider.md) -- [Operating system](/docs/operating-system.md) - - [OpenStack images](/docs/openstack-images.md) -- [Development](#development) -- [How to add a new provider](docs/howto-provider.md) -- [E2E Infra](/docs/e2e-infra.md) -- [TroubleShooting](#troubleshooting) -- [Contributing](#contributing) -- [Changelog](#changelog) - -# Features -## What works -- Creation of worker nodes on AWS, Digitalocean, Openstack, Azure, Google Cloud Platform, VMWare Vsphere, Linode, Hetzner cloud and Kubevirt (experimental) +## Table of Contents + +- [Kubermatic machine-controller](#kubermatic-machine-controller) + - [Table of Contents](#table-of-contents) + - [Features](#features) + - [What works](#what-works) + - [Supported Kubernetes versions](#supported-kubernetes-versions) + - [What does not work](#what-does-not-work) + - [Quickstart](#quickstart) + - [Deploy the machine-controller](#deploy-the-machine-controller) + - [Creating a machineDeployment](#creating-a-machinedeployment) + - [Advanced usage](#advanced-usage) + - [Specifying the apiserver endpoint](#specifying-the-apiserver-endpoint) + - [CA-data](#ca-data) + - [Apiserver endpoint](#apiserver-endpoint) + - [Example cluster-info ConfigMap](#example-cluster-info-configmap) + - [Development](#development) + - [Testing](#testing) + - [Unittests](#unittests) + - [End-to-End](#end-to-end) + - [Troubleshooting](#troubleshooting) + - [Contributing](#contributing) + - [Before you start](#before-you-start) + - [Pull requests](#pull-requests) + - [Changelog](#changelog) + +## Features + +### What works + +- Creation of worker nodes on AWS, Digitalocean, Openstack, Azure, Google Cloud Platform, Nutanix, VMWare Cloud Director, VMWare Vsphere, Linode, Hetzner cloud and Kubevirt (experimental) - Using Ubuntu, Flatcar or CentOS 7 distributions ([not all distributions work on all providers](/docs/operating-system.md)) -## Supported Kubernetes versions +### Supported Kubernetes versions + machine-controller tries to follow the Kubernetes version [support policy](https://kubernetes.io/docs/setup/release/version-skew-policy/) as close as possible. @@ -34,15 +46,17 @@ Currently supported K8S versions are: - 1.21 ## What does not work + - Master creation (Not planned at the moment) -# Quickstart +## Quickstart -## Deploy the machine-controller +### Deploy the machine-controller `make deploy` -## Creating a machineDeployment +### Creating a machineDeployment + ```bash # edit examples/$cloudprovider-machinedeployment.yaml & create the machineDeployment kubectl create -f examples/$cloudprovider-machinedeployment.yaml @@ -51,23 +65,26 @@ kubectl create -f examples/$cloudprovider-machinedeployment.yaml ## Advanced usage ### Specifying the apiserver endpoint + By default the controller looks for a `cluster-info` ConfigMap within the `kube-public` Namespace. If one is found which contains a minimal kubeconfig (kubeadm cluster have them by default), this kubeconfig will be used for the node bootstrapping. The kubeconfig only needs to contain two things: + - CA-Data - The public endpoint for the Apiserver If no ConfigMap can be found: -**CA-data** +### CA-data The CA will be loaded from the passed kubeconfig when running outside the cluster or from `/var/run/secrets/kubernetes.io/serviceaccount/ca.crt` when running inside the cluster. -**Apiserver endpoint** +### Apiserver endpoint The first endpoint from the kubernetes endpoints will be taken. `kubectl get endpoints kubernetes -o yaml` #### Example cluster-info ConfigMap + ```yaml apiVersion: v1 kind: ConfigMap @@ -89,22 +106,22 @@ data: users: [] ``` -# Development +## Development -## Testing +### Testing -### Unittests +#### Unittests Simply run `make test-unit` -### End-to-End +#### End-to-End This project provides easy to use e2e testing using Hetzner cloud. To run the e2e tests locally, the following steps are required: -* Populate the environment variable `HZ_E2E_TOKEN` with a valid Hetzner cloud token -* Run `make e2e-cluster` to get a simple kubeadm cluster on Hetzner -* Run `hack/run-machine-controller.sh` to locally run the machine-controller for your freshly created cluster +- Populate the environment variable `HZ_E2E_TOKEN` with a valid Hetzner cloud token +- Run `make e2e-cluster` to get a simple kubeadm cluster on Hetzner +- Run `hack/run-machine-controller.sh` to locally run the machine-controller for your freshly created cluster If you want to use an existing cluster to test against, you can simply set the `KUBECONFIG` environment variable. In this case, first make sure that a kubeconfig created by `make e2e-cluster` at `$(go env GOPATH)/src/github.com/kubermatic/machine-controller/.kubeconfig` @@ -112,41 +129,41 @@ doesn't exist, since the tests will default to this hardcoded path and only use Now you can either -* Run the tests for all providers via +- Run the tests for all providers via `go test -race -tags=e2e -parallel 240 -v -timeout 30m ./test/e2e/... -identifier $USER` -* Check `test/e2e/provisioning/all_e2e_test.go` for the available tests, then run only a specific one via +- Check `test/e2e/provisioning/all_e2e_test.go` for the available tests, then run only a specific one via `go test -race -tags=e2e -parallel 24 -v -timeout 20m ./test/e2e/... -identifier $USER -run $TESTNAME` -__Note:__ All e2e tests require corresponding credentials to be present, check +**Note:** All e2e tests require corresponding credentials to be present, check [`test/e2e/provisioning/all_e2e_test.go`](test/e2e/provisioning/all_e2e_test.go) for details -__Note:__ After finishing testing, please clean up after yourself: +**Note:** After finishing testing, please clean up after yourself: -* Execute `./test/tools/integration/cleanup_machines.sh` while the machine-controller is still running -* Execute `make e2e-destroy` to clean up the test control plane +- Execute `./test/tools/integration/cleanup_machines.sh` while the machine-controller is still running +- Execute `make e2e-destroy` to clean up the test control plane You can also insert your ssh key into the created instances by editing the manifests in [`test/e2e/provisioning/testdata/`](test/e2e/provisioning/testdata) -# Troubleshooting +## Troubleshooting If you encounter issues [file an issue][1] or talk to us on the [#kubermatic channel][2] on the [Kubermatic Slack][3]. -# Contributing +## Contributing Thanks for taking the time to join our community and start contributing! ### Before you start -* Please familiarize yourself with the [Code of Conduct][4] before contributing. -* See [CONTRIBUTING.md][5] for instructions on the developer certificate of origin that we require. -* Read how [we're using ZenHub][6] for project and roadmap planning +- Please familiarize yourself with the [Code of Conduct][4] before contributing. +- See [CONTRIBUTING.md][5] for instructions on the developer certificate of origin that we require. +- Read how [we're using ZenHub][6] for project and roadmap planning ### Pull requests -* We welcome pull requests. Feel free to dig through the [issues][1] and jump in. +- We welcome pull requests. Feel free to dig through the [issues][1] and jump in. -# Changelog +## Changelog See [the list of releases][7] to find out about feature changes. diff --git a/cmd/provision/README.md b/cmd/provision/README.md index a365f4ac0..4811de7dc 100644 --- a/cmd/provision/README.md +++ b/cmd/provision/README.md @@ -3,11 +3,13 @@ This command offers all required functionality to provision an host to join a Kubernetes cluster. The following operating systems are supported + - Ubuntu 18.04 - CentOS 7 - Flatcar ## Requirements + - The cluster needs to use the bootstrap token authentication ## CLI @@ -26,6 +28,7 @@ The following operating systems are supported Nodes will boot with a cloud-init (Or Ignition) which writes required files & a shell script (called `setup.sh` here). ### cloud-init (Or ignition) + Parts which will be covered by cloud-init (or Ignition) - Install SSH keys @@ -34,7 +37,7 @@ Parts which will be covered by cloud-init (or Ignition) The CA certificate which got used to issue the certificates of the API server serving certificates - `cloud-config` A optional cloud-config used by the kubelet to interact with the cloud provider. -- `setup.sh` +- `setup.sh` Is responsible for downloading the `provision` binary and to execute it. The download of the binary might also be done using built-in `cloud-init` (or Ignition) features @@ -43,13 +46,16 @@ Parts which will be covered by cloud-init (or Ignition) The `provision` binary will identify the operating system and execute a set of provisioning steps. The provisioning process gets separated into 2 phases: + - Base provisioning Install and configure all required dependencies - Join Write & start the kubelet systemd unit #### Base provisioning + The following steps belong into the base provisioning: + - Install required packages (apt & yum action) - Configure required kernel parameter (Like ip forwarding, etc.) - Configure required kernel modules @@ -62,7 +68,8 @@ The following steps belong into the base provisioning: #### Join This part will: -- Write & start the kubelet systemd unit + +- Write & start the kubelet systemd unit ## Offline usage @@ -71,7 +78,7 @@ The `provision` binary should also be usable for "prebaking" images, which then ## Development process To make sure the local development version of the `provision` command gets used for new machines created by the local running machine controller, -a new flag `--provision-source` must be introduced. +a new flag `--provision-source` must be introduced. This flag will instruct the machine controller to download the `provision` binary from the specified location. -For simplicity the `/hack/run-machine-controller.sh` will be updated to include a step which will compile the `provoision` command & upload it to a gcs bucket. +For simplicity the `/hack/run-machine-controller.sh` will be updated to include a step which will compile the `provoision` command & upload it to a gcs bucket. diff --git a/docs/operating-system.md b/docs/operating-system.md index c9cbe7cda..998195a7a 100644 --- a/docs/operating-system.md +++ b/docs/operating-system.md @@ -13,11 +13,13 @@ | Hetzner | ✓ | ✓ | x | x | x | x | ✓ | | Equinix Metal | ✓ | ✓ | x | x | x | x | x | | Openstack | ✓ | ✓ | ✓ | ✓ | x | x | ✓ | +| VMware Cloud Director | ✓ | x | x | x | x | x | x | ## Configuring a operating system The operating system to use can be set via `machine.spec.providerConfig.operatingSystem`. Allowed values: + - `amzn2` - `centos` - `rhel` diff --git a/docs/vmware-cloud-director.md b/docs/vmware-cloud-director.md new file mode 100644 index 000000000..c38cf06cc --- /dev/null +++ b/docs/vmware-cloud-director.md @@ -0,0 +1,14 @@ +# VMware Cloud Director + +## Prerequisites + +The following things should be configured before managing machines on VMware Cloud Director: + +- Dedicated Organization VDC has been created. +- Required catalog and templates for creating VMs have been added to the organization VDC. +- VApp has been created that will be used to encapsulate all the VMs. +- Direct, routed or isolated network has been created. And the virtual machines within the vApp can communicate over that network. + +## Configuration Options + +An example `MachineDeployment` can be found [here](../examples/vmware-cloud-director-machinedeployment.yaml). diff --git a/examples/vmware-cloud-director-machinedeployment.yaml b/examples/vmware-cloud-director-machinedeployment.yaml new file mode 100644 index 000000000..9c2d17044 --- /dev/null +++ b/examples/vmware-cloud-director-machinedeployment.yaml @@ -0,0 +1,79 @@ +apiVersion: v1 +kind: Secret +metadata: + # If you change the namespace/name, you must also + # adjust the rbac rules + name: machine-controller-vcloud-director + namespace: kube-system +type: Opaque +stringData: + password: << VCD_PASSWORD >> +--- +apiVersion: "cluster.k8s.io/v1alpha1" +kind: MachineDeployment +metadata: + name: vcloud-director-machinedeployment + namespace: kube-system +spec: + paused: false + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + minReadySeconds: 0 + selector: + matchLabels: + foo: bar + template: + metadata: + labels: + foo: bar + spec: + providerSpec: + value: + sshPublicKeys: + - "<< YOUR_PUBLIC_KEY >>" + cloudProvider: "vmware-cloud-director" + cloudProviderSpec: + # Can also be set via the env var 'VCD_USER' on the machine-controller + username: "<< VCD_USER >>" + # Can also be set via the env var 'VCD_URL' on the machine-controller + # example: 'https://your-vcloud-director:8443/'. '/api' gets appended automatically + url: "<< VCD_URL >>" + # Can also be set via the env var 'VCD_PASSWORD' on the machine-controller + password: + secretKeyRef: + namespace: kube-system + name: machine-controller-vcloud-director + key: password + # Can also be set via the env var 'VCD_ORG' on the machine-controller + organization: "<< VCD_ORG >>" + # Can also be set via the env var 'VCD_VDC' on the machine-controller + vdc: "<< VCD_VDC >>" + # Can also be set via the env var 'VCD_ALLOW_UNVERIFIED_SSL' on the machine-controller + allowInsecure: false + # vApp to associate the VM with. This should be created before the machine is created + vapp: "<< VCD_VAPP >>" + # Name of catalog where the VM template is located + catalog: "<< VCD_CATALOG >>" + # Name of OS template to be used for the VM + template: "<< VCD_TEMPLATE >>" + # Direct or routed network that can be used for the VM + network: "<< VCD_NETWORK >>" + ipAllocationMode: "DHCP" + cpus: 2 + cpuCores: 1 + memoryMB: 2048 + # Optional: Resize the root disk to this size. Must be bigger than the existing size + # Default is to leave the disk at the same size as the template + diskSizeGB: 10 + diskBusType: "paravirtual" + diskIOPS: 0 + storageProfile: "*" + operatingSystem: "ubuntu" + operatingSystemSpec: + distUpgradeOnBoot: false + versions: + kubelet: 1.22.5 diff --git a/go.mod b/go.mod index 77da41103..0de6b28b2 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7 github.com/sethvargo/go-password v0.2.0 github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda + github.com/vmware/go-vcloud-director/v2 v2.15.0 github.com/vmware/govmomi v0.23.1 golang.org/x/crypto v0.0.0-20220214200702-86341886e292 golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a @@ -73,6 +74,7 @@ require ( github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd // indirect github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect + github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/coreos/go-semver v0.3.0 // indirect @@ -98,12 +100,14 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/googleapis/gax-go/v2 v2.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/hashicorp/go-version v1.2.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kr/pretty v0.2.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/copystructure v1.0.0 // indirect @@ -114,6 +118,7 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect + github.com/peterhellberg/link v1.1.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect diff --git a/go.sum b/go.sum index ae24cfa09..50ef3075a 100644 --- a/go.sum +++ b/go.sum @@ -133,6 +133,8 @@ github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.m github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30/go.mod h1:4AJxUpXUhv4N+ziTvIcWWXgeorXpxPZOfk9HdEVr96M= +github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195 h1:c4mLfegoDw6OhSJXTd2jUEQgZUQuJWtocudb97Qn9EM= +github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -500,6 +502,7 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -717,6 +720,8 @@ github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtb github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/peterhellberg/link v1.1.0 h1:s2+RH8EGuI/mI4QwrWGSYQCRz7uNgip9BaM04HKu5kc= +github.com/peterhellberg/link v1.1.0/go.mod h1:gtSlOT4jmkY8P47hbTc8PTgiDDWpdPbFYl75keYyBB8= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -863,6 +868,8 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50 h1:uxE3GYdXIOfhMv3unJKETJEhw78gvzuQqRX/rVirc2A= github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= +github.com/vmware/go-vcloud-director/v2 v2.15.0 h1:idQ9NsHLr2dOSLBC8KIdBMq7XOvPiWmfxgWNaf580mk= +github.com/vmware/go-vcloud-director/v2 v2.15.0/go.mod h1:2BS1yw61VN34WI0/nUYoInFvBc3Zcuf84d4ESiAAl68= github.com/vmware/govmomi v0.23.1 h1:vU09hxnNR/I7e+4zCJvW+5vHu5dO64Aoe2Lw7Yi/KRg= github.com/vmware/govmomi v0.23.1/go.mod h1:Y+Wq4lst78L85Ge/F8+ORXIWiKYqaro1vhAulACy9Lc= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= diff --git a/pkg/cloudprovider/provider.go b/pkg/cloudprovider/provider.go index c238052dc..65f4eae7e 100644 --- a/pkg/cloudprovider/provider.go +++ b/pkg/cloudprovider/provider.go @@ -35,6 +35,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/nutanix" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/openstack" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/scaleway" + vcd "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vmware-cloud-director" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vsphere" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/providerconfig" @@ -104,6 +105,9 @@ var ( providerconfigtypes.CloudProviderNutanix: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return nutanix.New(cvr) }, + providerconfigtypes.CloudProviderVcloudDirector: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return vcd.New(cvr) + }, } ) diff --git a/pkg/cloudprovider/provider/vmware-cloud-director/client.go b/pkg/cloudprovider/provider/vmware-cloud-director/client.go new file mode 100644 index 000000000..3cd03ad83 --- /dev/null +++ b/pkg/cloudprovider/provider/vmware-cloud-director/client.go @@ -0,0 +1,172 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmwareclouddirector + +import ( + "errors" + "fmt" + "net/url" + "path" + "strings" + + "github.com/vmware/go-vcloud-director/v2/govcd" + "github.com/vmware/go-vcloud-director/v2/types/v56" + + cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" +) + +type Client struct { + Auth *Auth + VCDClient *govcd.VCDClient +} + +func NewClient(username, password, org, url, vdc string, allowInsecure bool) (*Client, error) { + client := Client{ + Auth: &Auth{ + Username: username, + Password: password, + Organization: org, + URL: url, + VDC: vdc, + AllowInsecure: allowInsecure, + }, + } + + vcdClient, err := client.GetAuthenticatedClient() + if err != nil { + return nil, err + } + + client.VCDClient = vcdClient + return &client, nil +} + +func (c *Client) GetAuthenticatedClient() (*govcd.VCDClient, error) { + // Ensure that all required fields for authentication are provided + // Fail early, without any API calls, if some required field is missing. + if c.Auth == nil { + return nil, fmt.Errorf("authentication configuration not provided") + } + if c.Auth.Username == "" { + return nil, fmt.Errorf("username not provided") + } + if c.Auth.Password == "" { + return nil, fmt.Errorf("password not provided") + } + if c.Auth.URL == "" { + return nil, fmt.Errorf("URL not provided") + } + if c.Auth.Organization == "" { + return nil, fmt.Errorf("organization name not provided") + } + + // Ensure that `/api` suffix exists in the cloud director URL. + apiEndpoint, err := url.Parse(c.Auth.URL) + if err != nil { + return nil, fmt.Errorf("unable to parse url '%s': %w", c.Auth.URL, err) + } + if !strings.HasSuffix(c.Auth.URL, "/api") { + apiEndpoint.Path = path.Join(apiEndpoint.Path, "api") + } + + vcdClient := govcd.NewVCDClient(*apiEndpoint, c.Auth.AllowInsecure) + + err = vcdClient.Authenticate(c.Auth.Username, c.Auth.Password, c.Auth.Organization) + if err != nil { + return nil, fmt.Errorf("failed to authenticate with VMware Cloud Director: %w", err) + } + + return vcdClient, nil +} + +func (c *Client) GetOrganization() (*govcd.Org, error) { + if c.Auth.Organization == "" { + return nil, errors.New("organization must be configured") + } + + org, err := c.VCDClient.GetOrgByNameOrId(c.Auth.Organization) + if err != nil { + return nil, fmt.Errorf("failed to get organization '%s': %w", c.Auth.Organization, err) + } + return org, err +} + +func (c *Client) GetVDCForOrg(org govcd.Org) (*govcd.Vdc, error) { + if c.Auth.VDC == "" { + return nil, errors.New("Organization VDC must be configured") + } + vcd, err := org.GetVDCByNameOrId(c.Auth.VDC, false) + if err != nil { + return nil, fmt.Errorf("failed to get Organization VDC '%s': %w", c.Auth.VDC, err) + } + return vcd, err +} + +func (c *Client) GetVMByName(vappName, vmName string) (*govcd.VM, error) { + _, _, vapp, err := c.GetOrganizationVDCAndVapp(vappName) + if err != nil { + return nil, err + } + + // We don't need ID here since we explicitly set the name field when creating the resource. + vm, err := vapp.GetVMByName(vmName, true) + if err != nil && errors.Is(err, govcd.ErrorEntityNotFound) { + return nil, cloudprovidererrors.ErrInstanceNotFound + } + return vm, err +} + +func (c *Client) GetOrganizationVDCAndVapp(vappName string) (*govcd.Org, *govcd.Vdc, *govcd.VApp, error) { + org, err := c.GetOrganization() + if err != nil { + return nil, nil, nil, err + } + + vdc, err := c.GetVDCForOrg(*org) + if err != nil { + return nil, nil, nil, err + } + + // Ensure that the vApp has already been created. + vapp, err := vdc.GetVAppByNameOrId(vappName, true) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get vApp '%s': %w", vappName, err) + } + return org, vdc, vapp, nil +} + +// GetVappNetworkType checks if the network exists and returns the network type. +func GetVappNetworkType(networkName string, vapp govcd.VApp) (NetworkType, error) { + networkConfig, err := vapp.GetNetworkConfig() + if err != nil { + return NoneNetworkType, fmt.Errorf("error getting vApp networks: %w", err) + } + + for _, netConfig := range networkConfig.NetworkConfig { + if netConfig.NetworkName == networkName || netConfig.ID == networkName { + switch { + case netConfig.NetworkName == types.NoneNetwork: + return NoneNetworkType, nil + case govcd.IsVappNetwork(netConfig.Configuration): + return VAppNetworkType, nil + default: + return OrgNetworkType, nil + } + } + } + return NoneNetworkType, fmt.Errorf("network '%s' not found: %w", networkName, err) +} diff --git a/pkg/cloudprovider/provider/vmware-cloud-director/helper.go b/pkg/cloudprovider/provider/vmware-cloud-director/helper.go new file mode 100644 index 000000000..0df79699f --- /dev/null +++ b/pkg/cloudprovider/provider/vmware-cloud-director/helper.go @@ -0,0 +1,287 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmwareclouddirector + +import ( + "encoding/base64" + "fmt" + "net/http" + "net/url" + "path" + + "github.com/vmware/go-vcloud-director/v2/govcd" + "github.com/vmware/go-vcloud-director/v2/types/v56" + vcdapitypes "github.com/vmware/go-vcloud-director/v2/types/v56" + + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + + "k8s.io/utils/pointer" +) + +var internalDiskBusTypes = map[string]string{ + "ide": "1", + "parallel": "3", + "sas": "4", + "paravirtual": "5", + "sata": "6", + "nvme": "7", +} + +func getComputePolicy(name string, policies []*govcd.VdcComputePolicy) *govcd.VdcComputePolicy { + for _, policy := range policies { + if policy.VdcComputePolicy == nil { + continue + } + if policy.VdcComputePolicy.Name == name || policy.VdcComputePolicy.ID == name { + return policy + } + } + return nil +} + +func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org *govcd.Org, vdc *govcd.Vdc, vapp *govcd.VApp) error { + // 1. We need the template HREF for the VM. + catalog, err := org.GetCatalogByNameOrId(c.Catalog, true) + if err != nil { + return fmt.Errorf("failed to get catalog '%s': %w", c.Catalog, err) + } + + // Catalog item can be a vApp template OVA or media ISO file. + catalogItem, err := catalog.GetCatalogItemByNameOrId(c.Template, true) + if err != nil { + return fmt.Errorf("failed to get catalog item '%s' in catalog '%s': %w", c.Template, c.Catalog, err) + } + + vAppTemplate, err := catalogItem.GetVAppTemplate() + if err != nil { + return fmt.Errorf("failed to get vApp template '%s' in catalog '%s': %w", c.Template, c.Catalog, err) + } + + templateHref := vAppTemplate.VAppTemplate.HREF + if vAppTemplate.VAppTemplate.Children != nil && len(vAppTemplate.VAppTemplate.Children.VM) != 0 { + templateHref = vAppTemplate.VAppTemplate.Children.VM[0].HREF + } + + // 2. Retrieve Sizing and Placement Compute Policy if required. + computePolicy := vcdapitypes.ComputePolicy{} + if c.SizingPolicy != nil || c.PlacementPolicy != nil { + allPolicies, err := org.GetAllVdcComputePolicies(url.Values{}) + if err != nil { + return fmt.Errorf("failed to get template all VDC compute policies: %w", err) + } + + if c.SizingPolicy != nil && *c.SizingPolicy != "" { + sizingPolicy := getComputePolicy(*c.SizingPolicy, allPolicies) + if sizingPolicy == nil { + return fmt.Errorf("sizing policy '%s' doesn't exist", *c.SizingPolicy) + } + computePolicy.VmSizingPolicy = &vcdapitypes.Reference{ + HREF: sizingPolicy.VdcComputePolicy.ID, + } + } + + if c.PlacementPolicy != nil && *c.PlacementPolicy != "" { + placementPolicy := getComputePolicy(*c.PlacementPolicy, allPolicies) + if placementPolicy == nil { + return fmt.Errorf("placement policy '%s' doesn't exist", *c.PlacementPolicy) + } + computePolicy.VmPlacementPolicy = &vcdapitypes.Reference{ + HREF: placementPolicy.VdcComputePolicy.ID, + } + } + } + + // 3. Retrieve Storage Profile + storageProfileRef := vcdapitypes.Reference{} + if c.StorageProfile != nil && *c.StorageProfile != defaultStorageProfile { + for _, sp := range vdc.Vdc.VdcStorageProfiles.VdcStorageProfile { + if sp.Name == *c.StorageProfile || sp.ID == *c.StorageProfile { + storageProfileRef = vcdapitypes.Reference{HREF: sp.HREF, Name: sp.Name, ID: sp.ID} + break + } + } + if storageProfileRef.HREF == "" { + if err != nil { + return fmt.Errorf("failed to get storage profile '%s': %w", *c.StorageProfile, err) + } + } + } + + // 4. At this point we are ready to create our initial VMs. + // + // Multiple API calls to re-compose the vApp are handled in a synchronous manner, where each request has to wait + // for the previous request to complete. This can cause a huge overhead in terms of time. + // + // It is not possible to customize compute, disk and network for a VM at initial creation time when we are using templates. So we rely on + // vApp re-composition to apply the needed customization, performed at later stages. + vAppRecomposition := &types.ReComposeVAppParams{ + Ovf: types.XMLNamespaceOVF, + Xsi: types.XMLNamespaceXSI, + Xmlns: types.XMLNamespaceVCloud, + Deploy: false, + Name: vapp.VApp.Name, + PowerOn: false, + Description: vapp.VApp.Description, + SourcedItem: &types.SourcedCompositionItemParam{ + Source: &types.Reference{ + HREF: templateHref, + Name: machine.Name, + }, + InstantiationParams: &types.InstantiationParams{ + NetworkConnectionSection: &vcdapitypes.NetworkConnectionSection{ + NetworkConnection: []*vcdapitypes.NetworkConnection{ + { + Network: c.Network, + NeedsCustomization: false, + IsConnected: true, + IPAddressAllocationMode: string(c.IPAllocationMode), + NetworkAdapterType: "VMXNET3", + }, + }, + }, + }, + }, + AllEULAsAccepted: true, + } + + // Add storage profile + if storageProfileRef.HREF != "" { + vAppRecomposition.SourcedItem.StorageProfile = &storageProfileRef + } + + // Add compute policy + if computePolicy.HREF != "" { + vAppRecomposition.SourcedItem.ComputePolicy = &computePolicy + } + + apiEndpoint, err := url.Parse(vapp.VApp.HREF) + if err != nil { + return fmt.Errorf("error getting vapp href '%s': %w", c.Auth.URL, err) + } + apiEndpoint.Path = path.Join(apiEndpoint.Path, "action/recomposeVApp") + + task, err := client.VCDClient.Client.ExecuteTaskRequest(apiEndpoint.String(), http.MethodPost, + types.MimeRecomposeVappParams, "error instantiating a new VM: %s", vAppRecomposition) + if err != nil { + return fmt.Errorf("unable to execute API call to create VM: %w", err) + } + + // Wait for VM to be created this should take around 1-3 minutes + if err = task.WaitTaskCompletion(); err != nil { + return fmt.Errorf("error waiting for VM creation task to complete: %w", err) + } + return nil +} + +func recomposeComputeAndDisk(config *Config, vm *govcd.VM) (*govcd.VM, error) { + needsComputeRecomposition := false + needsDiskRecomposition := false + // Perform compute recomposition if SizingPolicy was not specified. + vmSpecSection := vm.VM.VmSpecSection + if config.SizingPolicy == nil || *config.SizingPolicy == "" { + vmSpecSection.MemoryResourceMb.Configured = config.MemoryMB + vmSpecSection.NumCpus = pointer.Int(int(config.CPUs)) + vmSpecSection.NumCoresPerSocket = pointer.Int(int(config.CPUCores)) + needsComputeRecomposition = true + } + + // Perform disk recomposition if required. + if vmSpecSection.DiskSection != nil { + for i, internalDisk := range vmSpecSection.DiskSection.DiskSettings { + // We are only concerned with template disk and not named/independent disks. + if internalDisk.Disk == nil { + if config.DiskSizeGB != nil && *config.DiskSizeGB > 0 { + vmSpecSection.DiskSection.DiskSettings[i].SizeMb = (*config.DiskSizeGB) * 1024 + needsDiskRecomposition = true + } + if config.DiskIOPS != nil && *config.DiskIOPS > 0 { + vmSpecSection.DiskSection.DiskSettings[i].Iops = pointer.Int64(*config.DiskIOPS) + needsDiskRecomposition = true + } + if config.DiskBusType != nil && *config.DiskBusType != "" { + vmSpecSection.DiskSection.DiskSettings[i].AdapterType = internalDiskBusTypes[*config.DiskBusType] + needsDiskRecomposition = true + } + } + } + } + + if !needsDiskRecomposition { + // Update treats same values as changes and fails. Although if set to nil, it assumes that no changes are required for this field. + vmSpecSection.DiskSection = nil + } + + var err error + // Execute disk and compute recomposition on our VM + if needsComputeRecomposition || needsDiskRecomposition { + description := vm.VM.Description + vm, err = vm.UpdateVmSpecSection(vmSpecSection, description) + if err != nil { + return nil, fmt.Errorf("error updating VM spec section: %w", err) + } + } + return vm, nil +} + +func setUserData(userdata string, vm *govcd.VM, providerConfig *providerconfigtypes.Config) error { + userdataBase64 := base64.StdEncoding.EncodeToString([]byte(userdata)) + props := map[string]string{ + "user-data": userdataBase64, + "disk.enableUUID": "1", + "instance-id": vm.VM.Name, + } + + vmProperties := &vcdapitypes.ProductSectionList{ + ProductSection: &vcdapitypes.ProductSection{ + Info: "Custom properties", + Property: []*vcdapitypes.Property{}, + }, + } + for key, value := range props { + property := &vcdapitypes.Property{ + UserConfigurable: true, + Type: "string", + Key: key, + Label: key, + Value: &vcdapitypes.Value{Value: value}, + } + vmProperties.ProductSection.Property = append(vmProperties.ProductSection.Property, property) + } + + // Set guest properties on the VM + _, err := vm.SetProductSectionList(vmProperties) + if err != nil { + return fmt.Errorf("error setting guest properties for VM: %w", err) + } + return nil +} + +func addMetadata(vm *govcd.VM, metadata *map[string]string) error { + // Nothing to do here. + if metadata == nil { + return nil + } + + for key, val := range *metadata { + err := vm.AddMetadataEntry(vcdapitypes.MetadataStringValue, key, val) + if err != nil { + return fmt.Errorf("error adding metadata for VM: %w", err) + } + } + return nil +} diff --git a/pkg/cloudprovider/provider/vmware-cloud-director/provider.go b/pkg/cloudprovider/provider/vmware-cloud-director/provider.go new file mode 100644 index 000000000..b24543d93 --- /dev/null +++ b/pkg/cloudprovider/provider/vmware-cloud-director/provider.go @@ -0,0 +1,554 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vmwareclouddirector + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + + "github.com/vmware/go-vcloud-director/v2/govcd" + + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" + vcdtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vmware-cloud-director/types" + cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "github.com/kubermatic/machine-controller/pkg/providerconfig" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/utils/pointer" +) + +const ( + defaultDiskType = "paravirtual" + defaultStorageProfile = "*" + defaultDiskIOPS = 0 +) + +type NetworkType string + +const ( + VAppNetworkType NetworkType = "vapp" + OrgNetworkType NetworkType = "org" + // Network with a NIC that is not attached to any network. + NoneNetworkType NetworkType = "none" +) + +type provider struct { + configVarResolver *providerconfig.ConfigVarResolver +} + +type Auth struct { + Username string + Password string + Organization string + URL string + VDC string + AllowInsecure bool +} + +type Config struct { + Auth `json:",inline"` + + // VM configuration. + VApp string + Template string + Catalog string + PlacementPolicy *string + SizingPolicy *string + + // Network configuration. + Network string + IPAllocationMode vcdtypes.IPAllocationMode + + // Compute configuration. + CPUs int64 + CPUCores int64 + MemoryMB int64 + + // Storage configuration. + DiskSizeGB *int64 + DiskBusType *string + DiskIOPS *int64 + StorageProfile *string + + // Metadata configuration. + Metadata *map[string]string +} + +// New returns a VMware Cloud Director provider. +func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return &provider{configVarResolver: configVarResolver} +} + +// Ensures that Server implements Instance interface. +var _ instance.Instance = &Server{} + +// Server holds VMware Cloud Director VM information. +type Server struct { + name string + id string + status instance.Status + addresses map[string]corev1.NodeAddressType +} + +func (s Server) Name() string { + return s.name +} + +func (s Server) ID() string { + return s.id +} + +func (s Server) Addresses() map[string]corev1.NodeAddressType { + return s.addresses +} + +func (s Server) Status() instance.Status { + return s.status +} + +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { + _, _, rawConfig, err := p.getConfig(spec.ProviderSpec) + if err != nil { + return spec, err + } + + // Set defaults. + if rawConfig.IPAllocationMode == "" { + rawConfig.IPAllocationMode = vcdtypes.DHCPIPAllocationMode + } + + // These defaults will have no effect if DiskSizeGB is not specified + if rawConfig.DiskBusType == nil { + rawConfig.DiskBusType = pointer.String(defaultDiskType) + } + if rawConfig.DiskIOPS == nil { + rawConfig.DiskIOPS = pointer.Int64(defaultDiskIOPS) + } + spec.ProviderSpec.Value, err = setProviderSpec(*rawConfig, spec.ProviderSpec) + return spec, err +} + +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { + c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return false, fmt.Errorf("failed to parse config: %w", err) + } + + client, err := NewClient(c.Username, c.Password, c.Organization, c.URL, c.VDC, c.AllowInsecure) + if err != nil { + return false, fmt.Errorf("failed to create VMware Cloud Director client: %w", err) + } + + vm, err := client.GetVMByName(c.VApp, machine.Name) + if err != nil { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { + return true, nil + } + return false, err + } + + vmStatus, err := vm.GetStatus() + if err != nil { + return false, fmt.Errorf("failed to get VM status: %w", err) + } + + // Turn off VM if it's `ON` + if vmStatus == "POWERED_ON" { + task, err := vm.PowerOff() + if err != nil { + return false, fmt.Errorf("failed to turn off VM: %w", err) + } + if err = task.WaitTaskCompletion(); err != nil { + return false, fmt.Errorf("error waiting for VM power off task to complete: %w", err) + } + } + + if err := vm.Delete(); err != nil { + return false, fmt.Errorf("failed to destroy vm %s: %w", vm.VM.Name, err) + } + return true, nil +} + +func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { + vm, err := p.create(ctx, machine, userdata) + if err != nil { + _, cleanupErr := p.Cleanup(ctx, machine, data) + if cleanupErr != nil { + return nil, fmt.Errorf("cleaning up failed with err %v after creation failed with err %w", cleanupErr, err) + } + return nil, err + } + return vm, nil +} + +func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { + c, providerConfig, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return nil, fmt.Errorf("failed to parse config: %w", err) + } + + client, err := NewClient(c.Username, c.Password, c.Organization, c.URL, c.VDC, c.AllowInsecure) + if err != nil { + return nil, fmt.Errorf("failed to create VMware Cloud Director client: %w", err) + } + + // Fetch the organization, VDC, and vApp resources. + org, vdc, vapp, err := client.GetOrganizationVDCAndVapp(c.VApp) + if err != nil { + return nil, err + } + + // 1. Create Standalone VM from template. + err = createVM(client, machine, c, org, vdc, vapp) + if err != nil { + return nil, fmt.Errorf("failed to create VM: %w", err) + } + + // 2. Fetch updated vApp + err = vapp.Refresh() + if err != nil { + return nil, fmt.Errorf("failed to get updated vApp '%s' after recompoisition: %w", c.VApp, err) + } + + // 3. Fetch updated VM + vm, err := vapp.GetVMByName(machine.Name, true) + if err != nil { + return nil, err + } + + // 4. Perform VM recomposition for compute and disks + vm, err = recomposeComputeAndDisk(c, vm) + if err != nil { + return nil, err + } + + // 5. Before powering on the VM, configure customization to attach userdata with the VM + // update guest properties. + err = setUserData(userdata, vm, providerConfig) + if err != nil { + return nil, err + } + + // 6. Fetch updated VM. + err = vm.Refresh() + if err != nil { + return nil, err + } + + // 7. Add Metadata to VM. + err = addMetadata(vm, c.Metadata) + if err != nil { + return nil, err + } + + // 8. Finally power on the VM after performing all required actions. + task, err := vm.PowerOn() + if err != nil { + return nil, fmt.Errorf("failed to turn on VM: %w", err) + } + if err = task.WaitTaskCompletion(); err != nil { + return nil, fmt.Errorf("error waiting for VM bootstrap to complete: %w", err) + } + + return p.getInstance(vm) +} + +func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { + c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return nil, fmt.Errorf("failed to parse config: %w", err) + } + + client, err := NewClient(c.Username, c.Password, c.Organization, c.URL, c.VDC, c.AllowInsecure) + if err != nil { + return nil, fmt.Errorf("failed to create VMware Cloud Director client: %w", err) + } + + vm, err := client.GetVMByName(c.VApp, machine.Name) + if err != nil { + return nil, err + } + + return p.getInstance(vm) +} + +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { + return "", "", nil +} + +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *vcdtypes.RawConfig, error) { + if provSpec.Value == nil { + return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") + } + + pconfig, err := providerconfigtypes.GetConfig(provSpec) + if err != nil { + return nil, nil, nil, err + } + + if pconfig.OperatingSystemSpec.Raw == nil { + return nil, nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") + } + + rawConfig, err := vcdtypes.GetConfig(*pconfig) + if err != nil { + return nil, nil, nil, err + } + + c := Config{} + c.Username, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Username, "VCD_USER") + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get the value of \"username\" field, error = %w", err) + } + + c.Password, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Password, "VCD_PASSWORD") + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get the value of \"password\" field, error = %w", err) + } + + c.Organization, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Organization, "VCD_ORG") + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get the value of \"organization\" field, error = %w", err) + } + + c.URL, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.URL, "VCD_URL") + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get the value of \"url\" field, error = %w", err) + } + + c.VDC, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.VDC, "VCD_VDC") + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get the value of \"vdc\" field, error = %w", err) + } + + c.AllowInsecure, err = p.configVarResolver.GetConfigVarBoolValueOrEnv(rawConfig.AllowInsecure, "VCD_ALLOW_UNVERIFIED_SSL") + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get the value of \"allowInsecure\" field, error = %w", err) + } + + c.VApp, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VApp) + if err != nil { + return nil, nil, nil, err + } + + c.Template, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Template) + if err != nil { + return nil, nil, nil, err + } + + c.Catalog, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Catalog) + if err != nil { + return nil, nil, nil, err + } + + c.Network, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Network) + if err != nil { + return nil, nil, nil, err + } + + c.IPAllocationMode = rawConfig.IPAllocationMode + + if rawConfig.DiskSizeGB != nil && *rawConfig.DiskSizeGB < 0 { + return nil, nil, nil, fmt.Errorf("value for \"diskSizeGB\" should either be nil or greater than or equal to 0") + } + c.DiskSizeGB = rawConfig.DiskSizeGB + + if rawConfig.DiskIOPS != nil && *rawConfig.DiskIOPS < 0 { + return nil, nil, nil, fmt.Errorf("value for \"diskIOPS\" should either be nil or greater than or equal to 0") + } + c.DiskIOPS = rawConfig.DiskIOPS + + if rawConfig.CPUs <= 0 { + return nil, nil, nil, fmt.Errorf("value for \"cpus\" should be greater than 0") + } + c.CPUs = rawConfig.CPUs + + if rawConfig.CPUCores <= 0 { + return nil, nil, nil, fmt.Errorf("value for \"cpuCores\" should be greater than 0") + } + c.CPUCores = rawConfig.CPUCores + + if rawConfig.MemoryMB <= 4 { + return nil, nil, nil, fmt.Errorf("value for \"memoryMB\" should be greater than 0") + } + if rawConfig.MemoryMB%4 != 0 { + return nil, nil, nil, fmt.Errorf("value for \"memoryMB\" should be a multiple of 4") + } + c.MemoryMB = rawConfig.MemoryMB + + c.DiskBusType = rawConfig.DiskBusType + c.StorageProfile = rawConfig.StorageProfile + c.Metadata = rawConfig.Metadata + return &c, pconfig, rawConfig, err +} + +func (p *provider) getInstance(vm *govcd.VM) (instance.Instance, error) { + vmStatus, err := vm.GetStatus() + if err != nil { + return nil, fmt.Errorf("failed to get VM status: %w", err) + } + + var status instance.Status + + switch vmStatus { + case "POWERED_ON": + status = instance.StatusRunning + case "POWERED_OFF", "PARTIALLY_POWERED_OFF": + status = instance.StatusCreating + default: + status = instance.StatusUnknown + } + + addresses := make(map[string]corev1.NodeAddressType) + if vm.VM.NetworkConnectionSection != nil && vm.VM.NetworkConnectionSection.NetworkConnection != nil { + for _, nic := range vm.VM.NetworkConnectionSection.NetworkConnection { + if nic.ExternalIPAddress != "" { + addresses[nic.ExternalIPAddress] = corev1.NodeExternalIP + } + if nic.IPAddress != "" { + addresses[nic.IPAddress] = corev1.NodeInternalIP + } + } + } + + return Server{name: vm.VM.Name, status: status, addresses: addresses, id: vm.VM.ID}, nil +} + +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { + labels := make(map[string]string) + + c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err == nil { + labels["size"] = fmt.Sprintf("%d-cpus-%d-mb", c.CPUs, c.MemoryMB) + labels["vapp"] = c.VApp + labels["vdc"] = c.VDC + labels["organization"] = c.Organization + } + + return labels, err +} + +func (p *provider) MigrateUID(_ context.Context, _ *clusterv1alpha1.Machine, _ types.UID) error { + return nil +} + +func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { + return nil +} + +func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { + c, _, _, err := p.getConfig(spec.ProviderSpec) + if err != nil { + return fmt.Errorf("failed to parse config: %w", err) + } + + client, err := NewClient(c.Username, c.Password, c.Organization, c.URL, c.VDC, c.AllowInsecure) + if err != nil { + return fmt.Errorf("failed to create VMware Cloud Director client: %w", err) + } + + // Ensure that the organization, VDC, and vApp exists. + org, vdc, vapp, err := client.GetOrganizationVDCAndVapp(c.VApp) + if err != nil { + return err + } + + // Ensure that the catalog exists. + catalog, err := org.GetCatalogByNameOrId(c.Catalog, true) + if err != nil { + return fmt.Errorf("failed to get catalog '%s': %w", c.Catalog, err) + } + + // Ensure that the template exists in the catalog + // Catalog item can be a vApp template OVA or media ISO file. + catalogItem, err := catalog.GetCatalogItemByNameOrId(c.Template, true) + if err != nil { + return fmt.Errorf("failed to get template '%s' in catalog '%s': %w", c.Template, c.Catalog, err) + } + if c.DiskSizeGB != nil && catalogItem.CatalogItem.Size > *c.DiskSizeGB { + return fmt.Errorf("diskSizeGB '%v' cannot be less than the template size '%v': %w", *c.DiskSizeGB, catalogItem.CatalogItem.Size, err) + } + + // Ensure that the network exists + // It can either be a vApp network or a vApp Org network. + _, err = GetVappNetworkType(c.Network, *vapp) + if err != nil { + return fmt.Errorf("failed to get network '%s' for vapp '%s': %w", c.Network, c.VApp, err) + } + + if c.SizingPolicy != nil || c.PlacementPolicy != nil { + allPolicies, err := org.GetAllVdcComputePolicies(url.Values{}) + if err != nil { + return fmt.Errorf("failed to get template all VDC compute policies: %w", err) + } + + if c.SizingPolicy != nil && *c.SizingPolicy != "" { + sizingPolicy := getComputePolicy(*c.SizingPolicy, allPolicies) + if sizingPolicy == nil { + return fmt.Errorf("sizing policy '%s' doesn't exist", *c.SizingPolicy) + } + } + + if c.PlacementPolicy != nil && *c.PlacementPolicy != "" { + placementPolicy := getComputePolicy(*c.PlacementPolicy, allPolicies) + if placementPolicy == nil { + return fmt.Errorf("placement policy '%s' doesn't exist", *c.SizingPolicy) + } + } + } + + // Ensure that the storage profile exists. + if c.StorageProfile != nil && *c.StorageProfile != defaultStorageProfile { + _, err = vdc.FindStorageProfileReference(*c.StorageProfile) + if err != nil { + return fmt.Errorf("failed to get storage profile '%s': %w", *c.StorageProfile, err) + } + } + return nil +} + +func setProviderSpec(rawConfig vcdtypes.RawConfig, provSpec clusterv1alpha1.ProviderSpec) (*runtime.RawExtension, error) { + if provSpec.Value == nil { + return nil, fmt.Errorf("machine.spec.providerconfig.value is nil") + } + + pconfig, err := providerconfigtypes.GetConfig(provSpec) + if err != nil { + return nil, err + } + + rawCloudProviderSpec, err := json.Marshal(rawConfig) + if err != nil { + return nil, err + } + + pconfig.CloudProviderSpec = runtime.RawExtension{Raw: rawCloudProviderSpec} + rawPconfig, err := json.Marshal(pconfig) + if err != nil { + return nil, err + } + + return &runtime.RawExtension{Raw: rawPconfig}, nil +} diff --git a/pkg/cloudprovider/provider/vmware-cloud-director/types/types.go b/pkg/cloudprovider/provider/vmware-cloud-director/types/types.go new file mode 100644 index 000000000..c8c5aa3de --- /dev/null +++ b/pkg/cloudprovider/provider/vmware-cloud-director/types/types.go @@ -0,0 +1,71 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" +) + +type IPAllocationMode string + +const ( + PoolIPAllocationMode IPAllocationMode = "POOL" + DHCPIPAllocationMode IPAllocationMode = "DHCP" +) + +// RawConfig represents VMware Cloud Director specific configuration. +type RawConfig struct { + // Provider configuration. + Username providerconfigtypes.ConfigVarString `json:"username"` + Password providerconfigtypes.ConfigVarString `json:"password"` + Organization providerconfigtypes.ConfigVarString `json:"organization"` + URL providerconfigtypes.ConfigVarString `json:"url"` + VDC providerconfigtypes.ConfigVarString `json:"vdc"` + AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` + + // VM configuration. + VApp providerconfigtypes.ConfigVarString `json:"vapp"` + Template providerconfigtypes.ConfigVarString `json:"template"` + Catalog providerconfigtypes.ConfigVarString `json:"catalog"` + PlacementPolicy *string `json:"placementPolicy,omitempty"` + + // Network configuration. + Network providerconfigtypes.ConfigVarString `json:"network"` + IPAllocationMode IPAllocationMode `json:"ipAllocationMode"` + + // Compute configuration. + CPUs int64 `json:"cpus"` + CPUCores int64 `json:"cpuCores"` + MemoryMB int64 `json:"memoryMB"` + SizingPolicy *string `json:"sizingPolicy,omitempty"` + + // Storage configuration. + DiskSizeGB *int64 `json:"diskSizeGB,omitempty"` + DiskBusType *string `json:"diskBusType,omitempty"` + DiskIOPS *int64 `json:"diskIOPS,omitempty"` + StorageProfile *string `json:"storageProfile,omitempty"` + + // Metadata configuration. + Metadata *map[string]string `json:"metadata,omitempty"` +} + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index db36990aa..1d47efeb7 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -205,6 +205,10 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return fmt.Errorf("failed to get config: %w", err) } + if pc.OperatingSystem == providerconfigtypes.OperatingSystemSLES { + return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, providerconfigtypes.ErrOSNotSupported) + } + session, err := NewSession(ctx, config) if err != nil { return fmt.Errorf("failed to create vCenter session: %w", err) @@ -279,9 +283,6 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return err } } - if pc.OperatingSystem == providerconfigtypes.OperatingSystemSLES { - return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, providerconfigtypes.ErrOSNotSupported) - } return nil } diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 8584746d6..1de43ec62 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -46,24 +46,25 @@ const ( type CloudProvider string const ( - CloudProviderAWS CloudProvider = "aws" - CloudProviderAzure CloudProvider = "azure" - CloudProviderDigitalocean CloudProvider = "digitalocean" - CloudProviderGoogle CloudProvider = "gce" - CloudProviderEquinixMetal CloudProvider = "equinixmetal" - CloudProviderPacket CloudProvider = "packet" - CloudProviderHetzner CloudProvider = "hetzner" - CloudProviderKubeVirt CloudProvider = "kubevirt" - CloudProviderLinode CloudProvider = "linode" - CloudProviderNutanix CloudProvider = "nutanix" - CloudProviderOpenstack CloudProvider = "openstack" - CloudProviderVsphere CloudProvider = "vsphere" - CloudProviderFake CloudProvider = "fake" - CloudProviderAlibaba CloudProvider = "alibaba" - CloudProviderAnexia CloudProvider = "anexia" - CloudProviderScaleway CloudProvider = "scaleway" - CloudProviderBaremetal CloudProvider = "baremetal" - CloudProviderExternal CloudProvider = "external" + CloudProviderAWS CloudProvider = "aws" + CloudProviderAzure CloudProvider = "azure" + CloudProviderDigitalocean CloudProvider = "digitalocean" + CloudProviderGoogle CloudProvider = "gce" + CloudProviderEquinixMetal CloudProvider = "equinixmetal" + CloudProviderPacket CloudProvider = "packet" + CloudProviderHetzner CloudProvider = "hetzner" + CloudProviderKubeVirt CloudProvider = "kubevirt" + CloudProviderLinode CloudProvider = "linode" + CloudProviderNutanix CloudProvider = "nutanix" + CloudProviderOpenstack CloudProvider = "openstack" + CloudProviderVsphere CloudProvider = "vsphere" + CloudProviderVcloudDirector CloudProvider = "vmware-cloud-director" + CloudProviderFake CloudProvider = "fake" + CloudProviderAlibaba CloudProvider = "alibaba" + CloudProviderAnexia CloudProvider = "anexia" + CloudProviderScaleway CloudProvider = "scaleway" + CloudProviderBaremetal CloudProvider = "baremetal" + CloudProviderExternal CloudProvider = "external" ) var ( @@ -94,6 +95,7 @@ var ( CloudProviderNutanix, CloudProviderOpenstack, CloudProviderVsphere, + CloudProviderVcloudDirector, CloudProviderFake, CloudProviderAlibaba, CloudProviderAnexia, diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index b3e778aae..6c2f59f88 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -222,7 +222,7 @@ write_files: socat \ wget \ curl \ - {{- if eq .CloudProviderName "vsphere" }} + {{- if or (eq .CloudProviderName "vsphere") (eq .CloudProviderName "vmware-cloud-director") }} open-vm-tools \ {{- end }} ipvsadm diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index 17b497a43..a67257ec9 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -230,7 +230,7 @@ write_files: socat \ wget \ curl \ - {{- if eq .CloudProviderName "vsphere" }} + {{- if or (eq .CloudProviderName "vsphere") (eq .CloudProviderName "vmware-cloud-director") }} open-vm-tools \ {{- end }} {{- if eq .CloudProviderName "nutanix" }} diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 4d9d123b7..2a1647e43 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -224,7 +224,7 @@ write_files: socat \ wget \ curl \ - {{- if eq .CloudProviderName "vsphere" }} + {{- if or (eq .CloudProviderName "vsphere") (eq .CloudProviderName "vmware-cloud-director") }} open-vm-tools \ {{- end }} {{- if eq .CloudProviderName "nutanix" }} diff --git a/pkg/userdata/rockylinux/provider.go b/pkg/userdata/rockylinux/provider.go index 44a406094..4fb5cea34 100644 --- a/pkg/userdata/rockylinux/provider.go +++ b/pkg/userdata/rockylinux/provider.go @@ -226,7 +226,7 @@ write_files: wget \ curl \ tar \ - {{- if eq .CloudProviderName "vsphere" }} + {{- if or (eq .CloudProviderName "vsphere") (eq .CloudProviderName "vmware-cloud-director") }} open-vm-tools \ {{- end }} {{- if eq .CloudProviderName "nutanix" }} diff --git a/pkg/userdata/sles/provider.go b/pkg/userdata/sles/provider.go index 6299c672c..50b14f888 100644 --- a/pkg/userdata/sles/provider.go +++ b/pkg/userdata/sles/provider.go @@ -187,7 +187,7 @@ write_files: e2fsprogs \ jq \ socat \ - {{- if eq .CloudProviderName "vsphere" }} + {{- if or (eq .CloudProviderName "vsphere") (eq .CloudProviderName "vmware-cloud-director") }} open-vm-tools \ {{- end }} ipvsadm diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index cccb7c7da..dedbcd1c0 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -217,7 +217,7 @@ write_files: nfs-common \ socat \ util-linux \ - {{- if eq .CloudProviderName "vsphere" }} + {{- if or (eq .CloudProviderName "vsphere") (eq .CloudProviderName "vmware-cloud-director") }} open-vm-tools \ {{- end }} {{- if eq .CloudProviderName "nutanix" }} diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index e725f54dd..43f585d74 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -63,6 +63,7 @@ const ( GCEManifest = "./testdata/machinedeployment-gce.yaml" HZManifest = "./testdata/machinedeployment-hetzner.yaml" LinodeManifest = "./testdata/machinedeployment-linode.yaml" + VMwareCloudDirectorManifest = "./testdata/machinedeployment-vmware-cloud-director.yaml" VSPhereManifest = "./testdata/machinedeployment-vsphere.yaml" VSPhereDSCManifest = "./testdata/machinedeployment-vsphere-datastore-cluster.yaml" VSPhereResourcePoolManifest = "./testdata/machinedeployment-vsphere-resource-pool.yaml" @@ -790,6 +791,38 @@ func TestLinodeProvisioningE2E(t *testing.T) { runScenarios(t, selector, params, LinodeManifest, fmt.Sprintf("linode-%s", *testRunIdentifier)) } +func getVMwareCloudDirectorTestParams(t *testing.T) []string { + // test data + password := os.Getenv("VCD_PASSWORD") + username := os.Getenv("VCD_USER") + organization := os.Getenv("VCD_ORG") + url := os.Getenv("VCD_URL") + vdc := os.Getenv("VCD_VDC") + + if password == "" || username == "" || organization == "" || url == "" || vdc == "" { + t.Fatal("unable to run the test suite, VCD_PASSWORD, VCD_USER, VCD_ORG, " + + "VCD_URL, or VCD_VDC environment variables cannot be empty") + } + + // set up parameters + params := []string{fmt.Sprintf("<< VCD_PASSWORD >>=%s", password), + fmt.Sprintf("<< VCD_USER >>=%s", username), + fmt.Sprintf("<< VCD_ORG >>=%s", organization), + fmt.Sprintf("<< VCD_URL >>=%s", url), + fmt.Sprintf("<< VCD_VDC >>=%s", vdc), + } + return params +} + +func TestVMwareCloudDirectorProvisioningE2E(t *testing.T) { + t.Parallel() + + selector := OsSelector("ubuntu") + params := getVMwareCloudDirectorTestParams(t) + + runScenarios(t, selector, params, VMwareCloudDirectorManifest, fmt.Sprintf("vcd-%s", *testRunIdentifier)) +} + func getVSphereTestParams(t *testing.T) []string { // test data vsPassword := os.Getenv("VSPHERE_E2E_PASSWORD") diff --git a/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml b/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml new file mode 100644 index 000000000..969dfb3d5 --- /dev/null +++ b/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml @@ -0,0 +1,58 @@ +apiVersion: "cluster.k8s.io/v1alpha1" +kind: MachineDeployment +metadata: + name: << MACHINE_NAME >> + namespace: kube-system +spec: + paused: false + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + name: << MACHINE_NAME >> + template: + metadata: + labels: + name: << MACHINE_NAME >> + spec: + providerSpec: + value: + sshPublicKeys: + - "<< YOUR_PUBLIC_KEY >>" + cloudProvider: "vmware-cloud-director" + cloudProviderSpec: + username: "<< VCD_USER >>" + url: "<< VCD_URL >>" + password: "<< VCD_PASSWORD >>" + organization: "<< VCD_ORG >>" + vdc: "<< VCD_VDC >>" + allowInsecure: false + vapp: "machine-controller-e2e" + catalog: "kubermatic" + template: "machine-controller-<< OS_NAME >>" + network: "machine-controller-e2e" + ipAllocationMode: "DHCP" + cpus: 2 + cpuCores: 1 + memoryMB: 2048 + diskSizeGB: << DISK_SIZE >> + diskBusType: "paravirtual" + diskIOPS: 0 + metadata: + key: value + operatingSystem: "<< OS_NAME >>" + operatingSystemSpec: + distUpgradeOnBoot: false + disableAutoUpdate: true + attachSubscription: false + # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` + rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" + # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` + rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" + rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" + versions: + kubelet: "<< KUBERNETES_VERSION >>" From 17f70818168eabe11165497640894e4c34cfaa41 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 25 May 2022 20:32:15 +0500 Subject: [PATCH 155/489] vcd: set computer name for VM (#1311) Signed-off-by: Waleed Malik --- .../provider/vmware-cloud-director/helper.go | 15 +++++++++++++++ .../provider/vmware-cloud-director/provider.go | 8 +++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/vmware-cloud-director/helper.go b/pkg/cloudprovider/provider/vmware-cloud-director/helper.go index 0df79699f..48dcdc8f1 100644 --- a/pkg/cloudprovider/provider/vmware-cloud-director/helper.go +++ b/pkg/cloudprovider/provider/vmware-cloud-director/helper.go @@ -285,3 +285,18 @@ func addMetadata(vm *govcd.VM, metadata *map[string]string) error { } return nil } + +func setComputerName(vm *govcd.VM, machineName string) error { + customizationSection, err := vm.GetGuestCustomizationSection() + if err != nil { + return fmt.Errorf("error retrieving guest customization section for VM: %w", err) + } + + customizationSection.ComputerName = machineName + + _, err = vm.SetGuestCustomizationSection(customizationSection) + if err != nil { + return fmt.Errorf("error adding metadata for VM: %w", err) + } + return nil +} diff --git a/pkg/cloudprovider/provider/vmware-cloud-director/provider.go b/pkg/cloudprovider/provider/vmware-cloud-director/provider.go index b24543d93..b01e1149b 100644 --- a/pkg/cloudprovider/provider/vmware-cloud-director/provider.go +++ b/pkg/cloudprovider/provider/vmware-cloud-director/provider.go @@ -263,7 +263,13 @@ func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, err } - // 8. Finally power on the VM after performing all required actions. + // 8. Set computer name for the VM + err = setComputerName(vm, machine.Name) + if err != nil { + return nil, err + } + + // 9. Finally power on the VM after performing all required actions. task, err := vm.PowerOn() if err != nil { return nil, fmt.Errorf("failed to turn on VM: %w", err) From f25294a9cabf0c41b9283b6412786f01e90f9f1b Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Thu, 26 May 2022 15:26:26 +0200 Subject: [PATCH 156/489] Block setting spec.ConfigSource for Kubernetes 1.24+ (#1312) * Block setting spec.ConfigSource for Kubernetes 1.24+ Signed-off-by: Marvin Beckers * Incorporate PR feedback Signed-off-by: Marvin Beckers --- pkg/admission/machines.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index 978dd5d98..295ebb135 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -139,6 +139,18 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec return fmt.Errorf("kubernetes version constraint didn't allow %q kubelet version", kubeletVer) } + // Do not allow 1.24+ to use config source (dynamic kubelet configuration) + constraint124, err := semver.NewConstraint(">= 1.24") + if err != nil { + return fmt.Errorf("failed to parse 1.24 constraint: %w", err) + } + + if constraint124.Check(kubeletVer) { + if spec.ConfigSource != nil { + return fmt.Errorf("setting spec.ConfigSource is not allowed for kubelet version %q", kubeletVer) + } + } + // Validate SSH keys if err := validatePublicKeys(providerConfig.SSHPublicKeys); err != nil { return fmt.Errorf("Invalid public keys specified: %w", err) From c6b2bfc272fcdec98f908031d60d9b243a32895f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Thu, 26 May 2022 19:40:20 +0200 Subject: [PATCH 157/489] Load IPv6 address on Azure only if IPFamily is IPv6 or Dualstack (#1315) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- pkg/cloudprovider/provider/azure/provider.go | 29 +++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 3313d3ae5..b5b8e7003 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -365,7 +365,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p return &c, pconfig, nil } -func getVMIPAddresses(ctx context.Context, c *config, vm *compute.VirtualMachine) (map[string]v1.NodeAddressType, error) { +func getVMIPAddresses(ctx context.Context, c *config, vm *compute.VirtualMachine, ipFamily util.IPFamily) (map[string]v1.NodeAddressType, error) { var ( ipAddresses = map[string]v1.NodeAddressType{} err error @@ -390,8 +390,8 @@ func getVMIPAddresses(ctx context.Context, c *config, vm *compute.VirtualMachine splitIfaceID := strings.Split(*iface.ID, "/") ifaceName := splitIfaceID[len(splitIfaceID)-1] - ipAddresses, err = getNICIPAddresses(ctx, c, ifaceName) - if vm.NetworkProfile.NetworkInterfaces == nil { + ipAddresses, err = getNICIPAddresses(ctx, c, ipFamily, ifaceName) + if err != nil || vm.NetworkProfile.NetworkInterfaces == nil { return nil, fmt.Errorf("failed to get addresses for interface %q: %w", ifaceName, err) } } @@ -399,7 +399,7 @@ func getVMIPAddresses(ctx context.Context, c *config, vm *compute.VirtualMachine return ipAddresses, nil } -func getNICIPAddresses(ctx context.Context, c *config, ifaceName string) (map[string]v1.NodeAddressType, error) { +func getNICIPAddresses(ctx context.Context, c *config, ipFamily util.IPFamily, ifaceName string) (map[string]v1.NodeAddressType, error) { ifClient, err := getInterfacesClient(c) if err != nil { return nil, fmt.Errorf("failed to create interfaces client: %w", err) @@ -438,12 +438,14 @@ func getNICIPAddresses(ctx context.Context, c *config, ifaceName string) (map[st ipAddresses[ip] = v1.NodeExternalIP } - publicIP6s, err := getIPAddressStrings(ctx, c, publicIPv6Name(ifaceName)) - if err != nil { - return nil, fmt.Errorf("failed to retrieve IP string for IP %q: %w", name, err) - } - for _, ip := range publicIP6s { - ipAddresses[ip] = v1.NodeExternalIP + if ipFamily == util.DualStack || ipFamily == util.IPv6 { + publicIP6s, err := getIPAddressStrings(ctx, c, publicIPv6Name(ifaceName)) + if err != nil { + return nil, fmt.Errorf("failed to retrieve IP string for IP %q: %w", name, err) + } + for _, ip := range publicIP6s { + ipAddresses[ip] = v1.NodeExternalIP + } } } @@ -707,7 +709,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, fmt.Errorf("failed to retrieve updated data for VM %q: %w", machine.Name, err) } - ipAddresses, err := getVMIPAddresses(ctx, config, &vm) + ipAddresses, err := getVMIPAddresses(ctx, config, &vm, ipFamily) if err != nil { return nil, fmt.Errorf("failed to retrieve IP addresses for VM %q: %w", machine.Name, err) } @@ -867,7 +869,7 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ } func (p *provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (*azureVM, error) { - config, _, err := p.getConfig(machine.Spec.ProviderSpec) + config, providerCfg, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("failed to parse MachineSpec: %w", err) } @@ -881,7 +883,8 @@ func (p *provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (* return nil, fmt.Errorf("failed to find machine %q by its UID: %w", machine.UID, err) } - ipAddresses, err := getVMIPAddresses(ctx, config, vm) + ipFamily := providerCfg.Network.GetIPFamily() + ipAddresses, err := getVMIPAddresses(ctx, config, vm, ipFamily) if err != nil { return nil, fmt.Errorf("failed to retrieve IP addresses for VM %v: %w", vm.Name, err) } From b30affc4562c11e3ea3aa6dc3c1a39a7df8ddf7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Tue, 31 May 2022 18:25:20 +0200 Subject: [PATCH 158/489] Support configuring MaxPods for Machines (#1317) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- pkg/apis/cluster/common/consts.go | 1 + pkg/userdata/helper/kubelet.go | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/pkg/apis/cluster/common/consts.go b/pkg/apis/cluster/common/consts.go index c1c12df27..ca58aafca 100644 --- a/pkg/apis/cluster/common/consts.go +++ b/pkg/apis/cluster/common/consts.go @@ -140,6 +140,7 @@ const ( EvictionHardKubeletConfig = "EvictionHard" ContainerLogMaxSizeKubeletConfig = "ContainerLogMaxSize" ContainerLogMaxFilesKubeletConfig = "ContainerLogMaxFiles" + MaxPodsKubeletConfig = "MaxPods" ) const ( diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index 74697a5d2..07f6691da 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -247,6 +247,16 @@ func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGate } } + if maxPods, ok := kubeletConfigs[common.MaxPodsKubeletConfig]; ok { + mp, err := strconv.ParseInt(maxPods, 10, 32) + if err != nil { + // Instead of breaking the workflow, just print a warning and skip the configuration + klog.Warningf("Skipping invalid MaxPods value %v for Kubelet configuration", maxPods) + } else { + cfg.MaxPods = int32(mp) + } + } + if containerLogMaxSize, ok := kubeletConfigs[common.ContainerLogMaxSizeKubeletConfig]; ok { cfg.ContainerLogMaxSize = containerLogMaxSize } From 7fcde862813596277b8a819ea0591f3b86d8722b Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 1 Jun 2022 20:51:25 +0500 Subject: [PATCH 159/489] Use kind for E2E tests (#1304) * Use KIND for E2E tests Signed-off-by: Waleed Malik * Fix installation of genisoimage & mkisofs Signed-off-by: Waleed Malik * Fin. Signed-off-by: Waleed Malik * FIN Signed-off-by: Waleed Malik * Refactor run-e2e-tests script Signed-off-by: Waleed Malik * Only install genisoimage when we are running e2e tests for vSphere Signed-off-by: Waleed Malik * Refactor: handle PR feedback Signed-off-by: Waleed Malik * Use flannel for e2e tests Signed-off-by: Waleed Malik * Test kindnet Signed-off-by: Waleed Malik * Switch back to flannel Signed-off-by: Waleed Malik --- .prow/e2e-features.yaml | 37 ++- .prow/postsubmits.yaml | 6 +- .prow/provider-alibaba.yaml | 9 +- .prow/provider-anexia.yaml | 9 +- .prow/provider-aws.yaml | 85 +++++-- .prow/provider-azure.yaml | 27 ++- .prow/provider-digitalocean.yaml | 9 +- .prow/provider-equinix-metal.yaml | 9 +- .prow/provider-gcp.yaml | 9 +- .prow/provider-hetzner.yaml | 9 +- .prow/provider-kubevirt.yaml | 9 +- .prow/provider-linode.yaml | 9 +- .prow/provider-nutanix.yaml | 9 +- .prow/provider-openstack.yaml | 18 +- .prow/provider-scaleway.yaml | 9 +- .prow/provider-vsphere.yaml | 27 ++- Makefile | 17 +- README.md | 33 +-- cmd/machine-controller/main.go | 35 +-- examples/machine-controller.yaml | 1 + hack/ci-e2e-test.sh | 111 --------- hack/ci/cleanup.sh | 42 ++++ .../download-gocache.sh} | 0 hack/ci/run-e2e-tests.sh | 83 +++++++ hack/ci/setup-cni-in-kind.sh | 43 ++++ hack/ci/setup-kind-cluster.sh | 228 ++++++++++++++++++ hack/ci/setup-machine-controller-in-kind.sh | 55 +++++ .../upload-gocache.sh} | 0 .../integration => hack}/cleanup_machines.sh | 15 +- hack/lib.sh | 184 +++++++++++++- pkg/controller/machine/kubeconfig.go | 12 + pkg/controller/machine/machine_controller.go | 11 +- test/e2e/provisioning/all_e2e_test.go | 2 + test/tools/integration/Makefile | 54 ----- test/tools/integration/README.md | 20 -- test/tools/integration/hetzner.tf | 33 --- .../integration/master_install_script.sh | 163 ------------- test/tools/integration/output.tf | 7 - test/tools/integration/provider.tf.disabled | 14 -- test/tools/integration/provision_master.sh | 70 ------ test/tools/integration/variables.tf | 14 -- test/tools/integration/versions.tf | 9 - 42 files changed, 903 insertions(+), 643 deletions(-) delete mode 100755 hack/ci-e2e-test.sh create mode 100755 hack/ci/cleanup.sh rename hack/{ci-download-gocache.sh => ci/download-gocache.sh} (100%) create mode 100755 hack/ci/run-e2e-tests.sh create mode 100755 hack/ci/setup-cni-in-kind.sh create mode 100755 hack/ci/setup-kind-cluster.sh create mode 100755 hack/ci/setup-machine-controller-in-kind.sh rename hack/{ci-upload-gocache.sh => ci/upload-gocache.sh} (100%) rename {test/tools/integration => hack}/cleanup_machines.sh (72%) mode change 100755 => 100644 delete mode 100644 test/tools/integration/Makefile delete mode 100644 test/tools/integration/README.md delete mode 100644 test/tools/integration/hetzner.tf delete mode 100644 test/tools/integration/master_install_script.sh delete mode 100644 test/tools/integration/output.tf delete mode 100644 test/tools/integration/provider.tf.disabled delete mode 100755 test/tools/integration/provision_master.sh delete mode 100644 test/tools/integration/variables.tf delete mode 100644 test/tools/integration/versions.tf diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 49d3d8ea0..93a9e3e22 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -30,13 +30,18 @@ presubmits: preset-kubevirt: "true" preset-alibaba: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestInvalidObjectsGetRejected" + securityContext: + privileged: true resources: requests: memory: 1Gi @@ -53,13 +58,18 @@ presubmits: preset-e2e-ssh: "true" preset-rhel: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestCustomCAsAreApplied" + securityContext: + privileged: true resources: requests: memory: 1Gi @@ -75,13 +85,18 @@ presubmits: preset-hetzner: "true" preset-e2e-ssh: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestUbuntuProvisioningWithUpgradeE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi @@ -96,14 +111,18 @@ presubmits: preset-hetzner: "true" preset-e2e-ssh: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestDeploymentControllerUpgradesMachineE2E" - env: + securityContext: + privileged: true resources: requests: memory: 1Gi diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 61dfdda06..f3e9361c8 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.13-4 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - /bin/bash - -c @@ -54,9 +54,9 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.13-4 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-upload-gocache.sh" + - "./hack/ci/upload-gocache.sh" resources: requests: cpu: 2 diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index 5680da163..da5dd7f80 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -25,13 +25,18 @@ presubmits: preset-hetzner: "true" preset-e2e-ssh: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestAlibabaProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 0da566fef..50825a12b 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -23,13 +23,18 @@ presubmits: preset-e2e-ssh: "true" preset-anexia: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestAnexiaProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index ec52a2400..834031126 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -24,17 +24,22 @@ presubmits: preset-e2e-ssh: "true" preset-rhel: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSProvisioningE2E" + securityContext: + privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 4Gi + cpu: 1 - name: pull-machine-controller-e2e-aws-arm always_run: true @@ -46,13 +51,18 @@ presubmits: preset-hetzner: "true" preset-e2e-ssh: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSARMProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi @@ -68,13 +78,18 @@ presubmits: preset-hetzner: "true" preset-e2e-ssh: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSEbsEncryptionEnabledProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi @@ -90,13 +105,18 @@ presubmits: preset-hetzner: "true" preset-e2e-ssh: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSFlatcarContainerdProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi @@ -113,13 +133,18 @@ presubmits: preset-e2e-ssh: "true" preset-goproxy: "true" preset-rhel: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSSpotInstanceProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi @@ -135,13 +160,18 @@ presubmits: preset-hetzner: "true" preset-e2e-ssh: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSSLESProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi @@ -157,13 +187,18 @@ presubmits: preset-hetzner: "true" preset-e2e-ssh: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSFlatcarCoreOSCloudInit8ProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi @@ -179,13 +214,18 @@ presubmits: preset-hetzner: "true" preset-e2e-ssh: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSCentOS8ProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi @@ -201,13 +241,18 @@ presubmits: preset-hetzner: "true" preset-e2e-ssh: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSAssumeRoleProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index a88e0b8e0..d1e9f689d 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -24,13 +24,18 @@ presubmits: preset-e2e-ssh: "true" preset-rhel: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestAzureProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi @@ -47,13 +52,18 @@ presubmits: preset-e2e-ssh: "true" preset-rhel: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestAzureCustomImageReferenceProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi @@ -71,13 +81,18 @@ presubmits: preset-e2e-ssh: "true" preset-rhel: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestAzureProvisioningE2ERedhatSatellite" + securityContext: + privileged: true resources: requests: memory: 1Gi diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 9617a8e84..7989696f6 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -23,13 +23,18 @@ presubmits: preset-hetzner: "true" preset-e2e-ssh: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestDigitalOceanProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index ead732a44..9754203b5 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -24,13 +24,18 @@ presubmits: preset-e2e-ssh: "true" preset-equinix-metal: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestEquinixMetalProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 7dc6f44b9..c2788019a 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -24,13 +24,18 @@ presubmits: preset-e2e-ssh: "true" preset-rhel: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestGCEProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 9fcd5c1e1..6f52328b1 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -22,13 +22,18 @@ presubmits: preset-hetzner: "true" preset-e2e-ssh: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestHetznerProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 61cdd508f..477f9ada2 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -25,13 +25,18 @@ presubmits: preset-e2e-ssh: "true" preset-rhel: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestKubevirtProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index fbfe4e97f..f1b416667 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -24,13 +24,18 @@ presubmits: preset-e2e-ssh: "true" preset-linode: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestLinodeProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 8e390ec77..6b3c68fc0 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -26,13 +26,18 @@ presubmits: preset-e2e-ssh: "true" preset-nutanix: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestNutanixProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 7437288a2..7fec6859a 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -24,13 +24,18 @@ presubmits: preset-e2e-ssh: "true" preset-rhel: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestOpenstackProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi @@ -47,13 +52,18 @@ presubmits: preset-e2e-ssh: "true" preset-rhel: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestOpenstackProjectAuthProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 6eba49268..3fe04bfc7 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -23,13 +23,18 @@ presubmits: preset-hetzner: "true" preset-e2e-ssh: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestScalewayProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 3e9750aed..10a9d9ed9 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -24,13 +24,18 @@ presubmits: preset-vsphere: "true" preset-rhel: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestVsphereProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi @@ -47,13 +52,18 @@ presubmits: preset-hetzner: "true" preset-e2e-ssh: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestVsphereDatastoreClusterProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi @@ -70,13 +80,18 @@ presubmits: preset-hetzner: "true" preset-e2e-ssh: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestVsphereResourcePoolProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi diff --git a/Makefile b/Makefile index 1df761943..8cce2f7a5 100644 --- a/Makefile +++ b/Makefile @@ -20,8 +20,6 @@ GOOS ?= $(shell go env GOOS) export CGO_ENABLED := 0 -export E2E_SSH_PUBKEY ?= $(shell test -f ~/.ssh/id_rsa.pub && cat ~/.ssh/id_rsa.pub) - export GIT_TAG ?= $(shell git tag --points-at HEAD) export GOFLAGS?=-mod=readonly -trimpath @@ -98,24 +96,13 @@ test-unit-docker: .PHONY: test-unit test-unit: @#The `-race` flag requires CGO - CGO_ENABLED=1 go test -race ./... + CGO_ENABLED=1 go test -v -race ./... .PHONY: build-tests build-tests: go test -run nope ./... go test -tags e2e -run nope ./... -.PHONY: e2e-cluster -e2e-cluster: machine-controller webhook - make -C test/tools/integration apply - ./test/tools/integration/provision_master.sh do-not-deploy-machine-controller - KUBECONFIG=$(shell pwd)/.kubeconfig kubectl apply -f examples/machine-controller.yaml -l local-testing="true" - -.PHONY: e2e-destroy -e2e-destroy: - ./test/tools/integration/cleanup_machines.sh - make -C test/tools/integration destroy - examples/ca-key.pem: openssl genrsa -out examples/ca-key.pem 4096 @@ -161,7 +148,7 @@ check-dependencies: .PHONY: download-gocache download-gocache: - @./hack/ci-download-gocache.sh + @./hack/ci/download-gocache.sh .PHONY: shfmt shfmt: diff --git a/README.md b/README.md index e6634b8d5..9674c1d0d 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ - [Development](#development) - [Testing](#testing) - [Unittests](#unittests) - - [End-to-End](#end-to-end) + - [End-to-End locally](#end-to-end-locally) - [Troubleshooting](#troubleshooting) - [Contributing](#contributing) - [Before you start](#before-you-start) @@ -114,36 +114,9 @@ data: Simply run `make test-unit` -#### End-to-End +#### End-to-End locally -This project provides easy to use e2e testing using Hetzner cloud. To run the e2e tests -locally, the following steps are required: - -- Populate the environment variable `HZ_E2E_TOKEN` with a valid Hetzner cloud token -- Run `make e2e-cluster` to get a simple kubeadm cluster on Hetzner -- Run `hack/run-machine-controller.sh` to locally run the machine-controller for your freshly created cluster - -If you want to use an existing cluster to test against, you can simply set the `KUBECONFIG` environment variable. -In this case, first make sure that a kubeconfig created by `make e2e-cluster` at `$(go env GOPATH)/src/github.com/kubermatic/machine-controller/.kubeconfig` -doesn't exist, since the tests will default to this hardcoded path and only use the env var as fallback. - -Now you can either - -- Run the tests for all providers via - `go test -race -tags=e2e -parallel 240 -v -timeout 30m ./test/e2e/... -identifier $USER` -- Check `test/e2e/provisioning/all_e2e_test.go` for the available tests, then run only a specific one via - `go test -race -tags=e2e -parallel 24 -v -timeout 20m ./test/e2e/... -identifier $USER -run $TESTNAME` - -**Note:** All e2e tests require corresponding credentials to be present, check - [`test/e2e/provisioning/all_e2e_test.go`](test/e2e/provisioning/all_e2e_test.go) for details - -**Note:** After finishing testing, please clean up after yourself: - -- Execute `./test/tools/integration/cleanup_machines.sh` while the machine-controller is still running -- Execute `make e2e-destroy` to clean up the test control plane - -You can also insert your ssh key into the created instances by editing the manifests in -[`test/e2e/provisioning/testdata/`](test/e2e/provisioning/testdata) +**_[WIP]_** ## Troubleshooting diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 0410ce203..fe5a52183 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -32,7 +32,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1/migrations" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - "github.com/kubermatic/machine-controller/pkg/clusterinfo" + clusterinfo "github.com/kubermatic/machine-controller/pkg/clusterinfo" "github.com/kubermatic/machine-controller/pkg/containerruntime" machinecontroller "github.com/kubermatic/machine-controller/pkg/controller/machine" machinedeploymentcontroller "github.com/kubermatic/machine-controller/pkg/controller/machinedeployment" @@ -72,17 +72,18 @@ var ( useOSM bool - nodeCSRApprover bool - nodeHTTPProxy string - nodeNoProxy string - nodeInsecureRegistries string - nodeRegistryMirrors string - nodePauseImage string - nodeContainerRuntime string - podCIDR string - nodePortRange string - nodeRegistryCredentialsSecret string - nodeContainerdRegistryMirrors = containerruntime.RegistryMirrorsFlags{} + nodeCSRApprover bool + nodeHTTPProxy string + nodeNoProxy string + nodeInsecureRegistries string + nodeRegistryMirrors string + nodePauseImage string + nodeContainerRuntime string + podCIDR string + nodePortRange string + nodeRegistryCredentialsSecret string + nodeContainerdRegistryMirrors = containerruntime.RegistryMirrorsFlags{} + overrideBootstrapKubeletAPIServer string ) const ( @@ -129,6 +130,8 @@ type controllerRunOptions struct { // A port range to reserve for services with NodePort visibility. nodePortRange string + + overrideBootstrapKubeletAPIServer string } func main() { @@ -167,6 +170,7 @@ func main() { flag.StringVar(&nodePortRange, "node-port-range", "30000-32767", "A port range to reserve for services with NodePort visibility") flag.StringVar(&nodeRegistryCredentialsSecret, "node-registry-credentials-secret", "", "A Secret object reference, that contains auth info for image registry in namespace/secret-name form, example: kube-system/registry-credentials. See doc at https://github.com/kubermaric/machine-controller/blob/master/docs/registry-authentication.md") flag.BoolVar(&useOSM, "use-osm", false, "use osm controller for node bootstrap") + flag.StringVar(&overrideBootstrapKubeletAPIServer, "override-bootstrap-kubelet-apiserver", "", "Override for the API server address used in worker nodes bootstrap-kubelet.conf") flag.Parse() kubeconfig = flag.Lookup("kubeconfig").Value.(flag.Getter).Get().(string) @@ -226,7 +230,6 @@ func main() { if err != nil { klog.Fatalf("error building kubernetes clientset for kubeClient: %v", err) } - kubeconfigProvider := clusterinfo.New(cfg, kubeClient) ctrlMetrics := machinecontroller.NewMachineControllerMetrics() @@ -262,8 +265,9 @@ func main() { RegistryCredentialsSecretRef: nodeRegistryCredentialsSecret, ContainerRuntime: containerRuntimeConfig, }, - useOSM: useOSM, - nodePortRange: nodePortRange, + useOSM: useOSM, + nodePortRange: nodePortRange, + overrideBootstrapKubeletAPIServer: overrideBootstrapKubeletAPIServer, } if err := nodeFlags.UpdateNodeSettings(&runOptions.node); err != nil { @@ -396,6 +400,7 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { bs.opt.node, bs.opt.useOSM, bs.opt.nodePortRange, + bs.opt.overrideBootstrapKubeletAPIServer, ); err != nil { return fmt.Errorf("failed to add Machine controller to manager: %w", err) } diff --git a/examples/machine-controller.yaml b/examples/machine-controller.yaml index e6011dde8..28c183780 100644 --- a/examples/machine-controller.yaml +++ b/examples/machine-controller.yaml @@ -240,6 +240,7 @@ spec: - -logtostderr - -v=3 - -worker-count=5 + - -node-csr-approver=true - -cluster-dns=10.10.10.10 - -metrics-address=0.0.0.0:8080 - -health-probe-address=0.0.0.0:8085 diff --git a/hack/ci-e2e-test.sh b/hack/ci-e2e-test.sh deleted file mode 100755 index 3a5f00a44..000000000 --- a/hack/ci-e2e-test.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2019 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail -set -o monitor - -export TF_IN_AUTOMATION=true -export TF_CLI_ARGS="-no-color" - -function cleanup { - set +e - - # Clean up machines - echo "Cleaning up machines." - ./test/tools/integration/cleanup_machines.sh - - cd test/tools/integration - for try in {1..20}; do - # Clean up master - echo "Cleaning up controller, attempt ${try}" - terraform apply -destroy -auto-approve - if [[ $? == 0 ]]; then break; fi - echo "Sleeping for $try seconds" - sleep ${try}s - done - - # Kill background port forward if it's there - pkill ssh || true -} -trap cleanup EXIT - -# Install dependencies -echo "Installing dependencies..." -apt update && apt install -y jq rsync unzip genisoimage -curl --retry 5 --location --remote-name \ - https://storage.googleapis.com/kubernetes-release/release/v1.22.2/bin/linux/amd64/kubectl && - chmod +x kubectl && - mv kubectl /usr/local/bin - -# Build binaries -echo "Building machine-controller and webhook..." -make download-gocache all - -# Copy individual plugins with success control. -echo "Copying machine-controller plugins..." -cp machine-controller-userdata-* /usr/local/bin -ls -l /usr/local/bin - -# Generate ssh key pair -echo "Generating SSH key pair..." -chmod 0700 $HOME/.ssh -ssh-keygen -t rsa -N "" -f ~/.ssh/id_ed25519 - -# Initialize terraform -echo "Initializing Terraform..." -cd test/tools/integration -make terraform -cp provider.tf{.disabled,} -terraform init --input=false --backend-config=key=$BUILD_ID -export TF_VAR_hcloud_token="${HZ_E2E_TOKEN}" -export TF_VAR_hcloud_sshkey_content="$(cat ~/.ssh/id_ed25519.pub)" -export TF_VAR_hcloud_sshkey_name="$BUILD_ID" -export TF_VAR_hcloud_test_server_name="machine-controller-test-${BUILD_ID}" - -for try in {1..20}; do - set +e - # Create environment at cloud provider - echo "Creating environment at cloud provider..." - terraform apply -auto-approve - TF_RC=$? - if [[ $TF_RC == 0 ]]; then break; fi - if [[ $TF_RC != 0 ]] && [[ $try -eq 20 ]]; then - echo "Creating cloud provider env failed!" - exit 1 - fi - echo "Sleeping for $try seconds..." - sleep ${try}s -done - -set -e -cd - - -echo "Creating kubeadm cluster and installing machine-controller into it..." -export E2E_SSH_PUBKEY="$(cat ~/.ssh/id_rsa.pub)" -./test/tools/integration/provision_master.sh - -echo "Running e2e tests..." -if [[ ! -z "${NUTANIX_E2E_PROXY_HOST:-}" ]]; then - vm_priv_addr=$(cat ./priv_addr) - export NUTANIX_E2E_PROXY_URL="http://${NUTANIX_E2E_PROXY_USERNAME}:${NUTANIX_E2E_PROXY_PASSWORD}@${vm_priv_addr}:${NUTANIX_E2E_PROXY_PORT}/" -fi - -export KUBECONFIG=$GOPATH/src/github.com/kubermatic/machine-controller/.kubeconfig -EXTRA_ARGS="" -if [[ $# -gt 0 ]]; then - EXTRA_ARGS="-run $1" -fi -go test -race -tags=e2e -parallel 240 -v -timeout 70m ./test/e2e/... -identifier=$BUILD_ID $EXTRA_ARGS diff --git a/hack/ci/cleanup.sh b/hack/ci/cleanup.sh new file mode 100755 index 000000000..a2caac2f7 --- /dev/null +++ b/hack/ci/cleanup.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail +set -x + +source hack/lib.sh + +if [ ! -f ~/.kube/config ] && [ -n "${PROW_JOB_ID:-}" ]; then + echodate "Kubeconfig for KIND cluster was not found while running in CI, nothing to delete." + exit 0 +fi + +export KUBECONFIG=~/.kube/config + +kubectl annotate --all=true --overwrite node kubermatic.io/skip-eviction=true +kubectl delete machinedeployment -n kube-system --all +kubectl delete machineset -n kube-system --all +kubectl delete machine -n kube-system --all +for try in {1..30}; do + if kubectl get machine -n kube-system 2>&1 | grep -q 'No resources found.'; then exit 0; fi + sleep 10s +done + +# Remove the cluster-exposer svc from CI cluster. +kubectl --kubeconfig /etc/kubeconfig/kubeconfig delete services -l prow.k8s.io/id=$PROW_JOB_ID + +echo "Error: couldn't delete all machines!" +exit 1 diff --git a/hack/ci-download-gocache.sh b/hack/ci/download-gocache.sh similarity index 100% rename from hack/ci-download-gocache.sh rename to hack/ci/download-gocache.sh diff --git a/hack/ci/run-e2e-tests.sh b/hack/ci/run-e2e-tests.sh new file mode 100755 index 000000000..ba4140807 --- /dev/null +++ b/hack/ci/run-e2e-tests.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +cd $(dirname $0)/../.. +source hack/lib.sh + +function cleanup { + set +e + + # Clean up machines and services + echo "Cleaning up machines and services..." + ./hack/ci/cleanup.sh + + # Kill background port forward if it's there + pkill ssh || true +} +trap cleanup EXIT + +export GIT_HEAD_HASH="$(git rev-parse HEAD)" +export MC_VERSION="${GIT_HEAD_HASH}" + +TEST_NAME="Pre-warm Go build cache" +echodate "Attempting to pre-warm Go build cache" + +beforeGocache=$(nowms) +make download-gocache +pushElapsed gocache_download_duration_milliseconds $beforeGocache + +beforeBuild=$(nowms) +echodate "Building machine-controller and webhook..." +make all +pushElapsed binary_build_duration_milliseconds $beforeBuild + +# Copy userdata plugins. +echodate "Copying machine-controller plugins..." +cp machine-controller-userdata-* /usr/local/bin +ls -l /usr/local/bin + +# Install genisoimage, this is required for generating user-data for vSphere +if [[ "${JOB_NAME:-}" = *"pull-machine-controller-e2e-vsphere"* ]]; then + echo "Installing genisoimage..." + apt install -y genisoimage +fi + +echodate "Creating kind cluster" +source hack/ci/setup-kind-cluster.sh + +echodate "Setting up machine-controller in kind on revision ${MC_VERSION}" + +beforeMCSetup=$(nowms) + +source hack/ci/setup-machine-controller-in-kind.sh +pushElapsed kind_mc_setup_duration_milliseconds $beforeMCSetup + +if [[ ! -z "${NUTANIX_E2E_PROXY_HOST:-}" ]]; then + vm_priv_addr=$(cat ./priv_addr) + export NUTANIX_E2E_PROXY_URL="http://${NUTANIX_E2E_PROXY_USERNAME}:${NUTANIX_E2E_PROXY_PASSWORD}@${vm_priv_addr}:${NUTANIX_E2E_PROXY_PORT}/" +fi + +echo "Running e2e tests..." +EXTRA_ARGS="" +if [[ $# -gt 0 ]]; then + EXTRA_ARGS="-run $1" +fi +go test -race -tags=e2e -parallel 240 -v -timeout 70m ./test/e2e/... -identifier=$BUILD_ID $EXTRA_ARGS + +echo "Cleaning up machines and services..." +source hack/ci/cleanup.sh diff --git a/hack/ci/setup-cni-in-kind.sh b/hack/ci/setup-cni-in-kind.sh new file mode 100755 index 000000000..50b075a8d --- /dev/null +++ b/hack/ci/setup-cni-in-kind.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CNI_VERSION="${CNI_VERSION:-v0.8.7}" + +cni_bin_dir=/opt/cni/bin +mkdir -p /etc/cni/net.d "$cni_bin_dir" +arch=${HOST_ARCH-} +if [ -z "$arch" ]; then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac +fi +cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" +cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" +curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" +cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") +cd "$cni_bin_dir" +sha256sum -c <<< "$cni_sum" +tar xvf "$cni_filename" +rm -f "$cni_filename" diff --git a/hack/ci/setup-kind-cluster.sh b/hack/ci/setup-kind-cluster.sh new file mode 100755 index 000000000..bd8587761 --- /dev/null +++ b/hack/ci/setup-kind-cluster.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash + +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source hack/lib.sh + +echodate "Setting up kind cluster..." + +if [ -z "${JOB_NAME:-}" ] || [ -z "${PROW_JOB_ID:-}" ]; then + echodate "This script should only be running in a CI environment." + exit 1 +fi + +export KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-machine-controller}" + +start_docker_daemon_ci + +# Make debugging a bit better +echodate "Configuring bash" +cat << EOF >> ~/.bashrc +# Gets set to the CI clusters kubeconfig from a preset +unset KUBECONFIG + +cn() { + kubectl config set-context --current --namespace=\$1 +} + +kubeconfig() { + TMP_KUBECONFIG=\$(mktemp); + kubectl get secret admin-kubeconfig -o go-template='{{ index .data "kubeconfig" }}' | base64 -d > \$TMP_KUBECONFIG; + export KUBECONFIG=\$TMP_KUBECONFIG; + cn kube-system +} + +# this alias makes it so that watch can be used with other aliases, like "watch k get pods" +alias watch='watch ' +alias k=kubectl +alias ll='ls -lh --file-type --group-directories-first' +alias lll='ls -lahF --group-directories-first' +source <(k completion bash ) +source <(k completion bash | sed s/kubectl/k/g) +EOF + +# Find external IP of node where this pod is running +echodate "Retrieving the external node IP where this pod is scheduled" +export NODE_NAME=$(kubectl --kubeconfig /etc/kubeconfig/kubeconfig get pods -l prow.k8s.io/id=$PROW_JOB_ID -o jsonpath="{.items..spec.nodeName}") +export NODE_IP=$(kubectl --kubeconfig /etc/kubeconfig/kubeconfig get node $NODE_NAME -o jsonpath="{.status.addresses[?(@.type=='ExternalIP')].address}") + +if [ -z "$NODE_NAME" ] || [ -z "$NODE_IP" ]; then + echodate "This script was unable to determine the external IP for kube-apiserver." + exit 1 +fi + +# Create kind cluster +TEST_NAME="Create kind cluster" + +echodate "Preloading the kindest/node image" +docker load --input /kindest.tar + +echodate "Creating the kind cluster" +export KUBECONFIG=~/.kube/config + +beforeKindCreate=$(nowms) + +cat << EOF > kind-config.yaml +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: "${KIND_CLUSTER_NAME}" +networking: + apiServerAddress: "0.0.0.0" + disableDefaultCNI: true # disable kindnet +kubeadmConfigPatches: +- | + kind: ClusterConfiguration + apiServer: + extraArgs: + "kubernetes-service-node-port": "31443" + certSANs: + - localhost + - 127.0.0.1 + - kubernetes + - kubernetes.default.svc + - kubernetes.default.svc.cluster.local + - 0.0.0.0 + - ${NODE_IP} + - ${KIND_CLUSTER_NAME} +nodes: + - role: control-plane +EOF + +if [ -n "${DOCKER_REGISTRY_MIRROR_ADDR:-}" ]; then + mirrorHost="$(echo "$DOCKER_REGISTRY_MIRROR_ADDR" | sed 's#http://##' | sed 's#/+$##g')" + + # make the registry mirror available as a socket, + # so we can mount it into the kind cluster + mkdir -p /mirror + socat UNIX-LISTEN:/mirror/mirror.sock,fork,reuseaddr,unlink-early,mode=777 TCP4:$mirrorHost & + + function end_socat_process { + echodate "Killing socat docker registry mirror processes..." + pkill -e socat + } + appendTrap end_socat_process EXIT + + cat << EOF >> kind-config.yaml + # mount the socket + extraMounts: + - hostPath: /mirror + containerPath: /mirror +containerdConfigPatches: + # point to the soon-to-start local socat process + - |- + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["http://127.0.0.1:5001/"] +EOF + + kind create cluster --config kind-config.yaml + pushElapsed kind_cluster_create_duration_milliseconds $beforeKindCreate + + # unwrap the socket inside the kind cluster and make it available on a TCP port, + # because containerd/Docker doesn't support sockets for mirrors. + docker exec $KIND_CLUSTER_NAME-control-plane bash -c 'socat TCP4-LISTEN:5001,fork,reuseaddr UNIX:/mirror/mirror.sock &' +else + kind create cluster --config kind-config.yaml +fi + +echodate "Kind cluster $KIND_CLUSTER_NAME is up and running." + +if [ ! -f cni-plugin-deployed ]; then + echodate "Installing CNI plugin." + ( + # Install CNI plugins since they are not installed by default in KIND. Also, kube-flannel doesn't install + # CNI plugins unlike other plugins so we have to do it manually. + setup_cni_in_kind=$(cat hack/ci/setup-cni-in-kind.sh) + docker exec $KIND_CLUSTER_NAME-control-plane bash -c "$setup_cni_in_kind &" + ) + kubectl create -f https://raw.githubusercontent.com/flannel-io/flannel/v0.18.0/Documentation/kube-flannel.yml + touch cni-plugin-deployed +fi + +if [ -z "${DISABLE_CLUSTER_EXPOSER:-}" ]; then + # Annotate kube-apiserver service so that the cluster exposer can expose it + kubectl annotate svc kubernetes -n default nodeport-proxy.k8s.io/expose=true + + # Start cluster exposer, which will expose services from within kind as + # a NodePort service on the host + echodate "Starting cluster exposer" + ( + # Clone kubermatic repo to build clusterexposer + mkdir -p /tmp/kubermatic + cd /tmp/kubermatic + echodate "Cloning cluster exposer" + KKP_REPO_URL="${KKP_REPO_URL:-https://github.com/kubermatic/kubermatic.git}" + KKP_REPO_TAG="${KKP_REPO_BRANCH:-master}" + git clone --depth 1 --branch "${KKP_REPO_TAG}" "${KKP_REPO_URL}" . + + echodate "Building cluster exposer" + CGO_ENABLED=0 go build --tags ce -v -o /tmp/clusterexposer ./pkg/test/clusterexposer/cmd + ) + + export KUBECONFIG=~/.kube/config + /tmp/clusterexposer \ + --kubeconfig-inner "$KUBECONFIG" \ + --kubeconfig-outer "/etc/kubeconfig/kubeconfig" \ + --build-id "$PROW_JOB_ID" &> /var/log/clusterexposer.log & + + function print_cluster_exposer_logs { + if [[ $? -ne 0 ]]; then + # Tolerate errors and just continue + set +e + echodate "Printing cluster exposer logs" + cat /var/log/clusterexposer.log + echodate "Done printing cluster exposer logs" + set -e + fi + } + appendTrap print_cluster_exposer_logs EXIT + + TEST_NAME="Wait for cluster exposer" + echodate "Waiting for cluster exposer to be running" + + retry 5 curl -s --fail http://127.0.0.1:2047/metrics -o /dev/null + echodate "Cluster exposer is running" + + echodate "Setting up iptables rules to make nodeports available" + KIND_NETWORK_IF=$(ip -br addr | grep -- 'br-' | cut -d' ' -f1) + KIND_CONTAINER_IP=$(docker inspect -f '{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $KIND_CLUSTER_NAME-control-plane) + + iptables -t nat -A PREROUTING -i eth0 -p tcp -m multiport --dports=30000:33000 -j DNAT --to-destination $KIND_CONTAINER_IP + # By default all traffic gets dropped unless specified (tested with docker server 18.09.1) + iptables -t filter -I DOCKER-USER -d $KIND_CONTAINER_IP/32 ! -i $KIND_NETWORK_IF -o $KIND_NETWORK_IF -p tcp -m multiport --dports=30000:33000 -j ACCEPT + # Docker sets up a MASQUERADE rule for postrouting, so nothing to do for us + + echodate "Successfully set up iptables rules for nodeports" + + # Compute external kube-apiserver address + # If svc is not found then we need to check cluster-exposer logs + PORT=$(kubectl --kubeconfig /etc/kubeconfig/kubeconfig get svc -l prow.k8s.io/id=$PROW_JOB_ID -o jsonpath="{.items..spec.ports[0].nodePort}") + + if [ -z "$PORT" ] || [ -z "$NODE_NAME" ] || [ -z "$NODE_IP" ]; then + echodate "This script was unable to determine the external IP for kube-apiserver." + exit 1 + fi + + export MASTER_URL="https://$NODE_IP:$PORT" + + retry 5 curl -ks --fail $MASTER_URL/version -o /dev/null + echodate "New api-server address is reachable" + + # Use kubeconfig with external kube-apiserver address for machine-controller + cp ~/.kube/config ~/.kube/config-external + sed -i "s;server.*;server: $MASTER_URL;g" ~/.kube/config-external + + export KUBECONFIG=~/.kube/config-external + echodate "kube-apiserver for KIND cluster successfully exposed." +fi diff --git a/hack/ci/setup-machine-controller-in-kind.sh b/hack/ci/setup-machine-controller-in-kind.sh new file mode 100755 index 000000000..3ba2a8c87 --- /dev/null +++ b/hack/ci/setup-machine-controller-in-kind.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash + +# Copyright 2022 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +source hack/lib.sh + +if [ -z "${KIND_CLUSTER_NAME:-}" ]; then + echodate "KIND_CLUSTER_NAME must be set by calling setup-kind-cluster.sh first." + exit 1 +fi + +export MC_VERSION="${MC_VERSION:-$(git rev-parse HEAD)}" + +# Build the Docker image for machine-controller +beforeDockerBuild=$(nowms) + +echodate "Building machine-controller Docker image" +TEST_NAME="Build machine-controller Docker image" +IMAGE_NAME="quay.io/kubermatic/machine-controller:latest" +time retry 5 docker build -t "$IMAGE_NAME" . +time retry 5 kind load docker-image "$IMAGE_NAME" --name "$KIND_CLUSTER_NAME" + +pushElapsed mc_docker_build_duration_milliseconds $beforeDockerBuild +echodate "Successfully built and loaded machine-controller image" + +if [ ! -f machine-controller-deployed ]; then + # The 10 minute window given by default for the node to appear is too short + # when we upgrade the instance during the upgrade test + if [[ ${LC_JOB_NAME:-} = "pull-machine-controller-e2e-ubuntu-upgrade" ]]; then + sed -i '/.*join-cluster-timeout=.*/d' examples/machine-controller.yaml + fi + sed -i -e 's/-worker-count=5/-worker-count=50/g' examples/machine-controller.yaml + # This is required for running e2e tests in KIND + url="-override-bootstrap-kubelet-apiserver=$MASTER_URL" + sed -i "s;-node-csr-approver=true;$url;g" examples/machine-controller.yaml + make deploy + touch machine-controller-deployed +fi + +sleep 10 +retry 10 check_all_deployments_ready kube-system + +echodate "Finished installing machine-controller" diff --git a/hack/ci-upload-gocache.sh b/hack/ci/upload-gocache.sh similarity index 100% rename from hack/ci-upload-gocache.sh rename to hack/ci/upload-gocache.sh diff --git a/test/tools/integration/cleanup_machines.sh b/hack/cleanup_machines.sh old mode 100755 new mode 100644 similarity index 72% rename from test/tools/integration/cleanup_machines.sh rename to hack/cleanup_machines.sh index 45da81750..46246eaea --- a/test/tools/integration/cleanup_machines.sh +++ b/hack/cleanup_machines.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright 2019 The Machine Controller Authors. +# Copyright 2022 The Machine Controller Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,25 +17,14 @@ set -euo pipefail set -x -cd $(dirname $0) - -export ADDR=$(terraform output -json|jq '.ip.value' -r) - - -ssh_exec() { ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@$ADDR $@; } - - -cat <&1|grep -q 'No resources found.'; then exit 0; fi + if kubectl get machine -n kube-system 2>&1 | grep -q 'No resources found.'; then exit 0; fi sleep 10s done echo "Error: couldn't delete all machines!" exit 1 -EOEXEC diff --git a/hack/lib.sh b/hack/lib.sh index 313119c90..9f0789c57 100644 --- a/hack/lib.sh +++ b/hack/lib.sh @@ -21,9 +21,28 @@ # receives a SIGINT set -o monitor -echodate() { - # do not use -Is to keep this compatible with macOS - echo "[$(date +%Y-%m-%dT%H:%M:%S%:z)]" "$@" +# appendTrap appends to existing traps, if any. It is needed because Bash replaces existing handlers +# rather than appending: https://stackoverflow.com/questions/3338030/multiple-bash-traps-for-the-same-signal +# Needing this func is a strong indicator that Bash is not the right language anymore. Also, this +# basically needs unit tests. +appendTrap() { + command="$1" + signal="$2" + + # Have existing traps, must append + if [[ "$(trap -p | grep $signal)" ]]; then + existingHandlerName="$(trap -p | grep $signal | awk '{print $3}' | tr -d "'")" + + newHandlerName="${command}_$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 13)" + # Need eval to get a random func name + eval "$newHandlerName() { $command; $existingHandlerName; }" + echodate "Appending $command as trap for $signal, existing command $existingHandlerName" + trap $newHandlerName $signal + # First trap + else + echodate "Using $command as trap for $signal" + trap $command $signal + fi } containerize() { @@ -57,3 +76,162 @@ containerize() { exit $? fi } + +echodate() { + # do not use -Is to keep this compatible with macOS + echo "[$(date +%Y-%m-%dT%H:%M:%S%:z)]" "$@" +} + +# returns the current time as a number of milliseconds +nowms() { + echo $(($(date +%s%N) / 1000000)) +} + +# returns the number of milliseconds elapsed since the given time +elapsed() { + echo $(($(nowms) - $1)) +} + +# pushes a Prometheus metric to a pushgateway +pushMetric() { + local metric="$1" + local value="$2" + local labels="${3:-}" + local kind="${4:-gauge}" + local help="${5:-}" + local pushgateway="${PUSHGATEWAY_URL:-}" + local job="ci" + local instance="${PROW_JOB_ID:-}" + local prowjob="${JOB_NAME:-}" + + if [ -z "$pushgateway" ]; then + return + fi + + local payload="# TYPE $metric $kind" + + if [ -n "$help" ]; then + payload="$payload\n# HELP $metric $help" + fi + + if [ -n "$labels" ]; then + labels=",$labels" + fi + + payload="$payload\n$metric{prowjob=\"$prowjob\"$labels} $value\n" + + echo -e "$payload" | curl --data-binary @- -s "$pushgateway/metrics/job/$job/instance/$instance" +} + +pushElapsed() { + pushMetric "$1" $(elapsed $2) "${3:-}" "${4:-}" "${5:-}" +} + +retry() { + # Works only with bash but doesn't fail on other shells + start_time=$(date +%s) + set +e + actual_retry $@ + rc=$? + set -e + elapsed_time=$(($(date +%s) - $start_time)) + write_junit "$rc" "$elapsed_time" + return $rc +} + +write_junit() { + # Doesn't make any sense if we don't know a testname + if [ -z "${TEST_NAME:-}" ]; then return; fi + # Only run in CI + if [ -z "${ARTIFACTS:-}" ]; then return; fi + + rc=$1 + duration=${2:-0} + errors=0 + failure="" + if [ "$rc" -ne 0 ]; then + errors=1 + failure='Step failed' + fi + TEST_CLASS="${TEST_CLASS:-Kubermatic}" + cat << EOF > ${ARTIFACTS}/junit.$(echo $TEST_NAME | sed 's/ /_/g' | tr '[:upper:]' '[:lower:]').xml + + + + + $failure + + + +EOF +} + +# We use an extra wrapping to write junit and have a timer +actual_retry() { + retries=$1 + shift + + count=0 + delay=1 + until "$@"; do + rc=$? + count=$((count + 1)) + if [ $count -lt "$retries" ]; then + echo "Retry $count/$retries exited $rc, retrying in $delay seconds..." > /dev/stderr + sleep $delay + else + echo "Retry $count/$retries exited $rc, no more retries left." > /dev/stderr + return $rc + fi + delay=$((delay * 2)) + done + return 0 +} + +start_docker_daemon_ci() { + # DOCKER_REGISTRY_MIRROR_ADDR is injected via Prow preset; + # start-docker.sh is part of the build image. + DOCKER_REGISTRY_MIRROR="${DOCKER_REGISTRY_MIRROR_ADDR:-}" DOCKER_MTU=1400 start-docker.sh +} + +start_docker_daemon() { + if docker stats --no-stream > /dev/null 2>&1; then + echodate "Not starting Docker again, it's already running." + return + fi + + # Start Docker daemon + echodate "Starting Docker" + dockerd > /tmp/docker.log 2>&1 & + + echodate "Started Docker successfully" + appendTrap docker_logs EXIT + + # Wait for Docker to start + echodate "Waiting for Docker" + retry 5 docker stats --no-stream + echodate "Docker became ready" +} + +check_all_deployments_ready() { + local namespace="$1" + + # check that Deployments have been created + local deployments + deployments=$(kubectl -n $namespace get deployments -o json) + + if [ $(echo "$deployments" | jq '.items | length') -eq 0 ]; then + echodate "No Deployments created yet." + return 1 + fi + + # check that all Deployments are ready + local unready + unready=$(echo "$deployments" | jq -r '[.items[] | select(.spec.replicas > 0) | select (.status.availableReplicas < .spec.replicas) | .metadata.name] | @tsv') + if [ -n "$unready" ]; then + echodate "Not all Deployments have finished rolling out, namely: $unready" + return 1 + fi + + return 0 +} diff --git a/pkg/controller/machine/kubeconfig.go b/pkg/controller/machine/kubeconfig.go index f4c91036f..d1926fee4 100644 --- a/pkg/controller/machine/kubeconfig.go +++ b/pkg/controller/machine/kubeconfig.go @@ -90,6 +90,18 @@ func (r *Reconciler) createBootstrapKubeconfig(ctx context.Context, name string) }, } + // This is supposed to have a length of 1. We have code further down the + // line that extracts the CA cert and errors out if that is not the case. + // + // This handles a very special case in which we want to override the API server + // address that will be used in the `bootstrap-kubelet.conf` in the worker nodes for + // our E2E tests that run in KIND clusters. + if r.overrideBootstrapKubeletAPIServer != "" { + for key := range outConfig.Clusters { + outConfig.Clusters[key].Server = r.overrideBootstrapKubeletAPIServer + } + } + outConfig.Contexts = map[string]*clientcmdapi.Context{contextIdentifier: {Cluster: contextIdentifier, AuthInfo: contextIdentifier}} outConfig.CurrentContext = contextIdentifier diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 3eb909bcd..df7fd73a4 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -117,8 +117,9 @@ type Reconciler struct { redhatSubscriptionManager rhsm.RedHatSubscriptionManager satelliteSubscriptionManager rhsm.SatelliteSubscriptionManager - useOSM bool - nodePortRange string + useOSM bool + nodePortRange string + overrideBootstrapKubeletAPIServer string } type NodeSettings struct { @@ -176,6 +177,7 @@ func Add( nodeSettings NodeSettings, useOSM bool, nodePortRange string, + overrideBootstrapKubeletAPIServer string, ) error { reconciler := &Reconciler{ kubeClient: kubeClient, @@ -192,8 +194,9 @@ func Add( redhatSubscriptionManager: rhsm.NewRedHatSubscriptionManager(), satelliteSubscriptionManager: rhsm.NewSatelliteSubscriptionManager(), - useOSM: useOSM, - nodePortRange: nodePortRange, + useOSM: useOSM, + nodePortRange: nodePortRange, + overrideBootstrapKubeletAPIServer: overrideBootstrapKubeletAPIServer, } m, err := userdatamanager.New() if err != nil { diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 43f585d74..2728ed658 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -394,7 +394,9 @@ func TestAWSProvisioningE2E(t *testing.T) { if len(awsKeyID) == 0 || len(awsSecret) == 0 { t.Fatal("unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } + selector := Not(OsSelector("sles")) + // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), fmt.Sprintf("<< AWS_SECRET_ACCESS_KEY >>=%s", awsSecret), diff --git a/test/tools/integration/Makefile b/test/tools/integration/Makefile deleted file mode 100644 index 84ae9926a..000000000 --- a/test/tools/integration/Makefile +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2019 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SHELL := /bin/bash - -BUILD_ID ?= $(USER)-local - -USER ?= prow - -export PATH := $(shell pwd):$(PATH) - -ifeq ($(MAKECMDGOALS),apply) - EXTRA_ARG = -auto-approve -endif - -.PHONY: terraform -terraform: - @if ! which terraform; then \ - curl https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_amd64.zip \ - --retry 5 \ - -o /tmp/terraform.zip && \ - unzip -n /tmp/terraform.zip terraform && \ - mv terraform /usr/local/bin; \ - fi - -.terraform: terraform - @if ! ls .terraform &>/dev/null; then \ - terraform init &>/dev/null; \ - fi - -.PHONY: plan apply destroy -plan apply destroy: .terraform - @terraform $@\ - $(EXTRA_ARG) \ - -var hcloud_token=$(HZ_E2E_TOKEN) \ - -var hcloud_sshkey_name=$(USER)-$(BUILD_ID) \ - -var hcloud_sshkey_content="$(shell cat ~/.ssh/id_rsa.pub)" \ - -var hcloud_test_server_name="machine-controller-test-$(BUILD_ID)" - -provision: apply - make -C ../../../ clean - make -C ../../../ all GOOS=linux - ./provision_master.sh diff --git a/test/tools/integration/README.md b/test/tools/integration/README.md deleted file mode 100644 index 1a73e3929..000000000 --- a/test/tools/integration/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Integration testing - -You can find some scripts here to do basic integration testing. Currently it -creates a single-node-cluster (servertype: `cx11`) via `kubeadm` at -[Hetzner cloud](https://www.hetzner.de/cloud) and verifies - -* If the `machine-controller` pod successfully comes up - -## Requirements - -* Docker -* A [Hetzner Cloud account](https://www.hetzner.de/cloud) -* A SSH pubkey in `~/.ssh/id_rsa.pub` - -## Usage - -* `export HZ_E2E_TOKEN=` -* `make provision` to create and provision the environment -* Wait for the tests -* `make destroy` diff --git a/test/tools/integration/hetzner.tf b/test/tools/integration/hetzner.tf deleted file mode 100644 index d12a907a2..000000000 --- a/test/tools/integration/hetzner.tf +++ /dev/null @@ -1,33 +0,0 @@ -provider "hcloud" { - token = var.hcloud_token -} - -resource "hcloud_ssh_key" "default" { - name = var.hcloud_sshkey_name - public_key = var.hcloud_sshkey_content -} - -resource "hcloud_network" "net" { - name = var.hcloud_test_server_name - ip_range = "192.168.0.0/16" -} - -resource "hcloud_server" "machine-controller-test" { - name = var.hcloud_test_server_name - image = "ubuntu-22.04" - server_type = "cx21" - ssh_keys = [hcloud_ssh_key.default.id] - location = "nbg1" -} - -resource "hcloud_network_subnet" "machine_controller" { - network_id = hcloud_network.net.id - type = "server" - network_zone = var.hcloud_network_zone - ip_range = "192.168.0.0/16" -} - -resource "hcloud_server_network" "machine_controller" { - server_id = hcloud_server.machine-controller-test.id - subnet_id = hcloud_network_subnet.machine_controller.id -} diff --git a/test/tools/integration/master_install_script.sh b/test/tools/integration/master_install_script.sh deleted file mode 100644 index 50b0e4cd9..000000000 --- a/test/tools/integration/master_install_script.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2019 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail -set -x - -K8S_VERSION=1.23.0 -echo "$LC_E2E_SSH_PUBKEY" >> .ssh/authorized_keys -echo "GatewayPorts clientspecified" >> /etc/ssh/sshd_config -systemctl restart sshd.service - -export DEBIAN_FRONTEND=noninteractive - -# Hetzner's Ubuntu Bionic comes with swap pre-configured, so we force it off. -systemctl mask swap.target -swapoff -a - -if ! which buildah; then - apt-get update - apt-get -y install buildah -fi -if ! which make; then - apt update - apt install make -fi -if ! which containerd; then - apt update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat </etc/apt/sources.list.d/kubernetes.list - deb http://apt.kubernetes.io/ kubernetes-xenial main -EOF - apt-get update - apt-get install -y \ - kubelet=${K8S_VERSION}-00 \ - kubeadm=${K8S_VERSION}-00 \ - kubectl=${K8S_VERSION}-00 - kubeadm init --kubernetes-version=${K8S_VERSION} \ - --apiserver-advertise-address=${LC_ADDR} --pod-network-cidr=10.244.0.0/16 --service-cidr=172.16.0.0/12 -fi -if ! ls $HOME/.kube/config; then - mkdir -p $HOME/.kube - cp -i /etc/kubernetes/admin.conf $HOME/.kube/config - kubectl taint nodes --all node-role.kubernetes.io/master- -fi -if ! ls kube-flannel.yml; then - kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/v0.15.0/Documentation/kube-flannel.yml -fi - -if ! grep -q kubectl /root/.bashrc; then - cat << 'EOF' >> /root/.bashrc -function cn { kubectl config set-context $(kubectl config current-context) --namespace=$1; } -source <(kubectl completion bash) -alias k=kubectl -source <(k completion bash | sed s/kubectl/k/) -EOF - function cn { kubectl config set-context $(kubectl config current-context) --namespace=$1; } - cn kube-system -fi - -if [[ "${LC_DEPLOY_MACHINE:-}" == "do-not-deploy-machine-controller" ]]; then - exit 0 -fi -if ! ls machine-controller-deployed; then - buildah build-using-dockerfile --format docker --file Dockerfile --tag kubermatic/machine-controller:latest - mkdir "images" - buildah push localhost/kubermatic/machine-controller oci-archive:./images/machine-controller.tar:localhost/kubermatic/machine-controller:latest - ctr --debug --namespace=k8s.io images import --all-platforms --no-unpack images/machine-controller.tar - sed -i "s_- image: quay.io/kubermatic/machine-controller:latest_- image: localhost/kubermatic/machine-controller:latest_g" examples/machine-controller.yaml - # The 10 minute window given by default for the node to appear is too short - # when we upgrade the instance during the upgrade test - if [[ ${LC_JOB_NAME:-} = "pull-machine-controller-e2e-ubuntu-upgrade" ]]; then - sed -i '/.*join-cluster-timeout=.*/d' examples/machine-controller.yaml - fi - sed -i -e 's/-worker-count=5/-worker-count=50/g' examples/machine-controller.yaml - make deploy - touch machine-controller-deployed -fi - -for try in {1..10}; do - if kubectl get pods -n kube-system|egrep '^machine-controller'|grep -v webhook|grep Running; then - echo "Success!" - exit 0 - fi - sleep 10s -done - -echo "Error: machine-controller didn't come up within 100 seconds!" -echo "Logs:" -kubectl logs -n kube-system $(kubectl get pods -n kube-system|egrep '^machine-controller'|awk '{ print $1}') -exit 1 diff --git a/test/tools/integration/output.tf b/test/tools/integration/output.tf deleted file mode 100644 index 71a9b3175..000000000 --- a/test/tools/integration/output.tf +++ /dev/null @@ -1,7 +0,0 @@ -output "ip" { - value = hcloud_server.machine-controller-test.ipv4_address -} - -output "private_ip" { - value = hcloud_server_network.machine_controller.ip -} diff --git a/test/tools/integration/provider.tf.disabled b/test/tools/integration/provider.tf.disabled deleted file mode 100644 index 2176115d2..000000000 --- a/test/tools/integration/provider.tf.disabled +++ /dev/null @@ -1,14 +0,0 @@ -terraform { - backend "s3" { - bucket = "terraform-machine-controller" - endpoint = "http://minio.minio:9000/" - access_key = "PMIC1HMXNB2R67RNPIX8" - secret_key = "NemiWx+uY79rcJ0hXrktzHk1dm9c0k85WepbuSlK" - region = "myregion" - skip_region_validation = "true" - skip_metadata_api_check = "true" - skip_requesting_account_id = "true" - skip_credentials_validation = "true" - force_path_style = "true" - } -} diff --git a/test/tools/integration/provision_master.sh b/test/tools/integration/provision_master.sh deleted file mode 100755 index 6df0e6279..000000000 --- a/test/tools/integration/provision_master.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2019 The Machine Controller Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -euo pipefail -set -x - -cd "$(dirname "${BASH_SOURCE[0]}")" -MC_ROOT="$(cd ./../../.. && pwd -P)" - -# We use variables prefixed with LC_* in order to be able to send them easily -# with SSH SendEnv, as SSH daemon this usually configured with: -# 'AcceptEnv LANG LC_*'. -export LC_DEPLOY_MACHINE="${1:-}" -export LC_ADDR=$(terraform output -json|jq '.ip.value' -r) -export LC_PRIV_ADDR=$(terraform output -json|jq '.private_ip.value' -r) -export LC_E2E_SSH_PUBKEY="${E2E_SSH_PUBKEY:-$(cat ~/.ssh/id_rsa.pub)}" -export LC_JOB_NAME="${JOB_NAME:-}" - - -ssh_exec() { ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${LC_ADDR} $@; } - -for try in {1..100}; do - if ssh_exec "systemctl stop apt-daily apt-daily-upgrade && systemctl mask apt-daily apt-daily-upgrade && exit"; then break; fi; - sleep 1; -done - -if [[ "${1:-deploy_machine_controller}" != "do-not-deploy-machine-controller" ]]; then -rsync -avR -e "ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" \ - ${MC_ROOT}/./{Makefile,examples/machine-controller.yaml,examples/webhook-certificate.cnf,machine-controller,machine-controller-userdata-*,Dockerfile,webhook} \ - root@${LC_ADDR}:/root/ -fi - -for try in {1..20}; do - ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - -o SendEnv=LC_ADDR \ - -o SendEnv=LC_DEPLOY_MACHINE \ - -o SendEnv=LC_E2E_SSH_PUBKEY \ - -o SendEnv=LC_JOB_NAME \ - root@${LC_ADDR} 'bash -s' < "${MC_ROOT}/test/tools/integration/master_install_script.sh" && break - sleep ${try}s -done - -for try in {1..20}; do -scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ - root@$LC_ADDR:/root/.kube/config \ - "${MC_ROOT}/.kubeconfig" - if [[ $? == 0 ]]; then break; fi - sleep ${try}s -done - -# set up SSH port-forwarding if necessary -if [[ ! -z "${NUTANIX_E2E_PROXY_HOST:-}" ]]; then - echo -n "${LC_PRIV_ADDR}" > ${MC_ROOT}/./priv_addr - - ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ServerAliveInterval=5 -fNT -R ${LC_PRIV_ADDR}:${NUTANIX_E2E_PROXY_PORT}:${NUTANIX_E2E_PROXY_HOST}:${NUTANIX_E2E_PROXY_PORT} root@${LC_ADDR} -fi - diff --git a/test/tools/integration/variables.tf b/test/tools/integration/variables.tf deleted file mode 100644 index a78472096..000000000 --- a/test/tools/integration/variables.tf +++ /dev/null @@ -1,14 +0,0 @@ -variable "hcloud_token" {} -variable "hcloud_sshkey_content" {} - -variable "hcloud_sshkey_name" { - default = "machine-controller-e2e" -} - -variable "hcloud_test_server_name" {} - -variable "hcloud_network_zone" { - default = "eu-central" - description = "network zone to use for private network" - type = string -} diff --git a/test/tools/integration/versions.tf b/test/tools/integration/versions.tf deleted file mode 100644 index 63d4f1c5e..000000000 --- a/test/tools/integration/versions.tf +++ /dev/null @@ -1,9 +0,0 @@ -terraform { - required_version = ">= 1.0.0" - required_providers { - hcloud = { - source = "hetznercloud/hcloud" - version = "~> 1.31.0" - } - } -} From 376cb3203ebfd10268caa647c1964f521b1665cb Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 1 Jun 2022 23:55:22 +0500 Subject: [PATCH 160/489] Fix upload-gocache script (#1319) Signed-off-by: Waleed Malik --- hack/ci/upload-gocache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/ci/upload-gocache.sh b/hack/ci/upload-gocache.sh index 0d7c07f49..2ff04c5ad 100755 --- a/hack/ci/upload-gocache.sh +++ b/hack/ci/upload-gocache.sh @@ -21,7 +21,7 @@ set -euo pipefail # receives a SIGINT set -o monitor -cd $(dirname $0)/.. +cd $(dirname $0)/../.. if [ -z "${GOCACHE_MINIO_ADDRESS:-}" ]; then echo "Fatal: env var GOCACHE_MINIO_ADDRESS unset" From 0a42c6643a15bb42d140d27dea59f4fc676af72f Mon Sep 17 00:00:00 2001 From: Matthias Osthues Date: Thu, 2 Jun 2022 13:22:22 +0200 Subject: [PATCH 161/489] Enable setup.service and disable it after first run (#1316) * enable setup.service and disable it after first run * fix indentation * Update provider-vcloud-director.yaml * Update provider-vcloud-director.yaml * Update provider-vcloud-director.yaml Co-authored-by: Waleed Malik --- .prow/provider-vcloud-director.yaml | 13 +++++++++---- pkg/userdata/amzn2/provider.go | 3 ++- .../amzn2/testdata/kubelet-v1.21-aws-external.yaml | 3 ++- pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml | 3 ++- .../testdata/kubelet-v1.21-vsphere-mirrors.yaml | 3 ++- .../amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml | 3 ++- .../amzn2/testdata/kubelet-v1.21-vsphere.yaml | 3 ++- pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml | 3 ++- pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml | 3 ++- pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml | 3 ++- pkg/userdata/centos/provider.go | 3 ++- .../centos/testdata/kubelet-v1.21-aws-external.yaml | 3 ++- pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml | 3 ++- .../testdata/kubelet-v1.21-vsphere-mirrors.yaml | 3 ++- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 3 ++- .../centos/testdata/kubelet-v1.21-vsphere.yaml | 3 ++- pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml | 3 ++- pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml | 3 ++- .../centos/testdata/kubelet-v1.23-nutanix.yaml | 3 ++- pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml | 3 ++- pkg/userdata/rhel/provider.go | 6 ++++-- pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml | 6 ++++-- pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml | 6 ++++-- .../rhel/testdata/kubelet-v1.22-nutanix.yaml | 6 ++++-- .../rhel/testdata/kubelet-v1.23-aws-external.yaml | 6 ++++-- pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml | 6 ++++-- .../testdata/kubelet-v1.23-vsphere-mirrors.yaml | 6 ++++-- .../rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml | 6 ++++-- .../rhel/testdata/kubelet-v1.23-vsphere.yaml | 6 ++++-- .../rhel/testdata/kubelet-v1.24-aws-external.yaml | 6 ++++-- pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml | 6 ++++-- pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml | 6 ++++-- pkg/userdata/rockylinux/provider.go | 3 ++- .../testdata/kubelet-v1.21-aws-external.yaml | 3 ++- .../rockylinux/testdata/kubelet-v1.21-aws.yaml | 3 ++- .../testdata/kubelet-v1.21-vsphere-mirrors.yaml | 3 ++- .../testdata/kubelet-v1.21-vsphere-proxy.yaml | 3 ++- .../rockylinux/testdata/kubelet-v1.21-vsphere.yaml | 3 ++- .../rockylinux/testdata/kubelet-v1.22-aws.yaml | 3 ++- .../rockylinux/testdata/kubelet-v1.23-aws.yaml | 3 ++- .../rockylinux/testdata/kubelet-v1.23-nutanix.yaml | 3 ++- .../rockylinux/testdata/kubelet-v1.24-aws.yaml | 3 ++- pkg/userdata/ubuntu/provider.go | 3 ++- pkg/userdata/ubuntu/testdata/containerd.yaml | 3 ++- .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 3 ++- pkg/userdata/ubuntu/testdata/docker.yaml | 3 ++- .../testdata/kubelet-version-without-v-prefix.yaml | 3 ++- .../ubuntu/testdata/multiple-dns-servers.yaml | 3 ++- pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml | 3 ++- pkg/userdata/ubuntu/testdata/nutanix.yaml | 3 ++- .../testdata/openstack-overwrite-cloud-config.yaml | 3 ++- pkg/userdata/ubuntu/testdata/openstack.yaml | 3 ++- pkg/userdata/ubuntu/testdata/version-1.21.10.yaml | 3 ++- pkg/userdata/ubuntu/testdata/version-1.22.7.yaml | 3 ++- pkg/userdata/ubuntu/testdata/version-1.23.5.yaml | 3 ++- pkg/userdata/ubuntu/testdata/version-1.24.0.yaml | 3 ++- pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml | 3 ++- pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml | 3 ++- pkg/userdata/ubuntu/testdata/vsphere.yaml | 3 ++- 59 files changed, 149 insertions(+), 74 deletions(-) diff --git a/.prow/provider-vcloud-director.yaml b/.prow/provider-vcloud-director.yaml index 331bdabd1..0ba8c84a8 100644 --- a/.prow/provider-vcloud-director.yaml +++ b/.prow/provider-vcloud-director.yaml @@ -15,23 +15,28 @@ presubmits: - name: pull-machine-controller-e2e-vmware-cloud-director always_run: false - run_if_changed: "(pkg/cloudprovider/provider/vcloud-director/|pkg/userdata)" decorate: true error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + run_if_changed: "(pkg/cloudprovider/provider/vcloud-director/|pkg/userdata)" labels: + preset-vcloud-director: "true" preset-hetzner: "true" preset-e2e-ssh: "true" - preset-vcloud-director: "true" preset-rhel: "true" preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" spec: containers: - - image: golang:1.18.2 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 command: - - "./hack/ci-e2e-test.sh" + - "./hack/ci/run-e2e-tests.sh" args: - "TestVMwareCloudDirectorProvisioningE2E" + securityContext: + privileged: true resources: requests: memory: 1Gi diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index 6c2f59f88..5b15b3b03 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -240,6 +240,7 @@ write_files: {{ end -}} systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -337,5 +338,5 @@ write_files: {{- end }} runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service ` diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml index 7e0f6bc5b..38cf73097 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml @@ -170,6 +170,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -432,4 +433,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml index 29ea96787..79710ae80 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml @@ -170,6 +170,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -432,4 +433,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 223c174e2..3ecb5eba1 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -185,6 +185,7 @@ write_files: systemctl enable --now vmtoolsd.service systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -449,4 +450,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml index 5c30291d4..563d9a827 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -185,6 +185,7 @@ write_files: systemctl enable --now vmtoolsd.service systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -449,4 +450,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml index b8e74651c..48dbf051b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml @@ -177,6 +177,7 @@ write_files: systemctl enable --now vmtoolsd.service systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -440,4 +441,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml index 3313f2065..b8845879e 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml @@ -170,6 +170,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -432,4 +433,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml index fe320b2a6..96ac2eeda 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml @@ -170,6 +170,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -430,4 +431,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml index cc323a27b..c8bf88bdb 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml @@ -173,6 +173,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -449,4 +450,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index a67257ec9..a2aaae6a3 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -258,6 +258,7 @@ write_files: {{- if eq .CloudProviderName "kubevirt" }} systemctl enable --now --no-block restart-kubelet.service {{ end }} + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -391,5 +392,5 @@ write_files: {{- end }} runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service ` diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml index 1b378e43b..f1b92e886 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml @@ -180,6 +180,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -442,4 +443,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml index 2c75bdb1f..442917bd7 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml @@ -180,6 +180,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -442,4 +443,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 4a9fde5dd..10d664f39 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -195,6 +195,7 @@ write_files: systemctl enable --now vmtoolsd.service systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -459,4 +460,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml index a8c507167..4ef234434 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -195,6 +195,7 @@ write_files: systemctl enable --now vmtoolsd.service systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -459,4 +460,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml index 28888a8a2..be2791b17 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml @@ -187,6 +187,7 @@ write_files: systemctl enable --now vmtoolsd.service systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -450,4 +451,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index fe3619423..de96a9328 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -180,6 +180,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -442,4 +443,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index b09de8e64..e912b1131 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -180,6 +180,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -440,4 +441,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml index 3087464bc..50a5ee4a0 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml @@ -187,6 +187,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -448,4 +449,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml index 72068f2f9..9830772bc 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml @@ -179,6 +179,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -455,4 +456,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 2a1647e43..b898e3094 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -258,6 +258,8 @@ write_files: {{- if eq .CloudProviderName "kubevirt" }} systemctl enable --now --no-block restart-kubelet.service {{ end }} + systemctl disable setup.service + systemctl disable disable-nm-cloud-setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -431,6 +433,6 @@ rh_subscription: {{- end }} runcmd: -- systemctl start setup.service -- systemctl start disable-nm-cloud-setup.service +- systemctl enable --now setup.service +- systemctl enable --now disable-nm-cloud-setup.service ` diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml index 2081ec509..9568a5550 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml @@ -180,6 +180,8 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + systemctl disable disable-nm-cloud-setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -475,5 +477,5 @@ rh_subscription: auto-attach: false runcmd: -- systemctl start setup.service -- systemctl start disable-nm-cloud-setup.service +- systemctl enable --now setup.service +- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index 7f53086b2..81fb687e8 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -180,6 +180,8 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + systemctl disable disable-nm-cloud-setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -475,5 +477,5 @@ rh_subscription: auto-attach: false runcmd: -- systemctl start setup.service -- systemctl start disable-nm-cloud-setup.service +- systemctl enable --now setup.service +- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml index 3e88d15a4..da5d9155c 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml @@ -188,6 +188,8 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + systemctl disable disable-nm-cloud-setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -484,5 +486,5 @@ rh_subscription: auto-attach: false runcmd: -- systemctl start setup.service -- systemctl start disable-nm-cloud-setup.service +- systemctl enable --now setup.service +- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index 883f450c6..c46206267 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -180,6 +180,8 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + systemctl disable disable-nm-cloud-setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -473,5 +475,5 @@ rh_subscription: auto-attach: false runcmd: -- systemctl start setup.service -- systemctl start disable-nm-cloud-setup.service +- systemctl enable --now setup.service +- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index b9707e3a9..50fd4811f 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -180,6 +180,8 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + systemctl disable disable-nm-cloud-setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -473,5 +475,5 @@ rh_subscription: auto-attach: false runcmd: -- systemctl start setup.service -- systemctl start disable-nm-cloud-setup.service +- systemctl enable --now setup.service +- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 6f9490298..467e63141 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -196,6 +196,8 @@ write_files: systemctl enable --now vmtoolsd.service systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + systemctl disable disable-nm-cloud-setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -491,5 +493,5 @@ rh_subscription: auto-attach: false runcmd: -- systemctl start setup.service -- systemctl start disable-nm-cloud-setup.service +- systemctl enable --now setup.service +- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index 7b9d5b9f1..aa9bc6081 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -196,6 +196,8 @@ write_files: systemctl enable --now vmtoolsd.service systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + systemctl disable disable-nm-cloud-setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -491,5 +493,5 @@ rh_subscription: auto-attach: false runcmd: -- systemctl start setup.service -- systemctl start disable-nm-cloud-setup.service +- systemctl enable --now setup.service +- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index 666cfccac..a193c46f3 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -188,6 +188,8 @@ write_files: systemctl enable --now vmtoolsd.service systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + systemctl disable disable-nm-cloud-setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -482,5 +484,5 @@ rh_subscription: auto-attach: false runcmd: -- systemctl start setup.service -- systemctl start disable-nm-cloud-setup.service +- systemctl enable --now setup.service +- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml index 25dc1e6cb..88dfc8387 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml @@ -179,6 +179,8 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + systemctl disable disable-nm-cloud-setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -488,5 +490,5 @@ rh_subscription: auto-attach: false runcmd: -- systemctl start setup.service -- systemctl start disable-nm-cloud-setup.service +- systemctl enable --now setup.service +- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml index 021ff6a99..f1905c82b 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml @@ -179,6 +179,8 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + systemctl disable disable-nm-cloud-setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -488,5 +490,5 @@ rh_subscription: auto-attach: false runcmd: -- systemctl start setup.service -- systemctl start disable-nm-cloud-setup.service +- systemctl enable --now setup.service +- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index 74743a1b9..055f69919 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -185,6 +185,8 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + systemctl disable disable-nm-cloud-setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -481,5 +483,5 @@ rh_subscription: auto-attach: false runcmd: -- systemctl start setup.service -- systemctl start disable-nm-cloud-setup.service +- systemctl enable --now setup.service +- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rockylinux/provider.go b/pkg/userdata/rockylinux/provider.go index 4fb5cea34..c2a130600 100644 --- a/pkg/userdata/rockylinux/provider.go +++ b/pkg/userdata/rockylinux/provider.go @@ -250,6 +250,7 @@ write_files: {{ end -}} systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -347,5 +348,5 @@ write_files: {{- end }} runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service ` diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml index 8e6bfb30a..ffe97a9d4 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml @@ -175,6 +175,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -437,4 +438,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml index e21b297d9..048093fd3 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml @@ -175,6 +175,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -437,4 +438,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 5eee9ee20..66de513af 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -190,6 +190,7 @@ write_files: systemctl enable --now vmtoolsd.service systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -454,4 +455,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml index c6625b5f3..43b97a65e 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -190,6 +190,7 @@ write_files: systemctl enable --now vmtoolsd.service systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -454,4 +455,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml index 781c40f8e..34da2675a 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml @@ -182,6 +182,7 @@ write_files: systemctl enable --now vmtoolsd.service systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -445,4 +446,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml index c3dd772a0..85cc06d59 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml @@ -175,6 +175,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -437,4 +438,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml index 397e92608..6c8dea179 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml @@ -175,6 +175,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -435,4 +436,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml index 07bc438ef..d8b9a49a3 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml @@ -182,6 +182,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -443,4 +444,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml index fcb39d701..2b29d711a 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml @@ -174,6 +174,7 @@ write_files: systemctl disable --now firewalld || true systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -450,4 +451,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index dedbcd1c0..034f4d88d 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -251,6 +251,7 @@ write_files: {{- if eq .CloudProviderName "kubevirt" }} systemctl enable --now --no-block restart-kubelet.service {{ end }} + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -387,5 +388,5 @@ write_files: {{- end }} runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service ` diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index 5837b5589..49a360520 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -180,6 +180,7 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -473,4 +474,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 611fa8072..a84ab4bc3 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -180,6 +180,7 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -446,4 +447,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml index 5e32ac8c5..a8193e4d4 100644 --- a/pkg/userdata/ubuntu/testdata/docker.yaml +++ b/pkg/userdata/ubuntu/testdata/docker.yaml @@ -180,6 +180,7 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -458,4 +459,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index 99fbdd863..b5460cb68 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -178,6 +178,7 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -444,4 +445,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 95e57ea40..8592ce601 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -178,6 +178,7 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -446,4 +447,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index fc6034a29..fb5098971 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -180,6 +180,7 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -446,4 +447,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 52107be9c..261f731bb 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -181,6 +181,7 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -451,4 +452,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index 2ddb7214b..a263c4a23 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -178,6 +178,7 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -448,4 +449,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 48183d1b4..8492a8e59 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -178,6 +178,7 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -448,4 +449,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml b/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml index aea35e1d3..f4d99bd0b 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml @@ -178,6 +178,7 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -444,4 +445,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml b/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml index 99fbdd863..b5460cb68 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml @@ -178,6 +178,7 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -444,4 +445,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml index 7b669de76..934498758 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml @@ -178,6 +178,7 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -442,4 +443,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml index 21dc8aebe..c6f718374 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml @@ -178,6 +178,7 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -458,4 +459,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index 7c3c03cf5..c2f9843a1 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -188,6 +188,7 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -459,4 +460,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index cd69189d6..c89efe60a 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -188,6 +188,7 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -459,4 +460,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 5fadc4b76..6c49a76d4 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -179,6 +179,7 @@ write_files: systemctl enable --now kubelet systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service - path: "/opt/bin/supervise.sh" permissions: "0755" @@ -449,4 +450,4 @@ write_files: runcmd: -- systemctl start setup.service +- systemctl enable --now setup.service From ea883ec56ba99a2f51eea4aeddd9c86faab13c44 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 3 Jun 2022 18:58:22 +0500 Subject: [PATCH 162/489] refactor: be consistent with VMware Cloud Director naming (#1321) Signed-off-by: Waleed Malik --- ...ml => provider-vmware-cloud-director.yaml} | 2 +- ...ware-cloud-director-machinedeployment.yaml | 6 +-- pkg/cloudprovider/provider.go | 4 +- .../client.go | 0 .../helper.go | 0 .../provider.go | 2 +- .../types/types.go | 0 pkg/providerconfig/types/types.go | 40 +++++++++---------- 8 files changed, 27 insertions(+), 27 deletions(-) rename .prow/{provider-vcloud-director.yaml => provider-vmware-cloud-director.yaml} (94%) rename pkg/cloudprovider/provider/{vmware-cloud-director => vmwareclouddirector}/client.go (100%) rename pkg/cloudprovider/provider/{vmware-cloud-director => vmwareclouddirector}/helper.go (100%) rename pkg/cloudprovider/provider/{vmware-cloud-director => vmwareclouddirector}/provider.go (99%) rename pkg/cloudprovider/provider/{vmware-cloud-director => vmwareclouddirector}/types/types.go (100%) diff --git a/.prow/provider-vcloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml similarity index 94% rename from .prow/provider-vcloud-director.yaml rename to .prow/provider-vmware-cloud-director.yaml index 0ba8c84a8..99704f8b6 100644 --- a/.prow/provider-vcloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -18,7 +18,7 @@ presubmits: decorate: true error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - run_if_changed: "(pkg/cloudprovider/provider/vcloud-director/|pkg/userdata)" + run_if_changed: "(pkg/cloudprovider/provider/vmwareclouddirector/|pkg/userdata)" labels: preset-vcloud-director: "true" preset-hetzner: "true" diff --git a/examples/vmware-cloud-director-machinedeployment.yaml b/examples/vmware-cloud-director-machinedeployment.yaml index 9c2d17044..5b5106dda 100644 --- a/examples/vmware-cloud-director-machinedeployment.yaml +++ b/examples/vmware-cloud-director-machinedeployment.yaml @@ -3,7 +3,7 @@ kind: Secret metadata: # If you change the namespace/name, you must also # adjust the rbac rules - name: machine-controller-vcloud-director + name: machine-controller-vmware-cloud-director namespace: kube-system type: Opaque stringData: @@ -12,7 +12,7 @@ stringData: apiVersion: "cluster.k8s.io/v1alpha1" kind: MachineDeployment metadata: - name: vcloud-director-machinedeployment + name: vmware-cloud-director-machinedeployment namespace: kube-system spec: paused: false @@ -46,7 +46,7 @@ spec: password: secretKeyRef: namespace: kube-system - name: machine-controller-vcloud-director + name: machine-controller-vmware-cloud-director key: password # Can also be set via the env var 'VCD_ORG' on the machine-controller organization: "<< VCD_ORG >>" diff --git a/pkg/cloudprovider/provider.go b/pkg/cloudprovider/provider.go index 65f4eae7e..8447be854 100644 --- a/pkg/cloudprovider/provider.go +++ b/pkg/cloudprovider/provider.go @@ -35,7 +35,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/nutanix" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/openstack" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/scaleway" - vcd "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vmware-cloud-director" + vcd "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vmwareclouddirector" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vsphere" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/providerconfig" @@ -105,7 +105,7 @@ var ( providerconfigtypes.CloudProviderNutanix: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return nutanix.New(cvr) }, - providerconfigtypes.CloudProviderVcloudDirector: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfigtypes.CloudProviderVMwareCloudDirector: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return vcd.New(cvr) }, } diff --git a/pkg/cloudprovider/provider/vmware-cloud-director/client.go b/pkg/cloudprovider/provider/vmwareclouddirector/client.go similarity index 100% rename from pkg/cloudprovider/provider/vmware-cloud-director/client.go rename to pkg/cloudprovider/provider/vmwareclouddirector/client.go diff --git a/pkg/cloudprovider/provider/vmware-cloud-director/helper.go b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go similarity index 100% rename from pkg/cloudprovider/provider/vmware-cloud-director/helper.go rename to pkg/cloudprovider/provider/vmwareclouddirector/helper.go diff --git a/pkg/cloudprovider/provider/vmware-cloud-director/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go similarity index 99% rename from pkg/cloudprovider/provider/vmware-cloud-director/provider.go rename to pkg/cloudprovider/provider/vmwareclouddirector/provider.go index b01e1149b..2414c1a39 100644 --- a/pkg/cloudprovider/provider/vmware-cloud-director/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -28,7 +28,7 @@ import ( clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - vcdtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vmware-cloud-director/types" + vcdtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vmwareclouddirector/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" diff --git a/pkg/cloudprovider/provider/vmware-cloud-director/types/types.go b/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go similarity index 100% rename from pkg/cloudprovider/provider/vmware-cloud-director/types/types.go rename to pkg/cloudprovider/provider/vmwareclouddirector/types/types.go diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 1de43ec62..b9ddbbcac 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -46,25 +46,25 @@ const ( type CloudProvider string const ( - CloudProviderAWS CloudProvider = "aws" - CloudProviderAzure CloudProvider = "azure" - CloudProviderDigitalocean CloudProvider = "digitalocean" - CloudProviderGoogle CloudProvider = "gce" - CloudProviderEquinixMetal CloudProvider = "equinixmetal" - CloudProviderPacket CloudProvider = "packet" - CloudProviderHetzner CloudProvider = "hetzner" - CloudProviderKubeVirt CloudProvider = "kubevirt" - CloudProviderLinode CloudProvider = "linode" - CloudProviderNutanix CloudProvider = "nutanix" - CloudProviderOpenstack CloudProvider = "openstack" - CloudProviderVsphere CloudProvider = "vsphere" - CloudProviderVcloudDirector CloudProvider = "vmware-cloud-director" - CloudProviderFake CloudProvider = "fake" - CloudProviderAlibaba CloudProvider = "alibaba" - CloudProviderAnexia CloudProvider = "anexia" - CloudProviderScaleway CloudProvider = "scaleway" - CloudProviderBaremetal CloudProvider = "baremetal" - CloudProviderExternal CloudProvider = "external" + CloudProviderAWS CloudProvider = "aws" + CloudProviderAzure CloudProvider = "azure" + CloudProviderDigitalocean CloudProvider = "digitalocean" + CloudProviderGoogle CloudProvider = "gce" + CloudProviderEquinixMetal CloudProvider = "equinixmetal" + CloudProviderPacket CloudProvider = "packet" + CloudProviderHetzner CloudProvider = "hetzner" + CloudProviderKubeVirt CloudProvider = "kubevirt" + CloudProviderLinode CloudProvider = "linode" + CloudProviderNutanix CloudProvider = "nutanix" + CloudProviderOpenstack CloudProvider = "openstack" + CloudProviderVsphere CloudProvider = "vsphere" + CloudProviderVMwareCloudDirector CloudProvider = "vmware-cloud-director" + CloudProviderFake CloudProvider = "fake" + CloudProviderAlibaba CloudProvider = "alibaba" + CloudProviderAnexia CloudProvider = "anexia" + CloudProviderScaleway CloudProvider = "scaleway" + CloudProviderBaremetal CloudProvider = "baremetal" + CloudProviderExternal CloudProvider = "external" ) var ( @@ -95,7 +95,7 @@ var ( CloudProviderNutanix, CloudProviderOpenstack, CloudProviderVsphere, - CloudProviderVcloudDirector, + CloudProviderVMwareCloudDirector, CloudProviderFake, CloudProviderAlibaba, CloudProviderAnexia, From e1bc81571495b97ddec9c7fd80562182a9eb7b2f Mon Sep 17 00:00:00 2001 From: Moritz Bracht <682686+dermorz@users.noreply.github.com> Date: Tue, 7 Jun 2022 11:27:20 +0200 Subject: [PATCH 163/489] Reduce instance profile validation to existence check (#1322) --- pkg/cloudprovider/provider/aws/provider.go | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index e9911e57b..a6436caef 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -31,7 +31,6 @@ import ( "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/iam" "github.com/aws/aws-sdk-go/service/sts" gocache "github.com/patrickmn/go-cache" "github.com/prometheus/client_golang/prometheus" @@ -536,14 +535,6 @@ func getAssumeRoleCredentials(session *session.Session, assumeRoleARN, assumeRol }) } -func getIAMclient(id, secret, region, assumeRoleArn, assumeRoleExternalID string) (*iam.IAM, error) { - sess, err := getSession(id, secret, "", region, assumeRoleArn, assumeRoleExternalID) - if err != nil { - return nil, awsErrorToTerminalError(err, "failed to get aws session") - } - return iam.New(sess), nil -} - func getEC2client(id, secret, region, assumeRoleArn, assumeRoleExternalID string) (*ec2.EC2, error) { sess, err := getSession(id, secret, "", region, assumeRoleArn, assumeRoleExternalID) if err != nil { @@ -646,16 +637,8 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) return fmt.Errorf("failed to validate security group id's: %w", err) } - iamClient, err := getIAMclient(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) - if err != nil { - return fmt.Errorf("failed to create iam client: %w", err) - } - if config.InstanceProfile == "" { - return fmt.Errorf("invalid instance profile specified %q: %w", config.InstanceProfile, err) - } - if _, err := iamClient.GetInstanceProfile(&iam.GetInstanceProfileInput{InstanceProfileName: aws.String(config.InstanceProfile)}); err != nil { - return fmt.Errorf("failed to validate instance profile: %w", err) + return errors.New("no instance profile specified") } if config.IsSpotInstance != nil && *config.IsSpotInstance { From f53b35a4aa73efc15007b792902c2214e4a642f9 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 9 Jun 2022 18:25:05 +0500 Subject: [PATCH 164/489] Update OSM to v0.4.4 (#1324) * Update OSM to v0.4.4 Signed-off-by: Waleed Malik * Pin AMI for Ubuntu 20.04 in e2e tests Signed-off-by: Waleed Malik * Pin AMI for Ubuntu 20.04 in AWS ARM e2e tests Signed-off-by: Waleed Malik * Fix AWS EBS encryption E2E test Signed-off-by: Waleed Malik --- go.mod | 26 ++++----- go.sum | 54 +++++++++++-------- pkg/admission/machinedeployments.go | 4 +- test/e2e/provisioning/helper.go | 11 ++++ .../machinedeployment-aws-arm-machines.yaml | 3 +- ...deployment-aws-ebs-encryption-enabled.yaml | 1 + 6 files changed, 61 insertions(+), 38 deletions(-) diff --git a/go.mod b/go.mod index 0de6b28b2..a17893129 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/Azure/azure-sdk-for-go v64.1.0+incompatible github.com/Azure/go-autorest/autorest/azure/auth v0.5.5 github.com/Azure/go-autorest/autorest/to v0.4.0 - github.com/BurntSushi/toml v0.3.1 + github.com/BurntSushi/toml v1.1.0 github.com/Masterminds/semver/v3 v3.1.1 github.com/Masterminds/sprig/v3 v3.2.2 github.com/aliyun/alibaba-cloud-sdk-go v1.61.751 @@ -36,14 +36,14 @@ require ( github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda github.com/vmware/go-vcloud-director/v2 v2.15.0 github.com/vmware/govmomi v0.23.1 - golang.org/x/crypto v0.0.0-20220214200702-86341886e292 + golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a gomodules.xyz/jsonpatch/v2 v2.2.0 google.golang.org/api v0.74.0 google.golang.org/grpc v1.45.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b - k8c.io/operating-system-manager v0.4.0 + k8c.io/operating-system-manager v0.4.4 k8s.io/api v0.24.0 k8s.io/apiextensions-apiserver v0.24.0 k8s.io/apimachinery v0.24.0 @@ -73,7 +73,7 @@ require ( github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd // indirect - github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 // indirect + github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect @@ -83,9 +83,9 @@ require ( github.com/dimchansky/utfbom v1.1.0 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/emicklei/go-restful v2.15.0+incompatible // indirect - github.com/evanphx/json-patch v4.12.0+incompatible // indirect + github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect @@ -110,9 +110,9 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/mitchellh/copystructure v1.0.0 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/reflectwalk v1.0.1 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect @@ -120,18 +120,18 @@ require ( github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/peterhellberg/link v1.1.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/common v0.34.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect - github.com/shopspring/decimal v1.2.0 // indirect + github.com/shopspring/decimal v1.3.1 // indirect github.com/smartystreets/assertions v1.2.0 // indirect - github.com/spf13/cast v1.3.1 // indirect + github.com/spf13/cast v1.4.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50 // indirect + github.com/vincent-petithory/dataurl v1.0.0 // indirect go.opencensus.io v0.23.0 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/net v0.0.0-20220325170049-de3da57026de // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 // indirect + golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect diff --git a/go.sum b/go.sum index 50ef3075a..65ac18e0f 100644 --- a/go.sum +++ b/go.sum @@ -85,8 +85,9 @@ github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+Z github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= +github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -121,8 +122,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4= -github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.61.751 h1:PX0jCn9kBBgaybsFltpmQ8F7O74hQXY/3yNyCjInDag= github.com/aliyun/alibaba-cloud-sdk-go v1.61.751/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= github.com/anexia-it/go-anxcloud v0.3.26 h1:uStosj8srS6OA1OsPsMJBFqd4Znzl6fEhUv8b3+G8FU= @@ -266,8 +267,9 @@ github.com/evanphx/json-patch v4.0.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= +github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= @@ -280,8 +282,8 @@ github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVB github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -297,9 +299,11 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -308,8 +312,8 @@ github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbV github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= -github.com/go-logr/zapr v1.2.0 h1:n4JnPI1T3Qq1SFEi/F8rwLrZERp2bso19PJZDB9dayk= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= +github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= @@ -618,8 +622,9 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/copystructure v1.0.0 h1:Laisrj+bAB6b/yJwB5Bt3ITZhGJdqmxquMKeZ+mmkFQ= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -630,8 +635,8 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= -github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= @@ -759,8 +764,9 @@ github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.34.0 h1:RBmGO9d/FVjqHT0yUGQwBJhkwKV+wPCn7KGpvfab0uE= +github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -797,8 +803,9 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI= github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE= -github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -820,8 +827,9 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.1-0.20200713175500-884edc58ad08/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= @@ -866,8 +874,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50 h1:uxE3GYdXIOfhMv3unJKETJEhw78gvzuQqRX/rVirc2A= -github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= +github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= +github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vmware/go-vcloud-director/v2 v2.15.0 h1:idQ9NsHLr2dOSLBC8KIdBMq7XOvPiWmfxgWNaf580mk= github.com/vmware/go-vcloud-director/v2 v2.15.0/go.mod h1:2BS1yw61VN34WI0/nUYoInFvBc3Zcuf84d4ESiAAl68= github.com/vmware/govmomi v0.23.1 h1:vU09hxnNR/I7e+4zCJvW+5vHu5dO64Aoe2Lw7Yi/KRg= @@ -928,15 +936,15 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -945,7 +953,7 @@ go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= +go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -968,8 +976,9 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc= +golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1193,8 +1202,9 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 h1:eJv7u3ksNXoLbGSKuv2s/SIO4tJVxc/A+MTpzxDgz/Q= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1536,8 +1546,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8c.io/operating-system-manager v0.4.0 h1:6F9kxELwHmhqLDLAAlodihBOnSfWM+8FPtbWcOshPGU= -k8c.io/operating-system-manager v0.4.0/go.mod h1:pJImhsLb5GJdZunZ47r5Db0ydBwhWxhgL6mUKbU4Vps= +k8c.io/operating-system-manager v0.4.4 h1:uFwZN1WPVQYmXTV0PzZ6jnk5bApY3GnJTsudLpiAQMs= +k8c.io/operating-system-manager v0.4.4/go.mod h1:yxUFYirh0ge8Hf5wUFGDdu7A0czc+2QVzWEWD0hXDs4= k8s.io/api v0.0.0-20190725062911-6607c48751ae/go.mod h1:1O0xzX/RAtnm7l+5VEUxZ1ysO2ghatfq/OZED4zM9kA= k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= diff --git a/pkg/admission/machinedeployments.go b/pkg/admission/machinedeployments.go index 1e183aa80..ac3c03eb6 100644 --- a/pkg/admission/machinedeployments.go +++ b/pkg/admission/machinedeployments.go @@ -22,7 +22,7 @@ import ( "fmt" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - osmadmission "k8c.io/operating-system-manager/pkg/admission" + mdvalidation "k8c.io/operating-system-manager/pkg/admission/machinedeployment/validation" admissionv1 "k8s.io/api/admission/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -47,7 +47,7 @@ func (ad *admissionData) mutateMachineDeployments(ctx context.Context, ar admiss // If OSM is enabled then validate machine deployment against selected OSP if ad.useOSM { - if errs := osmadmission.ValidateMachineDeployment(machineDeployment, ad.client, ad.namespace); len(errs) > 0 { + if errs := mdvalidation.ValidateMachineDeployment(ctx, machineDeployment, ad.client, ad.namespace); len(errs) > 0 { return nil, fmt.Errorf("validation failed: %v", errs) } } diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index aaaf9cf9a..2592657aa 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -194,6 +194,17 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar scenarioParams = append(scenarioParams, fmt.Sprintf("<< CUSTOM-IMAGE >>=%v", "rhel-8-1-custom")) scenarioParams = append(scenarioParams, fmt.Sprintf("<< AMI >>=%s", "ami-08c04369895785ac4")) scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.08")) + } else if testCase.osName == string(providerconfigtypes.OperatingSystemUbuntu) { + // TODO: Remove this when https://github.com/kubermatic/kubermatic/issues/10022 is marked as resolved. + scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_DISK_SIZE >>=%v", 30)) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< DATA_DISK_SIZE >>=%v", 30)) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< AMI >>=%s", "ami-092f628832a8d22a5")) // ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220523 + scenarioParams = append(scenarioParams, fmt.Sprintf("<< DISK_SIZE >>=%v", 25)) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< CUSTOM-IMAGE >>=%v", "")) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< RHEL_SUBSCRIPTION_MANAGER_USER >>=%s", "")) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>=%s", "")) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>=%s", "")) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.03")) } else { scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_DISK_SIZE >>=%v", 30)) scenarioParams = append(scenarioParams, fmt.Sprintf("<< DATA_DISK_SIZE >>=%v", 30)) diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml index e4c0d6375..bc81a8a15 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml @@ -34,7 +34,8 @@ spec: diskSize: 50 diskType: "gp2" ebsVolumeEncrypted: false - ami: "<< AMI >>" + # TODO: Revert this to "<< AMI >>" when https://github.com/kubermatic/kubermatic/issues/10022 is marked as resolved. + ami: "ami-07d0e9bbaa6dad756" securityGroupIDs: - "sg-a2c195ca" tags: diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml index ba06debe1..a1bd27bed 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml @@ -34,6 +34,7 @@ spec: diskSize: 50 diskType: "gp2" ebsVolumeEncrypted: true + ami: "<< AMI >>" securityGroupIDs: - "sg-a2c195ca" tags: From 5e059218b11ed43ac1a7e8fd3aaebea1538cf288 Mon Sep 17 00:00:00 2001 From: Pratik Deoghare Date: Fri, 10 Jun 2022 14:43:58 +0530 Subject: [PATCH 165/489] add dual stack support for DigitalOcean (#1323) * add dual stack support for DigitalOcean * remove misleading comment --- .../provider/digitalocean/provider.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index fcbe3db09..1c5ba1f58 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -34,6 +34,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" digitaloceantypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/digitalocean/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -188,6 +189,17 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return fmt.Errorf("invalid operating system specified %q: %w", pc.OperatingSystem, err) } + switch f := pc.Network.GetIPFamily(); f { + case util.Unspecified, util.IPv4: + // noop + case util.IPv6: + return fmt.Errorf(util.ErrIPv6OnlyUnsupported) + case util.DualStack: + // noop + default: + return fmt.Errorf(util.ErrUnknownNetworkFamily, f) + } + client := getClient(ctx, c.Token) regions, _, err := client.Regions.List(ctx, &godo.ListOptions{PerPage: 1000}) @@ -298,7 +310,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, Name: machine.Spec.Name, Region: c.Region, Size: c.Size, - IPv6: c.IPv6, + IPv6: c.IPv6 || pc.Network.GetIPFamily() == util.DualStack, PrivateNetworking: c.PrivateNetworking, Backups: c.Backups, Monitoring: c.Monitoring, From 62c7a8c084b0d491d94f54993d9a66d461f15930 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 13 Jun 2022 16:18:49 +0500 Subject: [PATCH 166/489] Populate default OSP annotation if an empty value is provided for the OSP key (#1326) Signed-off-by: Waleed Malik --- pkg/admission/machinedeployments_validation.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/admission/machinedeployments_validation.go b/pkg/admission/machinedeployments_validation.go index 664ba6644..57442b589 100644 --- a/pkg/admission/machinedeployments_validation.go +++ b/pkg/admission/machinedeployments_validation.go @@ -148,8 +148,9 @@ func mutationsForMachineDeployment(md *v1alpha1.MachineDeployment, useOSM bool) } func ensureOSPAnnotation(md *v1alpha1.MachineDeployment, providerConfig providerconfigtypes.Config) error { - // Check for existing annotation - if _, ok := md.Annotations[osmresources.MachineDeploymentOSPAnnotation]; !ok { + // Check for existing annotation if it doesn't exist or if the value is empty + // inject the appropriate annotation. + if val, ok := md.Annotations[osmresources.MachineDeploymentOSPAnnotation]; !ok || val == "" { if md.Annotations == nil { md.Annotations = make(map[string]string) } From f90f5bcd1392cd0ceb8af12d3dcd7131a1284817 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Tue, 14 Jun 2022 10:26:48 +0200 Subject: [PATCH 167/489] Remove cluster.cluster.k8s.io CRD and api type (#1296) Signed-off-by: Marvin Beckers --- examples/machine-controller.yaml | 26 --- pkg/apis/cluster/v1alpha1/cluster_types.go | 166 ------------------ .../cluster/v1alpha1/zz_generated.deepcopy.go | 160 ----------------- 3 files changed, 352 deletions(-) delete mode 100644 pkg/apis/cluster/v1alpha1/cluster_types.go diff --git a/examples/machine-controller.yaml b/examples/machine-controller.yaml index 28c183780..fe5a3409e 100644 --- a/examples/machine-controller.yaml +++ b/examples/machine-controller.yaml @@ -155,32 +155,6 @@ spec: jsonPath: .metadata.deletionTimestamp priority: 1 --- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: clusters.cluster.k8s.io - labels: - local-testing: "true" - annotations: - "api-approved.kubernetes.io": "unapproved, legacy API" -spec: - group: cluster.k8s.io - scope: Namespaced - names: - kind: Cluster - plural: clusters - versions: - - name: v1alpha1 - served: true - storage: true - schema: - openAPIV3Schema: - x-kubernetes-preserve-unknown-fields: true - type: object - subresources: - # status enables the status subresource. - status: {} ---- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: diff --git a/pkg/apis/cluster/v1alpha1/cluster_types.go b/pkg/apis/cluster/v1alpha1/cluster_types.go deleted file mode 100644 index 3994e384a..000000000 --- a/pkg/apis/cluster/v1alpha1/cluster_types.go +++ /dev/null @@ -1,166 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/validation/field" -) - -const ClusterFinalizer = "cluster.cluster.k8s.io" - -// +genclient -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -/// [Cluster] -// Cluster is the Schema for the clusters API -// +k8s:openapi-gen=true -// +kubebuilder:resource:shortName=cl -// +kubebuilder:subresource:status -type Cluster struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec ClusterSpec `json:"spec,omitempty"` - Status ClusterStatus `json:"status,omitempty"` -} - -/// [Cluster] - -/// [ClusterSpec] -// ClusterSpec defines the desired state of Cluster. -type ClusterSpec struct { - // Cluster network configuration - ClusterNetwork ClusterNetworkingConfig `json:"clusterNetwork"` - - // Provider-specific serialized configuration to use during - // cluster creation. It is recommended that providers maintain - // their own versioned API types that should be - // serialized/deserialized from this field. - // +optional - ProviderSpec ProviderSpec `json:"providerSpec,omitempty"` -} - -/// [ClusterSpec] - -/// [ClusterNetworkingConfig] -// ClusterNetworkingConfig specifies the different networking -// parameters for a cluster. -type ClusterNetworkingConfig struct { - // The network ranges from which service VIPs are allocated. - Services NetworkRanges `json:"services"` - - // The network ranges from which Pod networks are allocated. - Pods NetworkRanges `json:"pods"` - - // Domain name for services. - ServiceDomain string `json:"serviceDomain"` -} - -/// [ClusterNetworkingConfig] - -/// [NetworkRanges] -// NetworkRanges represents ranges of network addresses. -type NetworkRanges struct { - CIDRBlocks []string `json:"cidrBlocks"` -} - -/// [NetworkRanges] - -/// [ClusterStatus] -// ClusterStatus defines the observed state of Cluster. -type ClusterStatus struct { - // APIEndpoint represents the endpoint to communicate with the IP. - // +optional - APIEndpoints []APIEndpoint `json:"apiEndpoints,omitempty"` - - // NB: Eventually we will redefine ErrorReason as ClusterStatusError once the - // following issue is fixed. - // https://github.com/kubernetes-incubator/apiserver-builder/issues/176 - - // If set, indicates that there is a problem reconciling the - // state, and will be set to a token value suitable for - // programmatic interpretation. - // +optional - ErrorReason common.ClusterStatusError `json:"errorReason,omitempty"` - - // If set, indicates that there is a problem reconciling the - // state, and will be set to a descriptive error message. - // +optional - ErrorMessage string `json:"errorMessage,omitempty"` - - // Provider-specific status. - // It is recommended that providers maintain their - // own versioned API types that should be - // serialized/deserialized from this field. - // +optional - ProviderStatus *runtime.RawExtension `json:"providerStatus,omitempty"` -} - -/// [ClusterStatus] - -/// [APIEndpoint] -// APIEndpoint represents a reachable Kubernetes API endpoint. -type APIEndpoint struct { - // The hostname on which the API server is serving. - Host string `json:"host"` - - // The port on which the API server is serving. - Port int `json:"port"` -} - -/// [APIEndpoint] - -func (o *Cluster) Validate() field.ErrorList { - errors := field.ErrorList{} - // perform validation here and add to errors using field.Invalid - if o.Spec.ClusterNetwork.ServiceDomain == "" { - errors = append(errors, field.Invalid( - field.NewPath("Spec", "ClusterNetwork", "ServiceDomain"), - o.Spec.ClusterNetwork.ServiceDomain, - "invalid cluster configuration: missing Cluster.Spec.ClusterNetwork.ServiceDomain")) - } - if len(o.Spec.ClusterNetwork.Pods.CIDRBlocks) == 0 { - errors = append(errors, field.Invalid( - field.NewPath("Spec", "ClusterNetwork", "Pods"), - o.Spec.ClusterNetwork.Pods, - "invalid cluster configuration: missing Cluster.Spec.ClusterNetwork.Pods")) - } - if len(o.Spec.ClusterNetwork.Services.CIDRBlocks) == 0 { - errors = append(errors, field.Invalid( - field.NewPath("Spec", "ClusterNetwork", "Services"), - o.Spec.ClusterNetwork.Services, - "invalid cluster configuration: missing Cluster.Spec.ClusterNetwork.Services")) - } - return errors -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// ClusterList contains a list of Cluster. -type ClusterList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []Cluster `json:"items"` -} - -func init() { - SchemeBuilder.Register(&Cluster{}, &ClusterList{}) -} diff --git a/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go index e213274a1..9c64da837 100644 --- a/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go @@ -28,145 +28,6 @@ import ( intstr "k8s.io/apimachinery/pkg/util/intstr" ) -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *APIEndpoint) DeepCopyInto(out *APIEndpoint) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIEndpoint. -func (in *APIEndpoint) DeepCopy() *APIEndpoint { - if in == nil { - return nil - } - out := new(APIEndpoint) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Cluster) DeepCopyInto(out *Cluster) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - in.Status.DeepCopyInto(&out.Status) - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cluster. -func (in *Cluster) DeepCopy() *Cluster { - if in == nil { - return nil - } - out := new(Cluster) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *Cluster) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ClusterList) DeepCopyInto(out *ClusterList) { - *out = *in - out.TypeMeta = in.TypeMeta - out.ListMeta = in.ListMeta - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]Cluster, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterList. -func (in *ClusterList) DeepCopy() *ClusterList { - if in == nil { - return nil - } - out := new(ClusterList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *ClusterList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ClusterNetworkingConfig) DeepCopyInto(out *ClusterNetworkingConfig) { - *out = *in - in.Services.DeepCopyInto(&out.Services) - in.Pods.DeepCopyInto(&out.Pods) - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterNetworkingConfig. -func (in *ClusterNetworkingConfig) DeepCopy() *ClusterNetworkingConfig { - if in == nil { - return nil - } - out := new(ClusterNetworkingConfig) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { - *out = *in - in.ClusterNetwork.DeepCopyInto(&out.ClusterNetwork) - in.ProviderSpec.DeepCopyInto(&out.ProviderSpec) - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSpec. -func (in *ClusterSpec) DeepCopy() *ClusterSpec { - if in == nil { - return nil - } - out := new(ClusterSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ClusterStatus) DeepCopyInto(out *ClusterStatus) { - *out = *in - if in.APIEndpoints != nil { - in, out := &in.APIEndpoints, &out.APIEndpoints - *out = make([]APIEndpoint, len(*in)) - copy(*out, *in) - } - if in.ProviderStatus != nil { - in, out := &in.ProviderStatus, &out.ProviderStatus - *out = new(runtime.RawExtension) - (*in).DeepCopyInto(*out) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterStatus. -func (in *ClusterStatus) DeepCopy() *ClusterStatus { - if in == nil { - return nil - } - out := new(ClusterStatus) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LastOperation) DeepCopyInto(out *LastOperation) { *out = *in @@ -758,27 +619,6 @@ func (in *MachineVersionInfo) DeepCopy() *MachineVersionInfo { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *NetworkRanges) DeepCopyInto(out *NetworkRanges) { - *out = *in - if in.CIDRBlocks != nil { - in, out := &in.CIDRBlocks, &out.CIDRBlocks - *out = make([]string, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkRanges. -func (in *NetworkRanges) DeepCopy() *NetworkRanges { - if in == nil { - return nil - } - out := new(NetworkRanges) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ProviderSpec) DeepCopyInto(out *ProviderSpec) { *out = *in From 73a1409ba0bacc5dca13deff25d946cf1cd1b1e5 Mon Sep 17 00:00:00 2001 From: Helene Durand <90329641+hdurand0710@users.noreply.github.com> Date: Mon, 20 Jun 2022 14:49:14 +0200 Subject: [PATCH 168/489] Fix KubeVirt e2e Os images http server (#1329) Signed-off-by: Helene Durand --- test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index 07bb38b40..29a42da25 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -34,7 +34,7 @@ spec: cpus: "1" memory: "4096M" primaryDisk: - osImage: http://10.244.1.19/<< OS_NAME >>.img + osImage: http://image-repo.kube-system.svc.cluster.local/images/<< OS_NAME >>.img size: "25Gi" storageClassName: longhorn dnsPolicy: "None" From eff0fe3b95279d32f311dd5cdafb2eb7620cd147 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Mon, 20 Jun 2022 17:19:12 +0200 Subject: [PATCH 169/489] update Go dependencies (#1330) --- go.mod | 72 ++--- go.sum | 283 ++++++------------ .../provider/anexia/helper_test.go | 2 +- pkg/cloudprovider/provider/anexia/instance.go | 2 +- pkg/cloudprovider/provider/anexia/provider.go | 10 +- .../provider/anexia/provider_test.go | 8 +- pkg/cloudprovider/provider/nutanix/client.go | 53 ++-- .../provider/nutanix/provider.go | 36 +-- 8 files changed, 183 insertions(+), 283 deletions(-) diff --git a/go.mod b/go.mod index a17893129..b257202df 100644 --- a/go.mod +++ b/go.mod @@ -3,46 +3,46 @@ module github.com/kubermatic/machine-controller go 1.18 require ( - cloud.google.com/go/logging v1.4.0 - cloud.google.com/go/monitoring v1.4.0 - github.com/Azure/azure-sdk-for-go v64.1.0+incompatible - github.com/Azure/go-autorest/autorest/azure/auth v0.5.5 + cloud.google.com/go/logging v1.4.2 + cloud.google.com/go/monitoring v1.5.0 + github.com/Azure/azure-sdk-for-go v65.0.0+incompatible + github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/BurntSushi/toml v1.1.0 github.com/Masterminds/semver/v3 v3.1.1 github.com/Masterminds/sprig/v3 v3.2.2 - github.com/aliyun/alibaba-cloud-sdk-go v1.61.751 - github.com/anexia-it/go-anxcloud v0.3.26 - github.com/aws/aws-sdk-go v1.36.2 + github.com/aliyun/alibaba-cloud-sdk-go v1.61.1645 + github.com/aws/aws-sdk-go v1.44.37 github.com/coreos/container-linux-config-transpiler v0.9.0 github.com/davecgh/go-spew v1.1.1 - github.com/digitalocean/godo v1.54.0 - github.com/embik/nutanix-client-go v0.0.0-20220106131900-50b8f27e5f60 + github.com/digitalocean/godo v1.81.0 + github.com/embik/nutanix-client-go v0.1.0 github.com/ghodss/yaml v1.0.0 - github.com/go-test/deep v1.0.7 - github.com/google/uuid v1.1.2 - github.com/gophercloud/gophercloud v0.24.0 + github.com/go-test/deep v1.0.8 + github.com/google/uuid v1.3.0 + github.com/gophercloud/gophercloud v0.25.0 github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 - github.com/hetznercloud/hcloud-go v1.33.1 - github.com/linode/linodego v0.24.0 - github.com/packethost/packngo v0.5.1 + github.com/hetznercloud/hcloud-go v1.34.0 + github.com/linode/linodego v1.8.0 + github.com/packethost/packngo v0.25.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 - github.com/prometheus/client_golang v1.12.1 - github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7 + github.com/prometheus/client_golang v1.12.2 + github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9 github.com/sethvargo/go-password v0.2.0 github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda github.com/vmware/go-vcloud-director/v2 v2.15.0 - github.com/vmware/govmomi v0.23.1 + github.com/vmware/govmomi v0.28.0 + go.anx.io/go-anxcloud v0.4.4 golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f - golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a + golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 gomodules.xyz/jsonpatch/v2 v2.2.0 google.golang.org/api v0.74.0 google.golang.org/grpc v1.45.0 gopkg.in/gcfg.v1 v1.2.3 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gopkg.in/yaml.v3 v3.0.1 k8c.io/operating-system-manager v0.4.4 k8s.io/api v0.24.0 k8s.io/apiextensions-apiserver v0.24.0 @@ -51,9 +51,9 @@ require ( k8s.io/klog v1.0.0 k8s.io/kubelet v0.24.0 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 - kubevirt.io/api v0.48.1 - kubevirt.io/containerized-data-importer-api v1.49.0 - sigs.k8s.io/controller-runtime v0.12.0 + kubevirt.io/api v0.54.0 + kubevirt.io/containerized-data-importer-api v1.50.0 + sigs.k8s.io/controller-runtime v0.12.1 sigs.k8s.io/yaml v1.3.0 ) @@ -61,9 +61,9 @@ require ( cloud.google.com/go v0.100.2 // indirect cloud.google.com/go/compute v1.5.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.18 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect - github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect + github.com/Azure/go-autorest/autorest v0.11.24 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect + github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect @@ -80,11 +80,10 @@ require ( github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/ignition v0.35.0 // indirect - github.com/dimchansky/utfbom v1.1.0 // indirect + github.com/dimchansky/utfbom v1.1.1 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/emicklei/go-restful v2.15.0+incompatible // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -92,13 +91,14 @@ require ( github.com/go-openapi/swag v0.21.1 // indirect github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v4 v4.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.7 // indirect - github.com/google/go-querystring v1.0.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/googleapis/gax-go/v2 v2.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/hashicorp/go-version v1.2.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect @@ -106,7 +106,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kr/pretty v0.2.1 // indirect + github.com/kr/pretty v0.3.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect @@ -122,25 +122,25 @@ require ( github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.34.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect + github.com/rogpeppe/go-internal v1.6.1 // indirect github.com/shopspring/decimal v1.3.1 // indirect - github.com/smartystreets/assertions v1.2.0 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect go.opencensus.io v0.23.0 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/net v0.0.0-20220325170049-de3da57026de // indirect + golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb // indirect - google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9 // indirect + google.golang.org/protobuf v1.28.0 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.62.0 // indirect + gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/component-base v0.24.0 // indirect diff --git a/go.sum b/go.sum index 65ac18e0f..83230eabc 100644 --- a/go.sum +++ b/go.sum @@ -41,10 +41,10 @@ cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6m cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/logging v1.4.0 h1:suMj8d7qzDC9Gzm14aBQGWYZl6TGVz9SyOJDxLN3kNE= -cloud.google.com/go/logging v1.4.0/go.mod h1:FKOKd0UX2KtN01HZbMlVug72OgiX27ZE8AG4lktFnGo= -cloud.google.com/go/monitoring v1.4.0 h1:05+IuNMbh40hbxcqQ4SnynbwZbLG1Wc9dysIJxnfv7U= -cloud.google.com/go/monitoring v1.4.0/go.mod h1:y6xnxfwI3hTFWOdkOaD7nfJVlwuC3/mS/5kvtT131p4= +cloud.google.com/go/logging v1.4.2 h1:Mu2Q75VBDQlW1HlBMjTX4X84UFR73G1TiLlRYc/b7tA= +cloud.google.com/go/logging v1.4.2/go.mod h1:jco9QZSx8HiVVqLJReq7z7bVdj0P1Jb9PDFs63T+axo= +cloud.google.com/go/monitoring v1.5.0 h1:ZltYv8e69fJVga7RTthUBGdx4+Pwz6GRF1V3zylERl4= +cloud.google.com/go/monitoring v1.5.0/go.mod h1:/o9y8NYX5j91JjD/JvGLYbi86kL11OjyJXq2XziLJu4= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -55,23 +55,22 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go v64.1.0+incompatible h1:FpsZmWR9FfEr9hP6K9S7RP0EkSFgGd6P1F2scHtbhnU= -github.com/Azure/azure-sdk-for-go v64.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= +github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.13/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= -github.com/Azure/go-autorest/autorest v0.11.18 h1:90Y4srNYrwOtAgVo3ndrQkTYn6kf1Eg/AjTFJ8Is2aM= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/adal v0.9.8/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= +github.com/Azure/go-autorest/autorest v0.11.24 h1:1fIGgHKqVm54KIPT+q8Zmd1QlVsmHqeUGso5qm2BqqE= +github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.5 h1:7HT2JTm2BOsBMPrT1/iWZW4+XmRvyICcbCejf9BkmYU= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.5/go.mod h1:ptW4D47I+eIUe/lulFLYTVfG4rAARZoXIe1vmTQ+ol8= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 h1:dMOmEJfkLKW/7JsokJqkyoYSgmR08hi9KrhjZb+JALY= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.2/go.mod h1:7qkJkT+j6b+hIpzMOwPChJhTqS8VbsqqgULzMNRugoM= +github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 h1:P6bYXFoao05z5uhOQzbC3Qd8JqF3jUoocoTeIxkp2cA= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.11/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 h1:0W/yGmFdTIT77fvdlGZ0LMISoLHFJ7Tx4U0yeB+uFs4= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= @@ -80,7 +79,6 @@ github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+X github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= -github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= @@ -102,10 +100,8 @@ github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXn github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= @@ -124,16 +120,13 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/aliyun/alibaba-cloud-sdk-go v1.61.751 h1:PX0jCn9kBBgaybsFltpmQ8F7O74hQXY/3yNyCjInDag= -github.com/aliyun/alibaba-cloud-sdk-go v1.61.751/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= -github.com/anexia-it/go-anxcloud v0.3.26 h1:uStosj8srS6OA1OsPsMJBFqd4Znzl6fEhUv8b3+G8FU= -github.com/anexia-it/go-anxcloud v0.3.26/go.mod h1:fiEBxEtBXx78/OWBJvL7+2o4TESrnEcrDYjLeonGkDw= +github.com/aliyun/alibaba-cloud-sdk-go v1.61.1645 h1:IEL/Da0Dtg9j/36UnzyxD84n0eDj0JIoTKTKobN2eks= +github.com/aliyun/alibaba-cloud-sdk-go v1.61.1645/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/appscode/jsonpatch v0.0.0-20190108182946-7c0e3b262f30/go.mod h1:4AJxUpXUhv4N+ziTvIcWWXgeorXpxPZOfk9HdEVr96M= github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195 h1:c4mLfegoDw6OhSJXTd2jUEQgZUQuJWtocudb97Qn9EM= github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -145,8 +138,8 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.36.2 h1:UAeFPct+jHqWM+tgiqDrC9/sfbWj6wkcvpsJ+zdcsvA= -github.com/aws/aws-sdk-go v1.36.2/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.44.37 h1:KvDxCX6dfJeEDC77U5GPGSP0ErecmNnhDHFxw+NIvlI= +github.com/aws/aws-sdk-go v1.44.37/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -156,7 +149,6 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/briandowns/spinner v1.8.0/go.mod h1://Zf9tMcxfRUA36V23M6YGEAv+kECGfvpnLTnb8n4XQ= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= @@ -214,20 +206,17 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892/go.mod h1:CTDl0pzVzE5DEzZhPfvhY/9sPFMQIxaJ9VAMs9AagrE= github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/digitalocean/godo v1.54.0 h1:KP0Nv87pgViR8k/7De3VrmflCL5pJqXbNnkcw0bwG10= -github.com/digitalocean/godo v1.54.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= -github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= -github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= -github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= -github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/digitalocean/godo v1.81.0 h1:sjb3fOfPfSlUQUK22E87BcI8Zx2qtnF7VUCCO4UK3C8= +github.com/digitalocean/godo v1.81.0/go.mod h1:BPCqvwbjbGqxuUnIKB4EvS/AX7IDnNmt5fwvIkWo+ew= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -235,7 +224,6 @@ github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompati github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -243,10 +231,9 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/embik/nutanix-client-go v0.0.0-20220106131900-50b8f27e5f60 h1:0FVKOkpksULFs6F7Kfd8ClBXVTvtiIKl07uV3HinOHk= -github.com/embik/nutanix-client-go v0.0.0-20220106131900-50b8f27e5f60/go.mod h1:gkKNSxfEt3QtYG3S/wKiN8OmrJ4fpU7JbTlbnrMDOL8= +github.com/embik/nutanix-client-go v0.1.0 h1:yPcozUczE2a12RRD/mfk8CehhKPAJWVpisPgqjILpas= +github.com/embik/nutanix-client-go v0.1.0/go.mod h1:gkKNSxfEt3QtYG3S/wKiN8OmrJ4fpU7JbTlbnrMDOL8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible h1:8KpYO/Xl/ZudZs5RNOEhWMBY4hmzlZhhRd9cu+jrZP4= @@ -263,20 +250,14 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch v4.0.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= -github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= @@ -286,7 +267,6 @@ github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwV github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -306,32 +286,23 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.3.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/zapr v0.2.0/go.mod h1:qhKdvif7YF5GI9NWEpyxTSSBdGmzkNguibrdCNVPunU= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= -github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= -github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= -github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= -github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= -github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= -github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= -github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU= @@ -348,11 +319,10 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= -github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/envy v1.7.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w= -github.com/gobuffalo/flect v0.2.2/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs= github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kETFqjFoFlOc= @@ -360,7 +330,6 @@ github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJA github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godror/godror v0.13.3/go.mod h1:2ouUT4kdhUBk7TAkHWD4SN0CdI0pgEQbo8FVHhbSKWg= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -368,6 +337,9 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -387,7 +359,6 @@ github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -429,9 +400,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= @@ -456,25 +426,22 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v0.0.0-20170306145142-6a5e28554805/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0 h1:s7jOdKSaksJVOxE0Y/S32otcfiP+UQ0cL8/GTKaONwE= github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gnostic v0.0.0-20170426233943-68f4ded48ba9/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= -github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/gax-go/v2 v2.3.0 h1:nRJtk3y8Fm770D42QV6T90ZnvFZyk7agSo3Q+Z9p3WI= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gophercloud/gophercloud v0.24.0 h1:jDsIMGJ1KZpAjYfQgGI2coNQj5Q83oPzuiGJRFWgMzw= -github.com/gophercloud/gophercloud v0.24.0/go.mod h1:Q8fZtyi5zZxPS/j9aj3sSxtvj41AdQMDwyo1myduD5c= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gophercloud/gophercloud v0.25.0 h1:C3Oae7y0fUVQGSsBrb3zliAjdX+riCSEh4lNMejFNI4= +github.com/gophercloud/gophercloud v0.25.0/go.mod h1:Q8fZtyi5zZxPS/j9aj3sSxtvj41AdQMDwyo1myduD5c= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -511,7 +478,6 @@ github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -519,8 +485,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 h1:GT4RsKmHh1uZyhmTkWJTDALRjSHYQp6FRKrotf0zhAs= github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= -github.com/hetznercloud/hcloud-go v1.33.1 h1:W1HdO2bRLTKU4WsyqAasDSpt54fYO4WNckWYfH5AuCQ= -github.com/hetznercloud/hcloud-go v1.33.1/go.mod h1:XX/TQub3ge0yWR2yHWmnDVIrB+MQbda1pHxkUmDlUME= +github.com/hetznercloud/hcloud-go v1.34.0 h1:yCmlDl+S9LDDuk0PkStn7XT/DAlBquE5WS4BEnDE5Xc= +github.com/hetznercloud/hcloud-go v1.34.0/go.mod h1:ztUc4lPyGRKJDJ6i8evK4kwAlYO0aZkVAMoZwX9nSjQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= @@ -529,7 +495,6 @@ github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmK github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= @@ -549,7 +514,6 @@ github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUB github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -561,7 +525,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= @@ -577,10 +540,10 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -590,15 +553,13 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.1-0.20191011153232-f91d3411e481/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/linode/linodego v0.24.0 h1:o6hNS0T7jeikOfUHoJhUhA/e2QTCsw9MGccVmRHRLE4= -github.com/linode/linodego v0.24.0/go.mod h1:GSBKPpjoQfxEfryoCRcgkuUOCuVtGHWhzI8OMdycNTE= +github.com/linode/linodego v1.8.0 h1:7B2UaWu6C48tZZZrtINWRElAcwzk4TLnL9USjKf3xm0= +github.com/linode/linodego v1.8.0/go.mod h1:heqhl91D8QTPVm2k9qZHP78zzbOdTFLXE9NJc3bcc50= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -606,11 +567,9 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-oci8 v0.0.7/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -638,12 +597,10 @@ github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= @@ -677,30 +634,28 @@ github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.10.4/go.mod h1:g/HbgYopi++010VEqkFgJHKC09uJiW9UkXvMUuKHUCQ= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= -github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/openshift/custom-resource-status v0.0.0-20200602122900-c002fd1547ca/go.mod h1:GDjWl0tX6FNIj82vIxeudWeSx2Ff6nDZ8uJn0ohUFvo= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= @@ -735,7 +690,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -747,8 +701,9 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= +github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -774,19 +729,19 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -795,9 +750,9 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7 h1:Do8ksLD4Nr3pA0x0hnLOLftZgkiTDvwPDShRTUxtXpE= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8= +github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9 h1:0roa6gXKgyta64uqh52AQG3wzZXH21unn+ltzQSXML0= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -814,10 +769,6 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= -github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= @@ -833,11 +784,9 @@ github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.1-0.20200713175500-884edc58ad08/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= -github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -853,14 +802,13 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/testcontainers/testcontainers-go v0.9.0/go.mod h1:b22BFXhRbg4PJmeMVWh6ftqjyZHgiIl3w274e9r3C2E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -878,9 +826,8 @@ github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8A github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vmware/go-vcloud-director/v2 v2.15.0 h1:idQ9NsHLr2dOSLBC8KIdBMq7XOvPiWmfxgWNaf580mk= github.com/vmware/go-vcloud-director/v2 v2.15.0/go.mod h1:2BS1yw61VN34WI0/nUYoInFvBc3Zcuf84d4ESiAAl68= -github.com/vmware/govmomi v0.23.1 h1:vU09hxnNR/I7e+4zCJvW+5vHu5dO64Aoe2Lw7Yi/KRg= -github.com/vmware/govmomi v0.23.1/go.mod h1:Y+Wq4lst78L85Ge/F8+ORXIWiKYqaro1vhAulACy9Lc= -github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= +github.com/vmware/govmomi v0.28.0 h1:VgeQ/Rvz79U9G8QIKLdgpsN9AndHJL+5iMJLgYIrBGI= +github.com/vmware/govmomi v0.28.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -892,12 +839,12 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +go.anx.io/go-anxcloud v0.4.4 h1:lnsF2H0xad7qbhxHl4wnExKwkaLvOP500SS/V5HnyxU= +go.anx.io/go-anxcloud v0.4.4/go.mod h1:rzQ48vxTWBgS62zNvaJlVfqZfySBBhNcY++rR+MVrPI= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= @@ -946,10 +893,8 @@ go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKY go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.8.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= @@ -965,7 +910,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -976,13 +920,12 @@ golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc= golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= @@ -1018,7 +961,6 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1076,8 +1018,9 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de h1:pZB1TWnKi+o4bENlbzAgLrEbY4RMYmUIRobMcSmfeYc= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5 h1:bRb386wvrE+oBNdF1d/Xh9mQrfQ4ecYhW5qJ5GvTGT4= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1089,16 +1032,16 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a h1:qfl7ob3DIEs3Ml9oLuPwY2N04gymzAW04WsUQHIClgM= golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1111,7 +1054,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1133,7 +1075,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1167,7 +1108,6 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1182,6 +1122,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1202,13 +1143,13 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1223,7 +1164,6 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1231,10 +1171,8 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20180810170437-e96c4e24768d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1244,7 +1182,6 @@ golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -1280,8 +1217,6 @@ golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616195046-dc31b401abb5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -1303,17 +1238,14 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= -gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -1336,7 +1268,7 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4ylg3I= google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= @@ -1399,7 +1331,6 @@ google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201026171402-d4b8fe4fd877/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201102152239-715cce707fb0/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1408,8 +1339,9 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210413151531-c14fb6ef47c3/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= @@ -1436,8 +1368,10 @@ google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2 google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb h1:0m9wktIpOxGw+SSKmydXWB3Z3GTfcPP6+q75HCQa6HI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9 h1:XGQ6tc+EnM35IAazg4y6AHmUg4oK8NXsXaILte1vRlk= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -1487,8 +1421,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1503,13 +1438,12 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= -gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= +gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -1532,8 +1466,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v0.0.0-20181223230014-1083505acf35/go.mod h1:R//lfYlUuTOTfblYI3lGoAAAebUdzjvbmQsuB7Ykd90= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= @@ -1548,54 +1483,31 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8c.io/operating-system-manager v0.4.4 h1:uFwZN1WPVQYmXTV0PzZ6jnk5bApY3GnJTsudLpiAQMs= k8c.io/operating-system-manager v0.4.4/go.mod h1:yxUFYirh0ge8Hf5wUFGDdu7A0czc+2QVzWEWD0hXDs4= -k8s.io/api v0.0.0-20190725062911-6607c48751ae/go.mod h1:1O0xzX/RAtnm7l+5VEUxZ1ysO2ghatfq/OZED4zM9kA= -k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= -k8s.io/api v0.20.2/go.mod h1:d7n6Ehyzx+S+cE3VhTGfVNNqtGc/oL9DCdYYahlurV8= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.24.0 h1:J0hann2hfxWr1hinZIDefw7Q96wmCBx6SSB8IY0MdDg= k8s.io/api v0.24.0/go.mod h1:5Jl90IUrJHUJYEMANRURMiVvJ0g7Ax7r3R1bqO8zx8I= -k8s.io/apiextensions-apiserver v0.20.1/go.mod h1:ntnrZV+6a3dB504qwC5PN/Yg9PBiDNt1EVqbW2kORVk= -k8s.io/apiextensions-apiserver v0.20.2/go.mod h1:F6TXp389Xntt+LUq3vw6HFOLttPa0V8821ogLGwb6Zs= k8s.io/apiextensions-apiserver v0.24.0 h1:JfgFqbA8gKJ/uDT++feAqk9jBIwNnL9YGdQvaI9DLtY= k8s.io/apiextensions-apiserver v0.24.0/go.mod h1:iuVe4aEpe6827lvO6yWQVxiPSpPoSKVjkq+MIdg84cM= -k8s.io/apimachinery v0.0.0-20190719140911-bfcf53abc9f8/go.mod h1:sBJWIJZfxLhp7mRsRyuAE/NfKTr3kXGR1iaqg8O0gJo= -k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.24.0 h1:ydFCyC/DjCvFCHK5OPMKBlxayQytB8pxy8YQInd5UyQ= k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= -k8s.io/apiserver v0.20.2/go.mod h1:2nKd93WyMhZx4Hp3RfgH2K5PhwyTrprrkWYnI7id7jA= k8s.io/apiserver v0.24.0/go.mod h1:WFx2yiOMawnogNToVvUYT9nn1jaIkMKj41ZYCVycsBA= k8s.io/client-go v0.24.0 h1:lbE4aB1gTHvYFSwm6eD3OF14NhFDKCejlnsGYlSJe5U= k8s.io/client-go v0.24.0/go.mod h1:VFPQET+cAFpYxh6Bq6f4xyMY80G6jKKktU6G0m00VDw= -k8s.io/code-generator v0.0.0-20190717022600-77f3a1fe56bb/go.mod h1:cDx5jQmWH25Ff74daM7NVYty9JWw9dvIS9zT9eIubCY= -k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= -k8s.io/code-generator v0.20.2/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/code-generator v0.24.0/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= -k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= -k8s.io/component-base v0.20.2/go.mod h1:pzFtCiwe/ASD0iV7ySMu8SYVJjCapNM9bjvk7ptpKh0= k8s.io/component-base v0.24.0 h1:h5jieHZQoHrY/lHG+HyrSbJeyfuitheBvqvKwKHVC0g= k8s.io/component-base v0.24.0/go.mod h1:Dgazgon0i7KYUsS8krG8muGiMVtUZxG037l1MKyXgrA= -k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= -k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= -k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20190709113604-33be087ad058/go.mod h1:nfDlWeOsu3pUf4yWGL+ERqohP4YsZcBJXWMK+gkzOA4= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= @@ -1603,38 +1515,25 @@ k8s.io/kube-openapi v0.0.0-20220413171646-5e7f5fdc6da6 h1:nBQrWPlrNIiw0BsX6a6MKr k8s.io/kube-openapi v0.0.0-20220413171646-5e7f5fdc6da6/go.mod h1:daOouuuwd9JXpv1L7Y34iV3yf6nxzipkKMWWlqlvK9M= k8s.io/kubelet v0.24.0 h1:fH+D6mSr4DGIeHp/O2+mCEJhkVq3Gpgv9BVOHI+GrWY= k8s.io/kubelet v0.24.0/go.mod h1:p3BBacmHTCMpUf+nluhlyzuGHmONKAspqCvpu9oPAyA= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -kubevirt.io/api v0.48.1 h1:C5i9h8ea7Xy3fJMoKEuzjRP74GnVMF7u2mQV8FGf2XE= -kubevirt.io/api v0.48.1/go.mod h1:RoYMmFt76vWvFtw/FSiL0YUHZ2Ao6UfXlgpZAQnRswo= -kubevirt.io/containerized-data-importer-api v1.41.0/go.mod h1:0xadDFtaMd8iy+/oD2+dYoPxACZ/YizKqay5QIrQ6cw= -kubevirt.io/containerized-data-importer-api v1.49.0 h1:V3eUSKL/kRoJSpQ3FA12vj1jod/QYVXlQEOsv8Cg7mc= -kubevirt.io/containerized-data-importer-api v1.49.0/go.mod h1:yjD8pGZVMCeqcN46JPUQdZ2JwRVoRCOXrTVyNuFvrLo= -kubevirt.io/controller-lifecycle-operator-sdk v0.2.1/go.mod h1:ZJhLceiY2Gl5CXFGSp5eMGt/sksOiJP0289nAZFCQf0= +kubevirt.io/api v0.54.0 h1:rVHaKrsxpYf5Cu6rhASOxNTChS76Nvtn5tArtG2M2Ds= +kubevirt.io/api v0.54.0/go.mod h1:mK8ilpVLcZraqgo7hv2OSNQ5vdsA3G9Pxn8LY2/1+IY= +kubevirt.io/containerized-data-importer-api v1.50.0 h1:O01F8L5K8qRLnkYICIfmAu0dU0P48jdO42uFPElht38= +kubevirt.io/containerized-data-importer-api v1.50.0/go.mod h1:yjD8pGZVMCeqcN46JPUQdZ2JwRVoRCOXrTVyNuFvrLo= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= -modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= -modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= -modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= -modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= -modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= -sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU= -sigs.k8s.io/controller-runtime v0.12.0 h1:gA4zphrmHFc7ihmY/+GyyE0BxKD+OYdb5+DjD2azFAQ= -sigs.k8s.io/controller-runtime v0.12.0/go.mod h1:BKhxlA4l7FPK4AQcsuL4X6vZeWnKDXez/vp1Y8dxTU0= -sigs.k8s.io/controller-tools v0.5.0/go.mod h1:JTsstrMpxs+9BUj6eGuAaEb6SDSPTeVtUyp0jmnAM/I= +sigs.k8s.io/controller-runtime v0.12.1 h1:4BJY01xe9zKQti8oRjj/NeHKRXthf1YkYJAgLONFFoI= +sigs.k8s.io/controller-runtime v0.12.1/go.mod h1:BKhxlA4l7FPK4AQcsuL4X6vZeWnKDXez/vp1Y8dxTU0= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= -sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= diff --git a/pkg/cloudprovider/provider/anexia/helper_test.go b/pkg/cloudprovider/provider/anexia/helper_test.go index f15ecaea2..0bcea21f8 100644 --- a/pkg/cloudprovider/provider/anexia/helper_test.go +++ b/pkg/cloudprovider/provider/anexia/helper_test.go @@ -21,8 +21,8 @@ import ( "net/http" "testing" - "github.com/anexia-it/go-anxcloud/pkg/vsphere/search" "github.com/gophercloud/gophercloud/testhelper" + "go.anx.io/go-anxcloud/pkg/vsphere/search" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" diff --git a/pkg/cloudprovider/provider/anexia/instance.go b/pkg/cloudprovider/provider/anexia/instance.go index 7aae264ba..80c404d38 100644 --- a/pkg/cloudprovider/provider/anexia/instance.go +++ b/pkg/cloudprovider/provider/anexia/instance.go @@ -17,7 +17,7 @@ limitations under the License. package anexia import ( - "github.com/anexia-it/go-anxcloud/pkg/vsphere/info" + "go.anx.io/go-anxcloud/pkg/vsphere/info" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 337389fa0..1a1aa50a6 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -25,11 +25,11 @@ import ( "net/http" "time" - anxclient "github.com/anexia-it/go-anxcloud/pkg/client" - anxaddr "github.com/anexia-it/go-anxcloud/pkg/ipam/address" - "github.com/anexia-it/go-anxcloud/pkg/vsphere" - "github.com/anexia-it/go-anxcloud/pkg/vsphere/provisioning/progress" - anxvm "github.com/anexia-it/go-anxcloud/pkg/vsphere/provisioning/vm" + anxclient "go.anx.io/go-anxcloud/pkg/client" + anxaddr "go.anx.io/go-anxcloud/pkg/ipam/address" + "go.anx.io/go-anxcloud/pkg/vsphere" + "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/progress" + anxvm "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index 9345f7450..d03feda8a 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -24,11 +24,11 @@ import ( "testing" "time" - anxclient "github.com/anexia-it/go-anxcloud/pkg/client" - "github.com/anexia-it/go-anxcloud/pkg/ipam/address" - "github.com/anexia-it/go-anxcloud/pkg/vsphere/provisioning/progress" - "github.com/anexia-it/go-anxcloud/pkg/vsphere/provisioning/vm" "github.com/gophercloud/gophercloud/testhelper" + anxclient "go.anx.io/go-anxcloud/pkg/client" + "go.anx.io/go-anxcloud/pkg/ipam/address" + "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/progress" + "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" diff --git a/pkg/cloudprovider/provider/nutanix/client.go b/pkg/cloudprovider/provider/nutanix/client.go index e337e86b8..e2ef6c9b3 100644 --- a/pkg/cloudprovider/provider/nutanix/client.go +++ b/pkg/cloudprovider/provider/nutanix/client.go @@ -17,6 +17,7 @@ limitations under the License. package nutanix import ( + "context" "encoding/base64" "encoding/json" "errors" @@ -92,18 +93,18 @@ func GetClientSet(config *Config) (*ClientSet, error) { }, nil } -func createVM(client *ClientSet, name string, conf Config, os providerconfigtypes.OperatingSystem, userdata string) (instance.Instance, error) { - cluster, err := getClusterByName(client, conf.ClusterName) +func createVM(ctx context.Context, client *ClientSet, name string, conf Config, os providerconfigtypes.OperatingSystem, userdata string) (instance.Instance, error) { + cluster, err := getClusterByName(ctx, client, conf.ClusterName) if err != nil { return nil, err } - subnet, err := getSubnetByName(client, conf.SubnetName, *cluster.Metadata.UUID) + subnet, err := getSubnetByName(ctx, client, conf.SubnetName, *cluster.Metadata.UUID) if err != nil { return nil, err } - image, err := getImageByName(client, conf.ImageName) + image, err := getImageByName(ctx, client, conf.ImageName) if err != nil { return nil, err } @@ -157,7 +158,7 @@ func createVM(client *ClientSet, name string, conf Config, os providerconfigtype } if conf.ProjectName != "" { - project, err := getProjectByName(client, conf.ProjectName) + project, err := getProjectByName(ctx, client, conf.ProjectName) if err != nil { return nil, fmt.Errorf("failed to get project: %w", err) } @@ -182,14 +183,14 @@ func createVM(client *ClientSet, name string, conf Config, os providerconfigtype request.Spec.Resources = resources - resp, err := client.Prism.V3.CreateVM(request) + resp, err := client.Prism.V3.CreateVM(ctx, request) if err != nil { return nil, wrapNutanixError(err) } taskUUID := resp.Status.ExecutionContext.TaskUUID.(string) - if err := waitForCompletion(client, taskUUID, time.Second*10, time.Minute*15); err != nil { + if err := waitForCompletion(ctx, client, taskUUID, time.Second*10, time.Minute*15); err != nil { return nil, fmt.Errorf("failed to wait for task: %w", err) } @@ -197,11 +198,11 @@ func createVM(client *ClientSet, name string, conf Config, os providerconfigtype return nil, errors.New("did not get response with UUID") } - if err := waitForPowerState(client, *resp.Metadata.UUID, time.Second*10, time.Minute*10); err != nil { + if err := waitForPowerState(ctx, client, *resp.Metadata.UUID, time.Second*10, time.Minute*10); err != nil { return nil, fmt.Errorf("failed to wait for power state: %w", err) } - vm, err := client.Prism.V3.GetVM(*resp.Metadata.UUID) + vm, err := client.Prism.V3.GetVM(ctx, *resp.Metadata.UUID) if err != nil { return nil, wrapNutanixError(err) } @@ -210,7 +211,7 @@ func createVM(client *ClientSet, name string, conf Config, os providerconfigtype return nil, fmt.Errorf("request for VM UUID '%s' did not return name", *resp.Metadata.UUID) } - addresses, err := getIPs(client, *vm.Metadata.UUID, time.Second*5, time.Minute*10) + addresses, err := getIPs(ctx, client, *vm.Metadata.UUID, time.Second*5, time.Minute*10) if err != nil { return nil, fmt.Errorf("failed to get addresses: %w", err) } @@ -223,9 +224,9 @@ func createVM(client *ClientSet, name string, conf Config, os providerconfigtype }, nil } -func getSubnetByName(client *ClientSet, name, clusterID string) (*nutanixv3.SubnetIntentResponse, error) { +func getSubnetByName(ctx context.Context, client *ClientSet, name, clusterID string) (*nutanixv3.SubnetIntentResponse, error) { filter := fmt.Sprintf("name==%s", name) - subnets, err := client.Prism.V3.ListAllSubnet(filter) + subnets, err := client.Prism.V3.ListAllSubnet(ctx, filter) if err != nil { return nil, wrapNutanixError(err) @@ -246,9 +247,9 @@ func getSubnetByName(client *ClientSet, name, clusterID string) (*nutanixv3.Subn } } -func getProjectByName(client *ClientSet, name string) (*nutanixv3.Project, error) { +func getProjectByName(ctx context.Context, client *ClientSet, name string) (*nutanixv3.Project, error) { filter := fmt.Sprintf("name==%s", name) - projects, err := client.Prism.V3.ListAllProject(filter) + projects, err := client.Prism.V3.ListAllProject(ctx, filter) if err != nil { return nil, wrapNutanixError(err) @@ -273,9 +274,9 @@ func getProjectByName(client *ClientSet, name string) (*nutanixv3.Project, error } } -func getClusterByName(client *ClientSet, name string) (*nutanixv3.ClusterIntentResponse, error) { +func getClusterByName(ctx context.Context, client *ClientSet, name string) (*nutanixv3.ClusterIntentResponse, error) { filter := fmt.Sprintf("name==%s", name) - clusters, err := client.Prism.V3.ListAllCluster(filter) + clusters, err := client.Prism.V3.ListAllCluster(ctx, filter) if err != nil { return nil, wrapNutanixError(err) @@ -300,9 +301,9 @@ func getClusterByName(client *ClientSet, name string) (*nutanixv3.ClusterIntentR } } -func getImageByName(client *ClientSet, name string) (*nutanixv3.ImageIntentResponse, error) { +func getImageByName(ctx context.Context, client *ClientSet, name string) (*nutanixv3.ImageIntentResponse, error) { filter := fmt.Sprintf("name==%s", name) - images, err := client.Prism.V3.ListAllImage(filter) + images, err := client.Prism.V3.ListAllImage(ctx, filter) if err != nil { return nil, wrapNutanixError(err) @@ -327,9 +328,9 @@ func getImageByName(client *ClientSet, name string) (*nutanixv3.ImageIntentRespo } } -func getVMByName(client *ClientSet, name string, projectID *string) (*nutanixv3.VMIntentResource, error) { +func getVMByName(ctx context.Context, client *ClientSet, name string, projectID *string) (*nutanixv3.VMIntentResource, error) { filter := fmt.Sprintf("vm_name==%s", name) - vms, err := client.Prism.V3.ListAllVM(filter) + vms, err := client.Prism.V3.ListAllVM(ctx, filter) if err != nil { return nil, wrapNutanixError(err) @@ -348,11 +349,11 @@ func getVMByName(client *ClientSet, name string, projectID *string) (*nutanixv3. return nil, cloudprovidererrors.ErrInstanceNotFound } -func getIPs(client *ClientSet, vmID string, interval time.Duration, timeout time.Duration) (map[string]corev1.NodeAddressType, error) { +func getIPs(ctx context.Context, client *ClientSet, vmID string, interval time.Duration, timeout time.Duration) (map[string]corev1.NodeAddressType, error) { addresses := make(map[string]corev1.NodeAddressType) if err := wait.Poll(interval, timeout, func() (bool, error) { - vm, err := client.Prism.V3.GetVM(vmID) + vm, err := client.Prism.V3.GetVM(ctx, vmID) if err != nil { return false, wrapNutanixError(err) } @@ -372,9 +373,9 @@ func getIPs(client *ClientSet, vmID string, interval time.Duration, timeout time return addresses, nil } -func waitForCompletion(client *ClientSet, taskID string, interval time.Duration, timeout time.Duration) error { +func waitForCompletion(ctx context.Context, client *ClientSet, taskID string, interval time.Duration, timeout time.Duration) error { return wait.Poll(interval, timeout, func() (bool, error) { - task, err := client.Prism.V3.GetTask(taskID) + task, err := client.Prism.V3.GetTask(ctx, taskID) if err != nil { return false, wrapNutanixError(err) } @@ -396,9 +397,9 @@ func waitForCompletion(client *ClientSet, taskID string, interval time.Duration, }) } -func waitForPowerState(client *ClientSet, vmID string, interval time.Duration, timeout time.Duration) error { +func waitForPowerState(ctx context.Context, client *ClientSet, vmID string, interval time.Duration, timeout time.Duration) error { return wait.Poll(interval, timeout, func() (bool, error) { - vm, err := client.Prism.V3.GetVM(vmID) + vm, err := client.Prism.V3.GetVM(ctx, vmID) if err != nil { return false, wrapNutanixError(err) } diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 5c6d7ea31..697a7c363 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -196,7 +196,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, nil } -func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { config, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse machineSpec: %w", err) @@ -207,22 +207,22 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) return fmt.Errorf("failed to construct client: %w", err) } - cluster, err := getClusterByName(client, config.ClusterName) + cluster, err := getClusterByName(ctx, client, config.ClusterName) if err != nil { return fmt.Errorf("failed to get cluster: %w", err) } if config.ProjectName != "" { - if _, err := getProjectByName(client, config.ProjectName); err != nil { + if _, err := getProjectByName(ctx, client, config.ProjectName); err != nil { return fmt.Errorf("failed to get project: %w", err) } } - if _, err := getSubnetByName(client, config.SubnetName, *cluster.Metadata.UUID); err != nil { + if _, err := getSubnetByName(ctx, client, config.SubnetName, *cluster.Metadata.UUID); err != nil { return fmt.Errorf("failed to get subnet: %w", err) } - image, err := getImageByName(client, config.ImageName) + image, err := getImageByName(ctx, client, config.ImageName) if err != nil { return fmt.Errorf("failed to get image: %w", err) } @@ -243,7 +243,7 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) } func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - vm, err := p.create(machine, userdata) + vm, err := p.create(ctx, machine, userdata) if err != nil { _, cleanupErr := p.Cleanup(ctx, machine, data) if cleanupErr != nil { @@ -254,7 +254,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return vm, nil } -func (p *provider) create(machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { +func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -271,14 +271,14 @@ func (p *provider) create(machine *clusterv1alpha1.Machine, userdata string) (in } } - return createVM(client, machine.Name, *config, pc.OperatingSystem, userdata) + return createVM(ctx, client, machine.Name, *config, pc.OperatingSystem, userdata) } -func (p *provider) Cleanup(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - return p.cleanup(machine, data) +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { + return p.cleanup(ctx, machine, data) } -func (p *provider) cleanup(machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, cloudprovidererrors.TerminalError{ @@ -298,7 +298,7 @@ func (p *provider) cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider var projectID *string if config.ProjectName != "" { - project, err := getProjectByName(client, config.ProjectName) + project, err := getProjectByName(ctx, client, config.ProjectName) if err != nil { return false, err } @@ -306,7 +306,7 @@ func (p *provider) cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider projectID = project.Metadata.UUID } - vm, err := getVMByName(client, machine.Name, projectID) + vm, err := getVMByName(ctx, client, machine.Name, projectID) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { // VM is gone already @@ -322,7 +322,7 @@ func (p *provider) cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider // TODO: figure out if VM is already in deleting state - resp, err := client.Prism.V3.DeleteVM(*vm.Metadata.UUID) + resp, err := client.Prism.V3.DeleteVM(ctx, *vm.Metadata.UUID) if err != nil { return false, err } @@ -332,14 +332,14 @@ func (p *provider) cleanup(machine *clusterv1alpha1.Machine, data *cloudprovider return false, errors.New("failed to parse deletion task UUID") } - if err := waitForCompletion(client, taskID, time.Second*5, time.Minute*10); err != nil { + if err := waitForCompletion(ctx, client, taskID, time.Second*5, time.Minute*10); err != nil { return false, fmt.Errorf("failed to wait for completion: %w", err) } return true, nil } -func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -359,7 +359,7 @@ func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, data var projectID *string if config.ProjectName != "" { - project, err := getProjectByName(client, config.ProjectName) + project, err := getProjectByName(ctx, client, config.ProjectName) if err != nil { return nil, err } @@ -367,7 +367,7 @@ func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, data projectID = project.Metadata.UUID } - vm, err := getVMByName(client, machine.Name, projectID) + vm, err := getVMByName(ctx, client, machine.Name, projectID) if err != nil { return nil, err } From f2ffd75df7c51a84b1ff2019b0f4d944d9d25837 Mon Sep 17 00:00:00 2001 From: Mara Sophie Grosch Date: Wed, 22 Jun 2022 14:43:33 +0200 Subject: [PATCH 170/489] anexia: increase HTTP client timeout to 120s (#1331) Signed-off-by: Mara Sophie Grosch --- pkg/cloudprovider/provider/anexia/provider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 1a1aa50a6..7d99885e2 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -484,7 +484,7 @@ func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { func getClient(token string) (anxclient.Client, error) { tokenOpt := anxclient.TokenFromString(token) - client := anxclient.HTTPClient(&http.Client{Timeout: 30 * time.Second}) + client := anxclient.HTTPClient(&http.Client{Timeout: 120 * time.Second}) return anxclient.New(tokenOpt, client) } From 4859a7b7f23c0657c187afd23a8da1eb20534b72 Mon Sep 17 00:00:00 2001 From: jojo Date: Thu, 23 Jun 2022 16:56:06 +0200 Subject: [PATCH 171/489] Add disableMachineServiceAccount configuration to GCE (#1328) * Add disableMachineServiceAccount configuration to GCE Signed-off-by: Jonatas Baldin * Minor fixes on doc, revert Makefile and google monitoring go-sdk version Signed-off-by: Jonatas Baldin Co-authored-by: Jonatas Baldin --- docs/cloud-provider.md | 2 + examples/gce-machinedeployment.yaml | 1 + pkg/cloudprovider/provider/gce/config.go | 44 +++++++++++-------- pkg/cloudprovider/provider/gce/provider.go | 25 ++++++----- .../provider/gce/provider_test.go | 3 +- pkg/cloudprovider/provider/gce/types/types.go | 33 +++++++------- .../testdata/machinedeployment-gce.yaml | 1 + 7 files changed, 62 insertions(+), 47 deletions(-) diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index 8a54fdb87..c18284f43 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -170,6 +170,8 @@ network: "my-cool-network" subnetwork: "my-cool-subnetwork" # assign a public IP Address. Required for Internet access assignPublicIPAddress: true +# if true, does not inject the Service Account from the controller in the machine, leaving it empty +disableMachineServiceAccount: false # set node labels labels: "kubernetesCluster": "my-cluster" diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index 6df8cdb06..37c8eecb2 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -63,6 +63,7 @@ spec: # Whether to assign a public IP Address. Required for Internet access assignPublicIPAddress: true customImage: "myCustomImage" + disableMachineServiceAccount: false # Can be 'ubuntu' or 'rhel' operatingSystem: "ubuntu" operatingSystemSpec: diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index 2e6e50f75..cacea48e6 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -92,25 +92,26 @@ func newCloudProviderSpec(provSpec v1alpha1.ProviderSpec) (*gcetypes.CloudProvid // config contains the configuration of the Provider. type config struct { - serviceAccount string - projectID string - zone string - machineType string - diskSize int64 - diskType string - network string - subnetwork string - preemptible bool - automaticRestart *bool - provisioningModel *string - labels map[string]string - tags []string - jwtConfig *jwt.Config - providerConfig *providerconfigtypes.Config - assignPublicIPAddress bool - multizone bool - regional bool - customImage string + serviceAccount string + projectID string + zone string + machineType string + diskSize int64 + diskType string + network string + subnetwork string + preemptible bool + automaticRestart *bool + provisioningModel *string + labels map[string]string + tags []string + jwtConfig *jwt.Config + providerConfig *providerconfigtypes.Config + assignPublicIPAddress bool + multizone bool + regional bool + customImage string + disableMachineServiceAccount bool } // newConfig creates a Provider configuration out of the passed resolver and spec. @@ -214,6 +215,11 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide return nil, fmt.Errorf("failed to retrieve gce custom image: %w", err) } + cfg.disableMachineServiceAccount, _, err = resolver.GetConfigVarBoolValue(cpSpec.DisableMachineServiceAccount) + if err != nil { + return nil, fmt.Errorf("failed to retrieve disable machine service account: %w", err) + } + return cfg, nil } diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 54f63f9d4..58245db03 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -245,17 +245,6 @@ func (p *Provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, Scheduling: &compute.Scheduling{ Preemptible: cfg.preemptible, }, - ServiceAccounts: []*compute.ServiceAccount{ - { - Email: cfg.jwtConfig.Email, - Scopes: append( - monitoring.DefaultAuthScopes(), - compute.ComputeScope, - compute.DevstorageReadOnlyScope, - logging.WriteScope, - ), - }, - }, Metadata: &compute.Metadata{ Items: []*compute.MetadataItems{ { @@ -269,6 +258,20 @@ func (p *Provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, }, } + if !cfg.disableMachineServiceAccount { + inst.ServiceAccounts = []*compute.ServiceAccount{ + { + Email: cfg.jwtConfig.Email, + Scopes: append( + monitoring.DefaultAuthScopes(), + compute.ComputeScope, + compute.DevstorageReadOnlyScope, + logging.WriteScope, + ), + }, + } + } + if cfg.automaticRestart != nil { inst.Scheduling.AutomaticRestart = cfg.automaticRestart } diff --git a/pkg/cloudprovider/provider/gce/provider_test.go b/pkg/cloudprovider/provider/gce/provider_test.go index d8a36cc42..c792c6555 100644 --- a/pkg/cloudprovider/provider/gce/provider_test.go +++ b/pkg/cloudprovider/provider/gce/provider_test.go @@ -53,7 +53,8 @@ func testProviderSpec() map[string]interface{} { "system-cluster-kdlj8sn58d", "system-project-sszxpzjcnm", }, - "zone": "europe-west2-a", + "zone": "europe-west2-a", + "disableMachineServiceAccount": false, }, "operatingSystem": "ubuntu", "operatingSystemSpec": map[string]interface{}{ diff --git a/pkg/cloudprovider/provider/gce/types/types.go b/pkg/cloudprovider/provider/gce/types/types.go index 7b05eef38..c1b059d8d 100644 --- a/pkg/cloudprovider/provider/gce/types/types.go +++ b/pkg/cloudprovider/provider/gce/types/types.go @@ -30,22 +30,23 @@ import ( // CloudProviderSpec contains the specification of the cloud provider taken // from the provider configuration. type CloudProviderSpec struct { - ServiceAccount providerconfigtypes.ConfigVarString `json:"serviceAccount,omitempty"` - Zone providerconfigtypes.ConfigVarString `json:"zone"` - MachineType providerconfigtypes.ConfigVarString `json:"machineType"` - DiskSize int64 `json:"diskSize"` - DiskType providerconfigtypes.ConfigVarString `json:"diskType"` - Network providerconfigtypes.ConfigVarString `json:"network"` - Subnetwork providerconfigtypes.ConfigVarString `json:"subnetwork"` - Preemptible providerconfigtypes.ConfigVarBool `json:"preemptible"` - AutomaticRestart *providerconfigtypes.ConfigVarBool `json:"automaticRestart,omitempty"` - ProvisioningModel *providerconfigtypes.ConfigVarString `json:"provisioningModel,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - Tags []string `json:"tags,omitempty"` - AssignPublicIPAddress *providerconfigtypes.ConfigVarBool `json:"assignPublicIPAddress,omitempty"` - MultiZone providerconfigtypes.ConfigVarBool `json:"multizone"` - Regional providerconfigtypes.ConfigVarBool `json:"regional"` - CustomImage providerconfigtypes.ConfigVarString `json:"customImage,omitempty"` + ServiceAccount providerconfigtypes.ConfigVarString `json:"serviceAccount,omitempty"` + Zone providerconfigtypes.ConfigVarString `json:"zone"` + MachineType providerconfigtypes.ConfigVarString `json:"machineType"` + DiskSize int64 `json:"diskSize"` + DiskType providerconfigtypes.ConfigVarString `json:"diskType"` + Network providerconfigtypes.ConfigVarString `json:"network"` + Subnetwork providerconfigtypes.ConfigVarString `json:"subnetwork"` + Preemptible providerconfigtypes.ConfigVarBool `json:"preemptible"` + AutomaticRestart *providerconfigtypes.ConfigVarBool `json:"automaticRestart,omitempty"` + ProvisioningModel *providerconfigtypes.ConfigVarString `json:"provisioningModel,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Tags []string `json:"tags,omitempty"` + AssignPublicIPAddress *providerconfigtypes.ConfigVarBool `json:"assignPublicIPAddress,omitempty"` + MultiZone providerconfigtypes.ConfigVarBool `json:"multizone"` + Regional providerconfigtypes.ConfigVarBool `json:"regional"` + CustomImage providerconfigtypes.ConfigVarString `json:"customImage,omitempty"` + DisableMachineServiceAccount providerconfigtypes.ConfigVarBool `json:"disableMachineServiceAccount,omitempty"` } // UpdateProviderSpec updates the given provider spec with changed diff --git a/test/e2e/provisioning/testdata/machinedeployment-gce.yaml b/test/e2e/provisioning/testdata/machinedeployment-gce.yaml index 998a29151..3548f02c5 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-gce.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-gce.yaml @@ -38,6 +38,7 @@ spec: "kubernetes_cluster": "gce-test-cluster" assignPublicIPAddress: true customImage: "<< CUSTOM-IMAGE >>" + disableMachineServiceAccount: false # Can be 'ubuntu' or 'rhel' operatingSystem: "<< OS_NAME >>" operatingSystemSpec: From dfc1e2af71e6c51bfabb88cb99bf70d6fbdebf3e Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Mon, 27 Jun 2022 20:27:09 +0200 Subject: [PATCH 172/489] Support rockylinux in Equinixmetal cloud provider (#1338) * add rockylinux to equinixmetal Signed-off-by: Moath Qasim * add rockylinux to equinixmetal Signed-off-by: Moath Qasim * choose a smaller gce machine Signed-off-by: Moath Qasim --- pkg/cloudprovider/provider/equinixmetal/provider.go | 2 ++ test/e2e/provisioning/all_e2e_test.go | 2 +- .../provisioning/testdata/machinedeployment-equinixmetal.yaml | 4 ++-- test/e2e/provisioning/testdata/machinedeployment-gce.yaml | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index 94ac01b82..9e0430fdc 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -446,6 +446,8 @@ func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { return "centos_7", nil case providerconfigtypes.OperatingSystemFlatcar: return "flatcar_stable", nil + case providerconfigtypes.OperatingSystemRockyLinux: + return "rocky_8", nil } return "", providerconfigtypes.ErrOSNotSupported } diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 2728ed658..5cfdb8334 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -738,7 +738,7 @@ func TestEquinixMetalProvisioningE2E(t *testing.T) { t.Fatal("unable to run the test suite, METAL_PROJECT_ID environment variable cannot be empty") } - selector := Not(OsSelector("sles", "rhel", "amzn2", "rockylinux")) + selector := Not(OsSelector("sles", "rhel", "amzn2")) // act params := []string{ diff --git a/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml b/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml index 28e52c4fa..09d6f043a 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml @@ -26,9 +26,9 @@ spec: cloudProviderSpec: token: << METAL_AUTH_TOKEN >> projectID: << METAL_PROJECT_ID >> - instanceType: "c1.small.x86" + instanceType: "c3.small.x86" facilities: - - "ams1" + - "am6" operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-gce.yaml b/test/e2e/provisioning/testdata/machinedeployment-gce.yaml index 3548f02c5..a2d9eb4d3 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-gce.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-gce.yaml @@ -29,7 +29,7 @@ spec: # See https://cloud.google.com/compute/docs/regions-zones/ zone: "europe-west3-a" # See https://cloud.google.com/compute/docs/machine-types - machineType: "n1-standard-2" + machineType: "n1-standard-1" # In GB diskSize: 25 # Can be 'pd-standard' or 'pd-ssd' From ff7dbe5a87bb806cb4a0401435290fb1aaffbcee Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Wed, 29 Jun 2022 15:03:19 +0200 Subject: [PATCH 173/489] add metro support for equinixmetal (#1339) Signed-off-by: Moath Qasim --- go.mod | 2 - go.sum | 4 ++ .../provider/equinixmetal/provider.go | 46 +++++++++++++++---- .../provider/equinixmetal/types/types.go | 3 +- test/e2e/provisioning/all_e2e_test.go | 2 +- test/e2e/provisioning/helper.go | 32 +++++++------ .../machinedeployment-equinixmetal.yaml | 5 +- 7 files changed, 62 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index b257202df..e44da1dba 100644 --- a/go.mod +++ b/go.mod @@ -152,8 +152,6 @@ require ( ) replace ( - github.com/packethost/packngo => github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888 - k8s.io/client-go => k8s.io/client-go v0.24.0 k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.24.0 ) diff --git a/go.sum b/go.sum index 83230eabc..6a6aaad78 100644 --- a/go.sum +++ b/go.sum @@ -216,6 +216,7 @@ github.com/digitalocean/godo v1.81.0 h1:sjb3fOfPfSlUQUK22E87BcI8Zx2qtnF7VUCCO4UK github.com/digitalocean/godo v1.81.0/go.mod h1:BPCqvwbjbGqxuUnIKB4EvS/AX7IDnNmt5fwvIkWo+ew= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= @@ -668,6 +669,8 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888 h1:ARzSqjQJcSR9IXGMEkXN1IQEZChwd2MqXWm6YNGvO5o= github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888/go.mod h1:RQHg5xR1F614BwJyepfMqrKN+32IH0i7yX+ey43rEeQ= +github.com/packethost/packngo v0.25.0 h1:ujGXL3lVqTiaQoX2/Go74lQAlYfTeop7jBNy5w99w2A= +github.com/packethost/packngo v0.25.0/go.mod h1:/UHguFdPs6Lf6FOkkSEPnRY5tgS0fsVM+Zv/bvBrmt0= github.com/packethost/pkg v0.0.0-20200903155310-0433e0605550/go.mod h1:GSv7cTtIjns4yc0pyajaM1RE/KE4djJONoblFIRDrxA= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -915,6 +918,7 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index 9e0430fdc..13de4e467 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -56,6 +56,7 @@ type Config struct { ProjectID string BillingCycle string InstanceType string + Metro string Facilities []string Tags []string } @@ -140,6 +141,10 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *e } c.Facilities = append(c.Facilities, facilityValue) } + c.Metro, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Metro) + if err != nil { + return nil, nil, nil, fmt.Errorf("failed to get the value of \"metro\" field, error = %w", err) + } // ensure we have defaults c.populateDefaults() @@ -187,18 +192,38 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) client := getClient(c.Token) - if len(c.Facilities) == 0 || c.Facilities[0] == "" { - return fmt.Errorf("must have at least one non-blank facility") + if c.Metro == "" && (len(c.Facilities) == 0 || c.Facilities[0] == "") { + return fmt.Errorf("must have at least one non-blank facility or a metro") } - // get all valid facilities - facilities, _, err := client.Facilities.List(nil) - if err != nil { - return fmt.Errorf("failed to list facilities: %w", err) + if c.Facilities != nil && (len(c.Facilities) > 0 || c.Facilities[0] != "") { + // get all valid facilities + facilities, _, err := client.Facilities.List(nil) + if err != nil { + return fmt.Errorf("failed to list facilities: %w", err) + } + // ensure our requested facilities are in those facilities + if missingFacilities := itemsNotInList(facilityProp(facilities, "Code"), c.Facilities); len(missingFacilities) > 0 { + return fmt.Errorf("unknown facilities: %s", strings.Join(missingFacilities, ",")) + } } - // ensure our requested facilities are in those facilities - if missingFacilities := itemsNotInList(facilityProp(facilities, "Code"), c.Facilities); len(missingFacilities) > 0 { - return fmt.Errorf("unknown facilities: %s", strings.Join(missingFacilities, ",")) + + if c.Metro != "" { + metros, _, err := client.Metros.List(nil) + if err != nil { + return fmt.Errorf("failed to list metros: %w", err) + } + + var metroExists bool + for _, metro := range metros { + if strings.EqualFold(metro.Code, c.Metro) { + metroExists = true + } + } + + if !metroExists { + return fmt.Errorf("unknown metro: %s", c.Metro) + } } // get all valid plans a.k.a. instance types @@ -239,6 +264,7 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d UserData: userdata, ProjectID: c.ProjectID, Facility: c.Facilities, + Metro: c.Metro, BillingCycle: c.BillingCycle, Plan: c.InstanceType, OS: imageName, @@ -273,7 +299,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine } client := getClient(c.Token) - res, err := client.Devices.Delete(instance.(*metalDevice).device.ID) + res, err := client.Devices.Delete(instance.(*metalDevice).device.ID, false) if err != nil { return false, metalErrorToTerminalError(err, res, "failed to delete the server") } diff --git a/pkg/cloudprovider/provider/equinixmetal/types/types.go b/pkg/cloudprovider/provider/equinixmetal/types/types.go index 676c0f14a..b34625af0 100644 --- a/pkg/cloudprovider/provider/equinixmetal/types/types.go +++ b/pkg/cloudprovider/provider/equinixmetal/types/types.go @@ -26,7 +26,8 @@ type RawConfig struct { ProjectID providerconfigtypes.ConfigVarString `json:"projectID,omitempty"` BillingCycle providerconfigtypes.ConfigVarString `json:"billingCycle"` InstanceType providerconfigtypes.ConfigVarString `json:"instanceType"` - Facilities []providerconfigtypes.ConfigVarString `json:"facilities"` + Metro providerconfigtypes.ConfigVarString `json:"metro,omitempty"` + Facilities []providerconfigtypes.ConfigVarString `json:"facilities,omitempty"` Tags []providerconfigtypes.ConfigVarString `json:"tags,omitempty"` } diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 5cfdb8334..c291f6885 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -738,7 +738,7 @@ func TestEquinixMetalProvisioningE2E(t *testing.T) { t.Fatal("unable to run the test suite, METAL_PROJECT_ID environment variable cannot be empty") } - selector := Not(OsSelector("sles", "rhel", "amzn2")) + selector := And(OsSelector("ubuntu", "centos", "rockylinux", "flatcar"), Not(NameSelector("migrateUID"))) // act params := []string{ diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 2592657aa..446b6d068 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -194,29 +194,32 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar scenarioParams = append(scenarioParams, fmt.Sprintf("<< CUSTOM-IMAGE >>=%v", "rhel-8-1-custom")) scenarioParams = append(scenarioParams, fmt.Sprintf("<< AMI >>=%s", "ami-08c04369895785ac4")) scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.08")) - } else if testCase.osName == string(providerconfigtypes.OperatingSystemUbuntu) { - // TODO: Remove this when https://github.com/kubermatic/kubermatic/issues/10022 is marked as resolved. - scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_DISK_SIZE >>=%v", 30)) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< DATA_DISK_SIZE >>=%v", 30)) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< AMI >>=%s", "ami-092f628832a8d22a5")) // ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-20220523 - scenarioParams = append(scenarioParams, fmt.Sprintf("<< DISK_SIZE >>=%v", 25)) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< CUSTOM-IMAGE >>=%v", "")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< RHEL_SUBSCRIPTION_MANAGER_USER >>=%s", "")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>=%s", "")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>=%s", "")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.03")) } else { scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_DISK_SIZE >>=%v", 30)) scenarioParams = append(scenarioParams, fmt.Sprintf("<< DATA_DISK_SIZE >>=%v", 30)) scenarioParams = append(scenarioParams, fmt.Sprintf("<< AMI >>=%s", "")) scenarioParams = append(scenarioParams, fmt.Sprintf("<< DISK_SIZE >>=%v", 25)) scenarioParams = append(scenarioParams, fmt.Sprintf("<< CUSTOM-IMAGE >>=%v", "")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< RHEL_SUBSCRIPTION_MANAGER_USER >>=%s", "")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>=%s", "")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>=%s", "")) scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.03")) } + if strings.Contains(cloudProvider, string(providerconfigtypes.CloudProviderEquinixMetal)) { + switch testCase.osName { + case string(providerconfigtypes.OperatingSystemCentOS): + scenarioParams = append(scenarioParams, fmt.Sprintf("<< INSTANCE_TYPE >>=%s", "c3.small.x86")) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< METRO_CODE >>=%s", "AM")) + case string(providerconfigtypes.OperatingSystemFlatcar): + scenarioParams = append(scenarioParams, fmt.Sprintf("<< INSTANCE_TYPE >>=%s", "c3.small.x86")) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< METRO_CODE >>=%s", "NY")) + case string(providerconfigtypes.OperatingSystemRockyLinux): + scenarioParams = append(scenarioParams, fmt.Sprintf("<< INSTANCE_TYPE >>=%s", "m3.small.x86")) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< METRO_CODE >>=%s", "AM")) + case string(providerconfigtypes.OperatingSystemUbuntu): + scenarioParams = append(scenarioParams, fmt.Sprintf("<< INSTANCE_TYPE >>=%s", "m3.small.x86")) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< METRO_CODE >>=%s", "TY")) + } + } + // only used by assume role scenario, otherwise empty (disabled) scenarioParams = append(scenarioParams, fmt.Sprintf("<< AWS_ASSUME_ROLE_ARN >>=%s", os.Getenv("AWS_ASSUME_ROLE_ARN"))) scenarioParams = append(scenarioParams, fmt.Sprintf("<< AWS_ASSUME_ROLE_EXTERNAL_ID >>=%s", os.Getenv("AWS_ASSUME_ROLE_EXTERNAL_ID"))) @@ -270,6 +273,5 @@ func buildScenarios() []scenario { osName: "ubuntu", executor: verifyMigrateUID, }) - return all } diff --git a/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml b/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml index 09d6f043a..52ecd2f2f 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml @@ -26,9 +26,8 @@ spec: cloudProviderSpec: token: << METAL_AUTH_TOKEN >> projectID: << METAL_PROJECT_ID >> - instanceType: "c3.small.x86" - facilities: - - "am6" + instanceType: << INSTANCE_TYPE >> + metro: << METRO_CODE >> operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false From ce44d90a9c1306e2a47df1b3fa5a71f8f336f8dd Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 29 Jun 2022 19:20:13 +0500 Subject: [PATCH 174/489] Upgrade to go 1.18.3 (#1341) Signed-off-by: Waleed Malik --- .prow/verify.yaml | 6 +++--- Dockerfile | 2 +- Makefile | 2 +- hack/update-fixtures.sh | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.prow/verify.yaml b/.prow/verify.yaml index dc256eec7..66eca1e82 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.2 + - image: golang:1.18.3 command: - make args: @@ -39,7 +39,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.2 + - image: golang:1.18.3 command: - make args: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.2 + - image: golang:1.18.3 command: - make args: diff --git a/Dockerfile b/Dockerfile index 9485e86db..e21dccc34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.18.2 +ARG GO_VERSION=1.18.3 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index 8cce2f7a5..71feb43fa 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.18.2 +GO_VERSION ?= 1.18.3 GOOS ?= $(shell go env GOOS) diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index a85226f41..f91ab4a2f 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.18.2 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.18.3 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... From 2ad3da0861e65c372afaf732d4186538090bfec8 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 29 Jun 2022 23:35:09 +0500 Subject: [PATCH 175/489] Consume bootstrap config generated by OSM (#1335) * Remove dead code Signed-off-by: Waleed Malik * Consume bootstrap config generated by OSM Signed-off-by: Waleed Malik * Refactor: extract API server token for baremetal nodes Signed-off-by: Waleed Malik * Update OSM depedency Signed-off-by: Waleed Malik --- go.mod | 53 +-- go.sum | 125 ++++-- .../provider/kubevirt/provider.go | 2 +- pkg/cloudprovider/util/cloud_init_settings.go | 51 +-- .../util/cloud_init_settings_test.go | 2 +- pkg/controller/machine/bootstrap.go | 379 +----------------- pkg/controller/machine/machine_controller.go | 59 ++- pkg/controller/util/machine.go | 9 +- pkg/userdata/amzn2/provider.go | 7 - pkg/userdata/centos/provider.go | 7 - pkg/userdata/helper/helper.go | 11 - pkg/userdata/rhel/provider.go | 7 - pkg/userdata/rockylinux/provider.go | 7 - pkg/userdata/sles/provider.go | 7 - pkg/userdata/ubuntu/provider.go | 7 - 15 files changed, 179 insertions(+), 554 deletions(-) diff --git a/go.mod b/go.mod index e44da1dba..c21b40bb2 100644 --- a/go.mod +++ b/go.mod @@ -36,20 +36,20 @@ require ( github.com/vmware/go-vcloud-director/v2 v2.15.0 github.com/vmware/govmomi v0.28.0 go.anx.io/go-anxcloud v0.4.4 - golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f - golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 + golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e + golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb gomodules.xyz/jsonpatch/v2 v2.2.0 google.golang.org/api v0.74.0 google.golang.org/grpc v1.45.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 - k8c.io/operating-system-manager v0.4.4 - k8s.io/api v0.24.0 - k8s.io/apiextensions-apiserver v0.24.0 - k8s.io/apimachinery v0.24.0 + k8c.io/operating-system-manager v0.5.0 + k8s.io/api v0.24.2 + k8s.io/apiextensions-apiserver v0.24.2 + k8s.io/apimachinery v0.24.2 k8s.io/client-go v12.0.0+incompatible k8s.io/klog v1.0.0 - k8s.io/kubelet v0.24.0 + k8s.io/kubelet v0.24.2 k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 kubevirt.io/api v0.54.0 kubevirt.io/containerized-data-importer-api v1.50.0 @@ -69,8 +69,6 @@ require ( github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect @@ -82,27 +80,29 @@ require ( github.com/coreos/ignition v0.35.0 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect - github.com/emicklei/go-restful v2.15.0+incompatible // indirect + github.com/emicklei/go-restful/v3 v3.8.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect + github.com/flatcar-linux/container-linux-config-transpiler v0.9.3 // indirect + github.com/flatcar-linux/ignition v0.36.1 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.21.1 // indirect github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.2.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.7 // indirect + github.com/google/gnostic v0.6.9 // indirect + github.com/google/go-cmp v0.5.8 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/googleapis/gax-go/v2 v2.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/hashicorp/go-version v1.2.0 // indirect github.com/huandu/xstrings v1.3.2 // indirect - github.com/imdario/mergo v0.3.12 // indirect + github.com/imdario/mergo v0.3.13 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -120,21 +120,24 @@ require ( github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/peterhellberg/link v1.1.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.34.0 // indirect + github.com/prometheus/common v0.35.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/rogpeppe/go-internal v1.6.1 // indirect github.com/shopspring/decimal v1.3.1 // indirect - github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/cast v1.5.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect go.opencensus.io v0.23.0 // indirect + go.uber.org/atomic v1.9.0 // indirect + go.uber.org/multierr v1.8.0 // indirect + go.uber.org/zap v1.21.0 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/net v0.0.0-20220412020605-290c469a71a5 // indirect + golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect + golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect + golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect + golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9 // indirect google.golang.org/protobuf v1.28.0 // indirect @@ -143,15 +146,15 @@ require ( gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/component-base v0.24.0 // indirect + k8s.io/component-base v0.24.2 // indirect k8s.io/klog/v2 v2.60.1 // indirect - k8s.io/kube-openapi v0.0.0-20220413171646-5e7f5fdc6da6 // indirect + k8s.io/kube-openapi v0.0.0-20220614142933-1062c7ade5f8 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect - sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect + sigs.k8s.io/json v0.0.0-20220525155127-227cbc7cc124 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect ) replace ( - k8s.io/client-go => k8s.io/client-go v0.24.0 - k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.24.0 + k8s.io/client-go => k8s.io/client-go v0.24.2 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.24.2 ) diff --git a/go.sum b/go.sum index 6a6aaad78..a42511c89 100644 --- a/go.sum +++ b/go.sum @@ -100,14 +100,13 @@ github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXn github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/ajeddeloh/go-json v0.0.0-20160803184958-73d058cf8437/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 h1:4SPQljF/GJ8Q+QlCWMWxRBepub4DresnOm4eI2ebFGc= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd h1:NlKlOv3aVJ5ODMC0JWPvddw05KENkL3cZttIuu8kJRo= @@ -118,6 +117,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1645 h1:IEL/Da0Dtg9j/36UnzyxD84n0eDj0JIoTKTKobN2eks= @@ -137,11 +137,13 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= +github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.44.37 h1:KvDxCX6dfJeEDC77U5GPGSP0ErecmNnhDHFxw+NIvlI= github.com/aws/aws-sdk-go v1.44.37/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -151,6 +153,7 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/briandowns/spinner v1.8.0/go.mod h1://Zf9tMcxfRUA36V23M6YGEAv+kECGfvpnLTnb8n4XQ= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -187,10 +190,12 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.1.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -237,8 +242,9 @@ github.com/embik/nutanix-client-go v0.1.0 h1:yPcozUczE2a12RRD/mfk8CehhKPAJWVpisP github.com/embik/nutanix-client-go v0.1.0/go.mod h1:gkKNSxfEt3QtYG3S/wKiN8OmrJ4fpU7JbTlbnrMDOL8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.15.0+incompatible h1:8KpYO/Xl/ZudZs5RNOEhWMBY4hmzlZhhRd9cu+jrZP4= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= +github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -257,11 +263,17 @@ github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flatcar-linux/container-linux-config-transpiler v0.9.3 h1:0Leh4HX8Wpe/PYuNidytk6v+2mIFHybK50DWipiCnng= +github.com/flatcar-linux/container-linux-config-transpiler v0.9.3/go.mod h1:AGVTulMzeIKwurV9ExYH3UiokET1Ur65g+EIeRDMwzM= +github.com/flatcar-linux/ignition v0.36.1 h1:yNvS9sQvm9HJ8VgxXskx88DsF73qdF35ALJkbTwcYhY= +github.com/flatcar-linux/ignition v0.36.1/go.mod h1:0jS5n4AopgOdwgi7QDo5MFgkMx/fQUDYjuxlGJC1Txg= +github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= @@ -276,6 +288,7 @@ github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aev github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= @@ -301,8 +314,9 @@ github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUe github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= @@ -328,6 +342,7 @@ github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8z github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kETFqjFoFlOc= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godror/godror v0.13.3/go.mod h1:2ouUT4kdhUBk7TAkHWD4SN0CdI0pgEQbo8FVHhbSKWg= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= @@ -385,8 +400,9 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/cel-go v0.10.1/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= +github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -399,8 +415,9 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -497,13 +514,14 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= -github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -667,8 +685,6 @@ github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxS github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888 h1:ARzSqjQJcSR9IXGMEkXN1IQEZChwd2MqXWm6YNGvO5o= -github.com/packethost/packngo v0.1.1-0.20190410075950-a02c426e4888/go.mod h1:RQHg5xR1F614BwJyepfMqrKN+32IH0i7yX+ey43rEeQ= github.com/packethost/packngo v0.25.0 h1:ujGXL3lVqTiaQoX2/Go74lQAlYfTeop7jBNy5w99w2A= github.com/packethost/packngo v0.25.0/go.mod h1:/UHguFdPs6Lf6FOkkSEPnRY5tgS0fsVM+Zv/bvBrmt0= github.com/packethost/pkg v0.0.0-20200903155310-0433e0605550/go.mod h1:GSv7cTtIjns4yc0pyajaM1RE/KE4djJONoblFIRDrxA= @@ -676,6 +692,7 @@ github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIw github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= @@ -687,6 +704,7 @@ github.com/peterhellberg/link v1.1.0 h1:s2+RH8EGuI/mI4QwrWGSYQCRz7uNgip9BaM04HKu github.com/peterhellberg/link v1.1.0/go.mod h1:gtSlOT4jmkY8P47hbTc8PTgiDDWpdPbFYl75keYyBB8= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pin/tftp v2.1.0+incompatible/go.mod h1:xVpZOMCXTy+A5QMjEVN0Glwa1sUvaJhFXbr/aAxuxGY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -723,8 +741,8 @@ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt2 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.34.0 h1:RBmGO9d/FVjqHT0yUGQwBJhkwKV+wPCn7KGpvfab0uE= -github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= +github.com/prometheus/common v0.35.0 h1:Eyr+Pw2VymWejHqCugNaQXkAi6KayVNxaHeu6khmFBE= +github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -765,6 +783,8 @@ github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sigma/bdoor v0.0.0-20160202064022-babf2a4017b0/go.mod h1:WBu7REWbxC/s/J06jsk//d+9DOz9BbsmcIrimuGRFbs= +github.com/sigma/vmw-guestinfo v0.0.0-20160204083807-95dd4126d6e8/go.mod h1:JrRFFC0veyh0cibh0DAhriSY7/gV3kDdNaVUOmfx01U= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -772,7 +792,9 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= @@ -782,8 +804,8 @@ github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTd github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= -github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.1-0.20200713175500-884edc58ad08/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= @@ -831,7 +853,12 @@ github.com/vmware/go-vcloud-director/v2 v2.15.0 h1:idQ9NsHLr2dOSLBC8KIdBMq7XOvPi github.com/vmware/go-vcloud-director/v2 v2.15.0/go.mod h1:2BS1yw61VN34WI0/nUYoInFvBc3Zcuf84d4ESiAAl68= github.com/vmware/govmomi v0.28.0 h1:VgeQ/Rvz79U9G8QIKLdgpsN9AndHJL+5iMJLgYIrBGI= github.com/vmware/govmomi v0.28.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= +github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= +github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -888,13 +915,16 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= +go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= @@ -902,12 +932,13 @@ go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go4.org v0.0.0-20160314031811-03efcb870d84/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -926,8 +957,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc= -golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -976,6 +1007,7 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1023,8 +1055,8 @@ golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5 h1:bRb386wvrE+oBNdF1d/Xh9mQrfQ4ecYhW5qJ5GvTGT4= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 h1:Yqz/iviulwKwAREEeUd3nbBFn0XuyJqkoft2IlrvOhc= +golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1044,8 +1076,8 @@ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb h1:8tDJ3aechhddbdPAxpycgXHJRMLpk/Ab+aa4OgdN5/g= +golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1068,7 +1100,6 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1149,14 +1180,17 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad h1:ntjMns5wyP/fN65tdBD4g8J5w8n015+iIIs9rtjXkY0= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c h1:aFV+BgZ4svzjfabn8ERpuB4JI4N6/rdy1iusx77G3oU= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= +golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20190321115727-fe223c5a2583/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -1169,8 +1203,9 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180810170437-e96c4e24768d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1471,6 +1506,7 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v0.0.0-20181223230014-1083505acf35/go.mod h1:R//lfYlUuTOTfblYI3lGoAAAebUdzjvbmQsuB7Ykd90= @@ -1485,23 +1521,23 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8c.io/operating-system-manager v0.4.4 h1:uFwZN1WPVQYmXTV0PzZ6jnk5bApY3GnJTsudLpiAQMs= -k8c.io/operating-system-manager v0.4.4/go.mod h1:yxUFYirh0ge8Hf5wUFGDdu7A0czc+2QVzWEWD0hXDs4= +k8c.io/operating-system-manager v0.5.0 h1:HRPPhJG27gl5T/HpGXHrzdbONVmdi1UnASZpKln1N04= +k8c.io/operating-system-manager v0.5.0/go.mod h1:8Q1xpjJomTG9X6lfx/y3+yGHCackHtqxuYEk0TIPMfA= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= -k8s.io/api v0.24.0 h1:J0hann2hfxWr1hinZIDefw7Q96wmCBx6SSB8IY0MdDg= -k8s.io/api v0.24.0/go.mod h1:5Jl90IUrJHUJYEMANRURMiVvJ0g7Ax7r3R1bqO8zx8I= -k8s.io/apiextensions-apiserver v0.24.0 h1:JfgFqbA8gKJ/uDT++feAqk9jBIwNnL9YGdQvaI9DLtY= -k8s.io/apiextensions-apiserver v0.24.0/go.mod h1:iuVe4aEpe6827lvO6yWQVxiPSpPoSKVjkq+MIdg84cM= +k8s.io/api v0.24.2 h1:g518dPU/L7VRLxWfcadQn2OnsiGWVOadTLpdnqgY2OI= +k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= +k8s.io/apiextensions-apiserver v0.24.2 h1:/4NEQHKlEz1MlaK/wHT5KMKC9UKYz6NZz6JE6ov4G6k= +k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZfJiGOW6mm2hLQ= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.24.0 h1:ydFCyC/DjCvFCHK5OPMKBlxayQytB8pxy8YQInd5UyQ= -k8s.io/apimachinery v0.24.0/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apiserver v0.24.0/go.mod h1:WFx2yiOMawnogNToVvUYT9nn1jaIkMKj41ZYCVycsBA= -k8s.io/client-go v0.24.0 h1:lbE4aB1gTHvYFSwm6eD3OF14NhFDKCejlnsGYlSJe5U= -k8s.io/client-go v0.24.0/go.mod h1:VFPQET+cAFpYxh6Bq6f4xyMY80G6jKKktU6G0m00VDw= +k8s.io/apimachinery v0.24.2 h1:5QlH9SL2C8KMcrNJPor+LbXVTaZRReml7svPEh4OKDM= +k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= +k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI= +k8s.io/client-go v0.24.2 h1:CoXFSf8if+bLEbinDqN9ePIDGzcLtqhfd6jpfnwGOFA= +k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/code-generator v0.24.0/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= -k8s.io/component-base v0.24.0 h1:h5jieHZQoHrY/lHG+HyrSbJeyfuitheBvqvKwKHVC0g= -k8s.io/component-base v0.24.0/go.mod h1:Dgazgon0i7KYUsS8krG8muGiMVtUZxG037l1MKyXgrA= +k8s.io/code-generator v0.24.2/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= +k8s.io/component-base v0.24.2 h1:kwpQdoSfbcH+8MPN4tALtajLDfSfYxBDYlXobNWI6OU= +k8s.io/component-base v0.24.2/go.mod h1:ucHwW76dajvQ9B7+zecZAP3BVqvrHoOxm8olHEg0nmM= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= @@ -1515,10 +1551,10 @@ k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= -k8s.io/kube-openapi v0.0.0-20220413171646-5e7f5fdc6da6 h1:nBQrWPlrNIiw0BsX6a6MKr1itkm0ZS0Nl97kNLitFfI= -k8s.io/kube-openapi v0.0.0-20220413171646-5e7f5fdc6da6/go.mod h1:daOouuuwd9JXpv1L7Y34iV3yf6nxzipkKMWWlqlvK9M= -k8s.io/kubelet v0.24.0 h1:fH+D6mSr4DGIeHp/O2+mCEJhkVq3Gpgv9BVOHI+GrWY= -k8s.io/kubelet v0.24.0/go.mod h1:p3BBacmHTCMpUf+nluhlyzuGHmONKAspqCvpu9oPAyA= +k8s.io/kube-openapi v0.0.0-20220614142933-1062c7ade5f8 h1:IyQ1DifCBk589JD4Cm2CT2poIdO3lfPzz3WwVh1Ugf8= +k8s.io/kube-openapi v0.0.0-20220614142933-1062c7ade5f8/go.mod h1:guXtiQW/y/AWAfPSOaI/1eY0TGBAmL5OygiIyUOKDRc= +k8s.io/kubelet v0.24.2 h1:VAvULig8RiylCtyxudgHV7nhKsLnNIrdVBCRD4bXQ3Y= +k8s.io/kubelet v0.24.2/go.mod h1:Xm9DkWQjwOs+uGOUIIGIPMvvmenvj0lDVOErvIKOOt0= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= @@ -1536,8 +1572,9 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lR sigs.k8s.io/controller-runtime v0.12.1 h1:4BJY01xe9zKQti8oRjj/NeHKRXthf1YkYJAgLONFFoI= sigs.k8s.io/controller-runtime v0.12.1/go.mod h1:BKhxlA4l7FPK4AQcsuL4X6vZeWnKDXez/vp1Y8dxTU0= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= -sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= +sigs.k8s.io/json v0.0.0-20220525155127-227cbc7cc124 h1:2sgAQQcY0dEW2SsQwTXhQV4vO6+rSslYx8K3XmM5hqQ= +sigs.k8s.io/json v0.0.0-20220525155127-227cbc7cc124/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 423a42909..030fce973 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -491,7 +491,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, resourceRequirements := kubevirtv1.ResourceRequirements{} labels := map[string]string{"kubevirt.io/vm": machine.Name} // Add a common label to all VirtualMachines spawned by the same MachineDeployment (= MachineDeployment name). - if mdName, err := controllerutil.GetMachineDeploymentNameForMachine(ctx, machine, data.Client); err == nil { + if mdName, _, err := controllerutil.GetMachineDeploymentNameAndRevisionForMachine(ctx, machine, data.Client); err == nil { labels[machineDeploymentLabelKey] = mdName } diff --git a/pkg/cloudprovider/util/cloud_init_settings.go b/pkg/cloudprovider/util/cloud_init_settings.go index fc199b92d..ed32c6e5a 100644 --- a/pkg/cloudprovider/util/cloud_init_settings.go +++ b/pkg/cloudprovider/util/cloud_init_settings.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "strings" "gopkg.in/yaml.v3" @@ -33,55 +32,27 @@ import ( ) const ( - CloudInitNamespace = "cloud-init-settings" - jwtTokenNamePrefix = "cloud-init-getter-token" + CloudInitNamespace = "cloud-init-settings" + cloudInitGetterSecret = "cloud-init-getter-token" ) -func ExtractAPIServerToken(ctx context.Context, client ctrlruntimeclient.Client) (string, error) { - secretList := corev1.SecretList{} - if err := client.List(ctx, &secretList, &ctrlruntimeclient.ListOptions{Namespace: CloudInitNamespace}); err != nil { - return "", fmt.Errorf("failed to list secrets in namespace %s: %w", CloudInitNamespace, err) +func ExtractTokenAndAPIServer(ctx context.Context, userdata string, client ctrlruntimeclient.Client) (string, string, error) { + secret := &corev1.Secret{} + if err := client.Get(ctx, types.NamespacedName{Name: cloudInitGetterSecret, Namespace: CloudInitNamespace}, secret); err != nil { + return "", "", fmt.Errorf("failed to get %s secrets in namespace %s: %w", cloudInitGetterSecret, CloudInitNamespace, err) } - for _, secret := range secretList.Items { - if strings.HasPrefix(secret.Name, jwtTokenNamePrefix) { - if secret.Data != nil { - jwtToken := secret.Data["token"] - if jwtToken != nil { - token := string(jwtToken) - return token, nil - } - } - } + token := secret.Data["token"] + if token == nil { + return "", "", errors.New("failed to extract token from cloud-init secret") } - return "", errors.New("failed to fetch api server token") -} - -func ExtractTokenAndAPIServer(ctx context.Context, userdata string, client ctrlruntimeclient.Client) (token string, apiServer string, err error) { - secretList := corev1.SecretList{} - if err := client.List(ctx, &secretList, &ctrlruntimeclient.ListOptions{Namespace: CloudInitNamespace}); err != nil { - return "", "", fmt.Errorf("failed to list secrets in namespace %s: %w", CloudInitNamespace, err) - } - - apiServer, err = extractAPIServer(userdata) + apiServer, err := extractAPIServer(userdata) if err != nil { return "", "", fmt.Errorf("failed to extract api server address: %w", err) } - for _, secret := range secretList.Items { - if strings.HasPrefix(secret.Name, jwtTokenNamePrefix) { - if secret.Data != nil { - jwtToken := secret.Data["token"] - if jwtToken != nil { - token = string(jwtToken) - return token, apiServer, nil - } - } - } - } - - return "", "", errors.New("failed to find cloud-init secret") + return string(token), apiServer, nil } func CreateMachineCloudInitSecret(ctx context.Context, userdata, machineName string, client ctrlruntimeclient.Client) error { diff --git a/pkg/cloudprovider/util/cloud_init_settings_test.go b/pkg/cloudprovider/util/cloud_init_settings_test.go index a4bf41131..8a72079f2 100644 --- a/pkg/cloudprovider/util/cloud_init_settings_test.go +++ b/pkg/cloudprovider/util/cloud_init_settings_test.go @@ -39,7 +39,7 @@ var testData = []struct { userdata: "./testdata/userdata.yaml", secret: &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: jwtTokenNamePrefix, + Name: cloudInitGetterSecret, Namespace: CloudInitNamespace, }, Data: map[string][]byte{ diff --git a/pkg/controller/machine/bootstrap.go b/pkg/controller/machine/bootstrap.go index 6e86b8fe3..e8e81d0bb 100644 --- a/pkg/controller/machine/bootstrap.go +++ b/pkg/controller/machine/bootstrap.go @@ -17,388 +17,29 @@ limitations under the License. package controller import ( - "bytes" - "context" - "encoding/base64" - "fmt" "regexp" - "text/template" + "strings" - "github.com/Masterminds/sprig/v3" - - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - "github.com/kubermatic/machine-controller/pkg/userdata/convert" - "github.com/kubermatic/machine-controller/pkg/userdata/helper" - "github.com/kubermatic/machine-controller/pkg/userdata/rhel" - - ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + corev1 "k8s.io/api/core/v1" ) -func getOSMBootstrapUserdata(ctx context.Context, client ctrlruntimeclient.Client, req plugin.UserDataRequest, secretName string) (string, error) { - var clusterName string - for key := range req.Kubeconfig.Clusters { - clusterName = key - } - - token, err := util.ExtractAPIServerToken(ctx, client) - if err != nil { - return "", fmt.Errorf("failed to fetch api-server token: %w", err) - } +const hostnamePlaceholder = "" - // Retrieve provider config from machine - pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) - if err != nil { - return "", fmt.Errorf("failed to get providerSpec: %w", err) - } +func getOSMBootstrapUserdata(machineName string, bootstrapSecret corev1.Secret) string { + bootstrapConfig := string(bootstrapSecret.Data["cloud-config"]) - bootstrapKubeconfig, err := helper.StringifyKubeconfig(req.Kubeconfig) - if err != nil { - return "", fmt.Errorf("failed to format bootstrap kubeconfig: %w", err) - } + // We have to inject the hostname i.e. machine name. + bootstrapConfig = strings.ReplaceAll(bootstrapConfig, hostnamePlaceholder, machineName) - // Regardless if the provisioningUtility is set to use cloud-init, we only allow using ignition to provision flatcar - // machines with osm. - if pconfig.OperatingSystem == providerconfigtypes.OperatingSystemFlatcar { - return getOSMBootstrapUserDataForIgnition(req, pconfig.SSHPublicKeys, token, secretName, clusterName, bootstrapKubeconfig) - } - - // cloud-init is used for all other operating systems. - return getOSMBootstrapUserDataForCloudInit(req, pconfig, token, secretName, clusterName, bootstrapKubeconfig) -} - -// getOSMBootstrapUserDataForIgnition returns the userdata for the ignition bootstrap config. -func getOSMBootstrapUserDataForIgnition(req plugin.UserDataRequest, sshPublicKeys []string, token, secretName, clusterName, bootstrapKfg string) (string, error) { - data := struct { - Token string - SecretName string - ServerURL string - }{ - Token: token, - SecretName: secretName, - ServerURL: req.Kubeconfig.Clusters[clusterName].Server, - } - bsScript, err := template.New("bootstrap-script").Parse(ignitionBootstrapBinContentTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse bootstrapBinContentTemplate template for ignition: %w", err) - } - script := &bytes.Buffer{} - err = bsScript.Execute(script, data) - if err != nil { - return "", fmt.Errorf("failed to execute bootstrapBinContentTemplate template for ignition: %w", err) - } - bsIgnitionConfig, err := template.New("bootstrap-ignition-config").Funcs(sprig.TxtFuncMap()).Parse(ignitionTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse bootstrap-ignition-config template: %w", err) - } - - ignitionConfig := &bytes.Buffer{} - err = bsIgnitionConfig.Execute(ignitionConfig, struct { - plugin.UserDataRequest - Script string - Service string - SSHPublicKeys []string - BootstrapKubeconfig string - }{ - UserDataRequest: req, - Script: script.String(), - Service: bootstrapServiceContentTemplate, - SSHPublicKeys: sshPublicKeys, - BootstrapKubeconfig: bootstrapKfg, - }) - if err != nil { - return "", fmt.Errorf("failed to execute ignitionTemplate template: %w", err) - } - - return convert.ToIgnition(ignitionConfig.String()) -} - -// getOSMBootstrapUserDataForCloudInit returns the userdata for the cloud-init bootstrap script. -func getOSMBootstrapUserDataForCloudInit(req plugin.UserDataRequest, pconfig *providerconfigtypes.Config, token, secretName, clusterName, bootstrapKfg string) (string, error) { - data := struct { - Token string - SecretName string - ServerURL string - MachineName string - EnterpriseLinux bool - ProviderSpec *providerconfigtypes.Config - RHELConfig rhel.Config - }{ - Token: token, - SecretName: secretName, - ServerURL: req.Kubeconfig.Clusters[clusterName].Server, - MachineName: req.MachineSpec.Name, - ProviderSpec: pconfig, - } - - var ( - rhelConfig *rhel.Config - bsScript *template.Template - err error - ) - - switch pconfig.OperatingSystem { - case providerconfigtypes.OperatingSystemUbuntu: - bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapAptBinContentTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse bootstrapAptBinContentTemplate template: %w", err) - } - case providerconfigtypes.OperatingSystemCentOS: - data.EnterpriseLinux = true - bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapYumBinContentTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse bootstrapYumBinContentTemplate template: %w", err) - } - case providerconfigtypes.OperatingSystemAmazonLinux2: - bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapYumBinContentTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse bootstrapYumBinContentTemplate template: %w", err) - } - case providerconfigtypes.OperatingSystemSLES: - bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapZypperBinContentTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse bootstrapZypperBinContentTemplate template: %w", err) - } - case providerconfigtypes.OperatingSystemRHEL: - rhelConfig, err = rhel.LoadConfig(pconfig.OperatingSystemSpec) - if err != nil { - return "", fmt.Errorf("failed to parse OperatingSystemSpec: %w", err) - } - bsScript, err = template.New("bootstrap-cloud-init").Parse(bootstrapYumBinContentTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse bootstrapYumBinContentTemplate template: %w", err) - } - } - - script := &bytes.Buffer{} - err = bsScript.Execute(script, data) - if err != nil { - return "", fmt.Errorf("failed to execute bootstrap script template: %w", err) - } - bsCloudInit, err := template.New("bootstrap-cloud-init").Parse(cloudInitTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse download-binaries template: %w", err) - } - - cloudInit := &bytes.Buffer{} - err = bsCloudInit.Execute(cloudInit, struct { - Script string - Service string - plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - BootstrapKubeconfig string - RHELConfig *rhel.Config - }{ - Script: base64.StdEncoding.EncodeToString(script.Bytes()), - Service: base64.StdEncoding.EncodeToString([]byte(bootstrapServiceContentTemplate)), - UserDataRequest: req, - ProviderSpec: pconfig, - BootstrapKubeconfig: base64.StdEncoding.EncodeToString([]byte(bootstrapKfg)), - RHELConfig: rhelConfig, - }) - if err != nil { - return "", fmt.Errorf("failed to execute cloudInitTemplate template: %w", err) - } - return cloudInit.String(), nil + return cleanupTemplateOutput(bootstrapConfig) } // cleanupTemplateOutput postprocesses the output of the template processing. Those // may exist due to the working of template functions like those of the sprig package // or template condition. -func cleanupTemplateOutput(output string) (string, error) { +func cleanupTemplateOutput(output string) string { // Valid YAML files are not allowed to have empty lines containing spaces or tabs. // So far only cleanup. woBlankLines := regexp.MustCompile(`(?m)^[ \t]+$`).ReplaceAllString(output, "") - return woBlankLines, nil + return woBlankLines } - -const ( - bootstrapAptBinContentTemplate = `#!/bin/bash -set -xeuo pipefail - -export DEBIAN_FRONTEND=noninteractive -apt update && apt install -y curl jq -curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-config"]' -r| base64 -d > /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg -cloud-init clean -cloud-init --file /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg init -systemctl daemon-reload - -{{- /* The default cloud-init configurations files have some bug on Digital Ocean which causes the machine to be in-accessible on 2nd cloud-init. We have to manually run the module */}} -{{- if and (eq .ProviderSpec.CloudProvider "digitalocean") (eq .ProviderSpec.OperatingSystem "ubuntu") }} -rm /etc/netplan/50-cloud-init.yaml -netplan generate -netplan apply -{{- end }} - -systemctl restart setup.service -systemctl restart kubelet.service -systemctl restart kubelet-healthcheck.service - ` - - bootstrapYumBinContentTemplate = `#!/bin/bash -set -xeuo pipefail -source /etc/os-release -if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* -fi -{{- if .EnterpriseLinux }} -yum install epel-release -y -{{- end }} - -yum install -y curl jq - -curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-config"]' -r| base64 -d > /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg -cloud-init clean -cloud-init --file /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg init -systemctl daemon-reload -systemctl restart setup.service -systemctl restart kubelet.service -systemctl restart kubelet-healthcheck.service - ` - - bootstrapZypperBinContentTemplate = `#!/bin/bash -set -xeuo pipefail - -# Install JQ -zypper -n --quiet addrepo -C https://download.opensuse.org/repositories/utilities/openSUSE_Leap_15.3/utilities.repo -zypper -n --no-gpg-checks refresh -zypper -n install jq - -# Install CURL -zypper -n install curl - -curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-config"]' -r| base64 -d > /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg -cloud-init clean -cloud-init --file /etc/cloud/cloud.cfg.d/{{ .SecretName }}.cfg init -systemctl daemon-reload -systemctl restart setup.service -systemctl restart kubelet.service -systemctl restart kubelet-healthcheck.service - ` - - bootstrapServiceContentTemplate = `[Install] -WantedBy=multi-user.target - -[Unit] -Requires=network-online.target -After=network-online.target -[Service] -Type=oneshot -RemainAfterExit=true -ExecStart=/opt/bin/bootstrap - ` - - cloudInitTemplate = `#cloud-config -{{ if ne .CloudProviderName "aws" }} -hostname: {{ .MachineSpec.Name }} -{{- /* Never set the hostname on AWS nodes. Kubernetes(kube-proxy) requires the hostname to be the private dns name */}} -{{ end }} -ssh_pwauth: false - -{{- if .ProviderSpec.SSHPublicKeys }} -ssh_authorized_keys: -{{- range .ProviderSpec.SSHPublicKeys }} -- "{{ . }}" -{{- end }} -{{- end }} - -write_files: -- path: /opt/bin/bootstrap - permissions: '0755' - encoding: b64 - content: | - {{ .Script }} -- path: /etc/kubernetes/bootstrap-kubelet.conf - permissions: '0600' - encoding: b64 - content: | - {{ .BootstrapKubeconfig }} -{{- if and (eq .ProviderSpec.CloudProvider "openstack") (or (eq .ProviderSpec.OperatingSystem "centos") (eq .ProviderSpec.OperatingSystem "rhel")) }} -{{- /* The normal way of setting it via cloud-init is broken, see */}} -{{- /* https://bugs.launchpad.net/cloud-init/+bug/1662542 */}} -- path: /etc/hostname - permissions: '0600' - content: | - {{ .MachineSpec.Name }} -{{ end }} -- path: /etc/systemd/system/bootstrap.service - permissions: '0644' - encoding: b64 - content: | - {{ .Service }} -{{- /* The default cloud-init configurations files have some bug on Digital Ocean which causes the machine to be in-accessible on 2nd cloud-init. Hence we disable network configuration */}} -{{- if and (eq .ProviderSpec.CloudProvider "digitalocean") (eq .ProviderSpec.OperatingSystem "ubuntu") }} -- path: /etc/cloud/cloud.cfg.d/99-custom-networking.cfg - permissions: '0644' - content: | - network: {config: disabled} -{{- end }} -runcmd: -- systemctl restart bootstrap.service -- systemctl daemon-reload -{{- if .RHELConfig }} -rh_subscription: -{{- if .RHELConfig.RHELUseSatelliteServer }} - org: "{{.RHELConfig.RHELOrganizationName}}" - activation-key: "{{.RHELConfig.RHELActivationKey}}" - server-hostname: {{ .RHELConfig.RHELSatelliteServer }} - rhsm-baseurl: https://{{ .RHELConfig.RHELSatelliteServer }}/pulp/repos -{{- else }} - username: "{{.RHELConfig.RHELSubscriptionManagerUser}}" - password: "{{.RHELConfig.RHELSubscriptionManagerPassword}}" - auto-attach: {{.RHELConfig.AttachSubscription}} -{{- end }} -{{- end }} -` - - ignitionBootstrapBinContentTemplate = `#!/bin/bash -set -xeuo pipefail -apt update && apt install -y curl jq -curl -s -k -v --header 'Authorization: Bearer {{ .Token }}' {{ .ServerURL }}/api/v1/namespaces/cloud-init-settings/secrets/{{ .SecretName }} | jq '.data["cloud-config"]' -r| base64 -d > /usr/share/oem/config.ign -touch /boot/flatcar/first_boot -systemctl disable bootstrap.service -rm /etc/systemd/system/bootstrap.service -rm /etc/machine-id -reboot -` - - ignitionTemplate = `passwd: -{{- if ne (len .SSHPublicKeys) 0 }} - users: - - name: core - ssh_authorized_keys: - {{range .SSHPublicKeys }}- {{.}} - {{end}} -{{- end }} -storage: - files: - - path: /etc/kubernetes/bootstrap-kubelet.conf - mode: 0600 - filesystem: root - contents: - inline: | -{{ .BootstrapKubeconfig | indent 10 }} - - path: /opt/bin/bootstrap - mode: 0755 - filesystem: root - contents: - inline: | -{{ .Script | indent 10}} -{{ if ne .CloudProviderName "aws" }} -{{- /* Never set the hostname on AWS nodes. Kubernetes(kube-proxy) requires the hostname to be the private dns name */}} - - path: /etc/hostname - mode: 0600 - filesystem: root - contents: - inline: '{{ .MachineSpec.Name }}' -{{ end }} -systemd: - units: - - name: bootstrap.service - enabled: true - contents: | -{{ .Service | indent 10 }} -` -) diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index df7fd73a4..39f4065e4 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -47,6 +47,8 @@ import ( userdatamanager "github.com/kubermatic/machine-controller/pkg/userdata/manager" userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" "github.com/kubermatic/machine-controller/pkg/userdata/rhel" + "k8c.io/operating-system-manager/pkg/controllers/osc" + osmresources "k8c.io/operating-system-manager/pkg/controllers/osc/resources" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -95,7 +97,7 @@ const ( // cluster-api provider to match Nodes to Machines. AnnotationAutoscalerIdentifier = "cluster.k8s.io/machine" - provisioningSuffix = "osc-provisioning" + CloudInitNotReadyError = "cloud-init configuration to %s machine: %v is not ready yet" ) // Reconciler is the controller implementation for machine resources. @@ -744,9 +746,14 @@ func (r *Reconciler) ensureInstanceExistsForMachine( if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { klog.V(3).Infof("Validated machine spec of %s", machine.Name) - kubeconfig, err := r.createBootstrapKubeconfig(ctx, machine.Name) - if err != nil { - return nil, fmt.Errorf("failed to create bootstrap kubeconfig: %w", err) + var kubeconfig *clientcmdapi.Config + + // OSM will take care of the bootstrap kubeconfig and token by itself. + if !r.useOSM { + kubeconfig, err = r.createBootstrapKubeconfig(ctx, machine.Name) + if err != nil { + return nil, fmt.Errorf("failed to create bootstrap kubeconfig: %w", err) + } } cloudConfig, kubeletCloudProviderName, err := prov.GetCloudConfig(machine.Spec) @@ -808,33 +815,51 @@ func (r *Reconciler) ensureInstanceExistsForMachine( var userdata string if r.useOSM { - referencedMachineDeployment, err := controllerutil.GetMachineDeploymentNameForMachine(ctx, machine, r.client) + referencedMachineDeployment, machineDeploymentRevision, err := controllerutil.GetMachineDeploymentNameAndRevisionForMachine(ctx, machine, r.client) if err != nil { return nil, fmt.Errorf("failed to find machine's MachineDployment: %w", err) } - cloudConfigSecretName := fmt.Sprintf("%s-%s-%s", + // We need to ensure that both provisoning and bootstrapping secrets have been created. And that the revision + // matches with the machine deployment revision + provisioningSecretName := fmt.Sprintf(osmresources.CloudConfigSecretNamePattern, referencedMachineDeployment, machine.Namespace, - provisioningSuffix) + osmresources.ProvisioningCloudConfig) - // It is important to check if the secret holding cloud-config exists + // Ensure that the provisioning secret exists + provisioningSecret := &corev1.Secret{} if err := r.client.Get(ctx, - types.NamespacedName{Name: cloudConfigSecretName, Namespace: util.CloudInitNamespace}, - &corev1.Secret{}); err != nil { - klog.Errorf("Cloud init configurations for machine: %v is not ready yet", machine.Name) + types.NamespacedName{Name: provisioningSecretName, Namespace: util.CloudInitNamespace}, + provisioningSecret); err != nil { + klog.Errorf(CloudInitNotReadyError, osmresources.ProvisioningCloudConfig, machine.Name) return nil, err } - userdata, err = getOSMBootstrapUserdata(ctx, r.client, req, cloudConfigSecretName) - if err != nil { - return nil, fmt.Errorf("failed get OSM userdata: %w", err) + provisioningSecretRevision := provisioningSecret.Annotations[osc.MachineDeploymentRevision] + if provisioningSecretRevision != machineDeploymentRevision { + return nil, fmt.Errorf(CloudInitNotReadyError, osmresources.ProvisioningCloudConfig, machine.Name) } - userdata, err = cleanupTemplateOutput(userdata) - if err != nil { - return nil, fmt.Errorf("failed to cleanup user-data template: %w", err) + bootstrapSecretName := fmt.Sprintf(osmresources.CloudConfigSecretNamePattern, + referencedMachineDeployment, + machine.Namespace, + osmresources.BootstrapCloudConfig) + + bootstrapSecret := &corev1.Secret{} + if err := r.client.Get(ctx, + types.NamespacedName{Name: bootstrapSecretName, Namespace: util.CloudInitNamespace}, + bootstrapSecret); err != nil { + klog.Errorf(CloudInitNotReadyError, osmresources.BootstrapCloudConfig, machine.Name) + return nil, err } + + bootstrapSecretRevision := bootstrapSecret.Annotations[osc.MachineDeploymentRevision] + if bootstrapSecretRevision != machineDeploymentRevision { + return nil, fmt.Errorf(CloudInitNotReadyError, osmresources.BootstrapCloudConfig, machine.Name) + } + + userdata = getOSMBootstrapUserdata(req.MachineSpec.Name, *bootstrapSecret) } else { userdata, err = userdataPlugin.UserData(req) if err != nil { diff --git a/pkg/controller/util/machine.go b/pkg/controller/util/machine.go index 85a6c5c21..a8e9b090a 100644 --- a/pkg/controller/util/machine.go +++ b/pkg/controller/util/machine.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func GetMachineDeploymentNameForMachine(ctx context.Context, machine *clusterv1alpha1.Machine, c client.Client) (string, error) { +func GetMachineDeploymentNameAndRevisionForMachine(ctx context.Context, machine *clusterv1alpha1.Machine, c client.Client) (string, string, error) { var ( machineSetName string machineDeploymentName string @@ -40,7 +40,7 @@ func GetMachineDeploymentNameForMachine(ctx context.Context, machine *clusterv1a if machineSetName != "" { machineSet := &clusterv1alpha1.MachineSet{} if err := c.Get(ctx, types.NamespacedName{Name: machineSetName, Namespace: "kube-system"}, machineSet); err != nil { - return "", err + return "", "", err } for _, ownerRef := range machineSet.OwnerReferences { @@ -49,10 +49,11 @@ func GetMachineDeploymentNameForMachine(ctx context.Context, machine *clusterv1a } } + revision := machineSet.Annotations[RevisionAnnotation] if machineDeploymentName != "" { - return machineDeploymentName, nil + return machineDeploymentName, revision, nil } } - return "", fmt.Errorf("failed to find machine deployment reference for the machine %s", machine.Name) + return "", "", fmt.Errorf("failed to find machine deployment reference for the machine %s", machine.Name) } diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index 5b15b3b03..f79ac612c 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -66,11 +66,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { return "", fmt.Errorf("failed to parse OperatingSystemSpec: %w", err) } - serverAddr, err := userdatahelper.GetServerAddressFromKubeconfig(req.Kubeconfig) - if err != nil { - return "", fmt.Errorf("error extracting server address from kubeconfig: %w", err) - } - kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) if err != nil { return "", err @@ -102,7 +97,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ProviderSpec *providerconfigtypes.Config OSConfig *Config KubeletVersion string - ServerAddr string Kubeconfig string KubernetesCACert string NodeIPScript string @@ -118,7 +112,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ProviderSpec: pconfig, OSConfig: amznConfig, KubeletVersion: kubeletVersion.String(), - ServerAddr: serverAddr, Kubeconfig: kubeconfigString, KubernetesCACert: kubernetesCACert, NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index a2aaae6a3..5a96b70de 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -66,11 +66,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { return "", fmt.Errorf("failed to parse OperatingSystemSpec: %w", err) } - serverAddr, err := userdatahelper.GetServerAddressFromKubeconfig(req.Kubeconfig) - if err != nil { - return "", fmt.Errorf("error extracting server address from kubeconfig: %w", err) - } - kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) if err != nil { return "", err @@ -102,7 +97,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ProviderSpec *providerconfigtypes.Config OSConfig *Config KubeletVersion string - ServerAddr string Kubeconfig string KubernetesCACert string NodeIPScript string @@ -118,7 +112,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ProviderSpec: pconfig, OSConfig: centosConfig, KubeletVersion: kubeletVersion.String(), - ServerAddr: serverAddr, Kubeconfig: kubeconfigString, KubernetesCACert: kubernetesCACert, NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), diff --git a/pkg/userdata/helper/helper.go b/pkg/userdata/helper/helper.go index 9ee7ab926..291513e20 100644 --- a/pkg/userdata/helper/helper.go +++ b/pkg/userdata/helper/helper.go @@ -30,17 +30,6 @@ const ( DefaultDockerContainerLogMaxSize = "100m" ) -func GetServerAddressFromKubeconfig(kubeconfig *clientcmdapi.Config) (string, error) { - if len(kubeconfig.Clusters) != 1 { - return "", fmt.Errorf("kubeconfig does not contain exactly one cluster, can not extract server address") - } - // Clusters is a map so we have to use range here - for _, clusterConfig := range kubeconfig.Clusters { - return strings.Replace(clusterConfig.Server, "https://", "", -1), nil - } - return "", fmt.Errorf("no server address found") -} - func GetCACert(kubeconfig *clientcmdapi.Config) (string, error) { if len(kubeconfig.Clusters) != 1 { return "", fmt.Errorf("kubeconfig does not contain exactly one cluster, can not extract server address") diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index b898e3094..75efa5d45 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -66,11 +66,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { return "", fmt.Errorf("failed to parse OperatingSystemSpec: %w", err) } - serverAddr, err := userdatahelper.GetServerAddressFromKubeconfig(req.Kubeconfig) - if err != nil { - return "", fmt.Errorf("error extracting server address from kubeconfig: %w", err) - } - kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) if err != nil { return "", err @@ -102,7 +97,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ProviderSpec *providerconfigtypes.Config OSConfig *Config KubeletVersion string - ServerAddr string Kubeconfig string KubernetesCACert string NodeIPScript string @@ -118,7 +112,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ProviderSpec: pconfig, OSConfig: rhelConfig, KubeletVersion: kubeletVersion.String(), - ServerAddr: serverAddr, Kubeconfig: kubeconfigString, KubernetesCACert: kubernetesCACert, NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), diff --git a/pkg/userdata/rockylinux/provider.go b/pkg/userdata/rockylinux/provider.go index c2a130600..450bde849 100644 --- a/pkg/userdata/rockylinux/provider.go +++ b/pkg/userdata/rockylinux/provider.go @@ -66,11 +66,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { return "", fmt.Errorf("failed to parse OperatingSystemSpec: %w", err) } - serverAddr, err := userdatahelper.GetServerAddressFromKubeconfig(req.Kubeconfig) - if err != nil { - return "", fmt.Errorf("error extracting server address from kubeconfig: %w", err) - } - kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) if err != nil { return "", err @@ -102,7 +97,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ProviderSpec *providerconfigtypes.Config OSConfig *Config KubeletVersion string - ServerAddr string Kubeconfig string KubernetesCACert string NodeIPScript string @@ -118,7 +112,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { ProviderSpec: pconfig, OSConfig: rockyLinuxConfig, KubeletVersion: kubeletVersion.String(), - ServerAddr: serverAddr, Kubeconfig: kubeconfigString, KubernetesCACert: kubernetesCACert, NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), diff --git a/pkg/userdata/sles/provider.go b/pkg/userdata/sles/provider.go index 50b14f888..7a7130d80 100644 --- a/pkg/userdata/sles/provider.go +++ b/pkg/userdata/sles/provider.go @@ -66,11 +66,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { return "", fmt.Errorf("failed to get sles config from provider config: %w", err) } - serverAddr, err := userdatahelper.GetServerAddressFromKubeconfig(req.Kubeconfig) - if err != nil { - return "", fmt.Errorf("error extracting server address from kubeconfig: %w", err) - } - kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) if err != nil { return "", err @@ -96,7 +91,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { plugin.UserDataRequest ProviderSpec *providerconfigtypes.Config OSConfig *Config - ServerAddr string KubeletVersion string Kubeconfig string KubernetesCACert string @@ -111,7 +105,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { UserDataRequest: req, ProviderSpec: pconfig, OSConfig: slesConfig, - ServerAddr: serverAddr, KubeletVersion: kubeletVersion.String(), Kubeconfig: kubeconfigString, KubernetesCACert: kubernetesCACert, diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index 034f4d88d..4fb0920dd 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -66,11 +66,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { return "", fmt.Errorf("failed to get ubuntu config from provider config: %w", err) } - serverAddr, err := userdatahelper.GetServerAddressFromKubeconfig(req.Kubeconfig) - if err != nil { - return "", fmt.Errorf("error extracting server address from kubeconfig: %w", err) - } - kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) if err != nil { return "", err @@ -101,7 +96,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { plugin.UserDataRequest ProviderSpec *providerconfigtypes.Config OSConfig *Config - ServerAddr string KubeletVersion string Kubeconfig string KubernetesCACert string @@ -117,7 +111,6 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { UserDataRequest: req, ProviderSpec: pconfig, OSConfig: ubuntuConfig, - ServerAddr: serverAddr, KubeletVersion: kubeletVersion.String(), Kubeconfig: kubeconfigString, KubernetesCACert: kubernetesCACert, From 6cb8c2cdf7e09b2e91f69eb2e00a8c0945a3019b Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 30 Jun 2022 17:25:05 +0500 Subject: [PATCH 176/489] Bump docker version to 20.10 (#1337) --- pkg/containerruntime/docker.go | 4 ++-- pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml | 2 +- .../amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml | 4 ++-- pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml | 4 ++-- .../centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml | 4 ++-- pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml | 4 ++-- pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml | 4 ++-- pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml | 4 ++-- pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml | 4 ++-- pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml | 4 ++-- pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml | 4 ++-- pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml | 4 ++-- pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml | 4 ++-- pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml | 4 ++-- pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml | 4 ++-- pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml | 4 ++-- pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml | 4 ++-- pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml | 4 ++-- pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml | 4 ++-- .../rockylinux/testdata/kubelet-v1.21-aws-external.yaml | 4 ++-- pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml | 4 ++-- .../rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml | 4 ++-- .../rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml | 4 ++-- pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml | 4 ++-- pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml | 4 ++-- pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml | 4 ++-- pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml | 4 ++-- pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml | 4 ++-- pkg/userdata/ubuntu/testdata/docker.yaml | 4 ++-- .../ubuntu/testdata/kubelet-version-without-v-prefix.yaml | 4 ++-- pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml | 4 ++-- pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml | 4 ++-- pkg/userdata/ubuntu/testdata/nutanix.yaml | 4 ++-- .../ubuntu/testdata/openstack-overwrite-cloud-config.yaml | 4 ++-- pkg/userdata/ubuntu/testdata/openstack.yaml | 4 ++-- pkg/userdata/ubuntu/testdata/version-1.21.10.yaml | 4 ++-- pkg/userdata/ubuntu/testdata/version-1.22.7.yaml | 4 ++-- pkg/userdata/ubuntu/testdata/version-1.23.5.yaml | 4 ++-- pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml | 4 ++-- pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml | 4 ++-- pkg/userdata/ubuntu/testdata/vsphere.yaml | 4 ++-- 47 files changed, 87 insertions(+), 87 deletions(-) diff --git a/pkg/containerruntime/docker.go b/pkg/containerruntime/docker.go index dbbc1e58d..e79920d7b 100644 --- a/pkg/containerruntime/docker.go +++ b/pkg/containerruntime/docker.go @@ -28,8 +28,8 @@ import ( const ( DefaultDockerContainerdVersion = "1.4" - DefaultDockerVersion = "19.03" - LegacyDockerVersion = "18.09" + DefaultDockerVersion = "20.10" + LegacyDockerVersion = "19.03" ) type Docker struct { diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml index 38cf73097..d08b2f73a 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml @@ -90,7 +90,7 @@ write_files: yum install -y \ containerd-1.4* \ - docker-19.03* \ + docker-20.10* \ yum-plugin-versionlock yum versionlock add docker containerd diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml index 79710ae80..7a8b853b9 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml @@ -90,7 +90,7 @@ write_files: yum install -y \ containerd-1.4* \ - docker-19.03* \ + docker-20.10* \ yum-plugin-versionlock yum versionlock add docker containerd diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 3ecb5eba1..e2fcbf6f3 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -103,7 +103,7 @@ write_files: yum install -y \ containerd-1.4* \ - docker-19.03* \ + docker-20.10* \ yum-plugin-versionlock yum versionlock add docker containerd diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml index 563d9a827..5c96b174e 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -103,7 +103,7 @@ write_files: yum install -y \ containerd-1.4* \ - docker-19.03* \ + docker-20.10* \ yum-plugin-versionlock yum versionlock add docker containerd diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml index 48dbf051b..4c4c07bb7 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml @@ -95,7 +95,7 @@ write_files: yum install -y \ containerd-1.4* \ - docker-19.03* \ + docker-20.10* \ yum-plugin-versionlock yum versionlock add docker containerd diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml index b8845879e..42864bd2e 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml @@ -90,7 +90,7 @@ write_files: yum install -y \ containerd-1.4* \ - docker-19.03* \ + docker-20.10* \ yum-plugin-versionlock yum versionlock add docker containerd diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml index 96ac2eeda..4733cab46 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml @@ -90,7 +90,7 @@ write_files: yum install -y \ containerd-1.4* \ - docker-19.03* \ + docker-20.10* \ yum-plugin-versionlock yum versionlock add docker containerd diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml index f1b92e886..fa03994b4 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml @@ -98,9 +98,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml index 442917bd7..4f7bee9e1 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml @@ -98,9 +98,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 10d664f39..3c7b56718 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -111,9 +111,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml index 4ef234434..7dd0ddc5a 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -111,9 +111,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml index be2791b17..b325de29d 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml @@ -103,9 +103,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index de96a9328..92555b815 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -98,9 +98,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index e912b1131..bcb8c3ade 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -98,9 +98,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml index 50a5ee4a0..ad28ad691 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml @@ -105,9 +105,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml index 9568a5550..ccaf019ea 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml @@ -93,9 +93,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index 81fb687e8..dbdb97a14 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -93,9 +93,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml index da5d9155c..82ee10299 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml @@ -101,9 +101,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index c46206267..98a5f9b61 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -93,9 +93,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index 50fd4811f..deeef0ad8 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -93,9 +93,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 467e63141..749268ce1 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -107,9 +107,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index aa9bc6081..1ec73c6a7 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -107,9 +107,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index a193c46f3..49a601c3d 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -99,9 +99,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index 055f69919..0a661c056 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -98,9 +98,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml index ffe97a9d4..379993a03 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml @@ -94,9 +94,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml index 048093fd3..733f76cd2 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml @@ -94,9 +94,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml index 66de513af..a9f0f2121 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml @@ -107,9 +107,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml index 43b97a65e..36f5e4acf 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml @@ -107,9 +107,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml index 34da2675a..2a373b371 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml @@ -99,9 +99,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml index 85cc06d59..2e8b1aba4 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml @@ -94,9 +94,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml index 6c8dea179..587dc6d17 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml @@ -94,9 +94,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml index d8b9a49a3..bf9f1047e 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml @@ -101,9 +101,9 @@ write_files: EOF yum install -y \ - docker-ce-cli-19.03* \ + docker-ce-cli-20.10* \ containerd.io-1.4* \ - docker-ce-19.03* \ + docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index a84ab4bc3..72876104d 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -102,8 +102,8 @@ write_files: apt-get install --allow-downgrades -y \ containerd.io=1.4* \ - docker-ce-cli=5:19.03* \ - docker-ce=5:19.03* + docker-ce-cli=5:20.10* \ + docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml index a8193e4d4..0f2ee1524 100644 --- a/pkg/userdata/ubuntu/testdata/docker.yaml +++ b/pkg/userdata/ubuntu/testdata/docker.yaml @@ -102,8 +102,8 @@ write_files: apt-get install --allow-downgrades -y \ containerd.io=1.4* \ - docker-ce-cli=5:19.03* \ - docker-ce=5:19.03* + docker-ce-cli=5:20.10* \ + docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index b5460cb68..77428740e 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -100,8 +100,8 @@ write_files: apt-get install --allow-downgrades -y \ containerd.io=1.4* \ - docker-ce-cli=5:19.03* \ - docker-ce=5:19.03* + docker-ce-cli=5:20.10* \ + docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 8592ce601..f0d5f55f0 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -100,8 +100,8 @@ write_files: apt-get install --allow-downgrades -y \ containerd.io=1.4* \ - docker-ce-cli=5:19.03* \ - docker-ce=5:19.03* + docker-ce-cli=5:20.10* \ + docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index fb5098971..dc18f0a19 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -102,8 +102,8 @@ write_files: apt-get install --allow-downgrades -y \ containerd.io=1.4* \ - docker-ce-cli=5:19.03* \ - docker-ce=5:19.03* + docker-ce-cli=5:20.10* \ + docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 261f731bb..f49afbc6d 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -103,8 +103,8 @@ write_files: apt-get install --allow-downgrades -y \ containerd.io=1.4* \ - docker-ce-cli=5:19.03* \ - docker-ce=5:19.03* + docker-ce-cli=5:20.10* \ + docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index a263c4a23..3866be991 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -100,8 +100,8 @@ write_files: apt-get install --allow-downgrades -y \ containerd.io=1.4* \ - docker-ce-cli=5:19.03* \ - docker-ce=5:19.03* + docker-ce-cli=5:20.10* \ + docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 8492a8e59..6725d1b9b 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -100,8 +100,8 @@ write_files: apt-get install --allow-downgrades -y \ containerd.io=1.4* \ - docker-ce-cli=5:19.03* \ - docker-ce=5:19.03* + docker-ce-cli=5:20.10* \ + docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml b/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml index f4d99bd0b..a40f49324 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml @@ -100,8 +100,8 @@ write_files: apt-get install --allow-downgrades -y \ containerd.io=1.4* \ - docker-ce-cli=5:19.03* \ - docker-ce=5:19.03* + docker-ce-cli=5:20.10* \ + docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml b/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml index b5460cb68..77428740e 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml @@ -100,8 +100,8 @@ write_files: apt-get install --allow-downgrades -y \ containerd.io=1.4* \ - docker-ce-cli=5:19.03* \ - docker-ce=5:19.03* + docker-ce-cli=5:20.10* \ + docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml index 934498758..d05672a00 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml @@ -100,8 +100,8 @@ write_files: apt-get install --allow-downgrades -y \ containerd.io=1.4* \ - docker-ce-cli=5:19.03* \ - docker-ce=5:19.03* + docker-ce-cli=5:20.10* \ + docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index c2f9843a1..be7f75f56 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -110,8 +110,8 @@ write_files: apt-get install --allow-downgrades -y \ containerd.io=1.4* \ - docker-ce-cli=5:19.03* \ - docker-ce=5:19.03* + docker-ce-cli=5:20.10* \ + docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index c89efe60a..7dfa8d510 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -110,8 +110,8 @@ write_files: apt-get install --allow-downgrades -y \ containerd.io=1.4* \ - docker-ce-cli=5:19.03* \ - docker-ce=5:19.03* + docker-ce-cli=5:20.10* \ + docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 6c49a76d4..f73e59b71 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -101,8 +101,8 @@ write_files: apt-get install --allow-downgrades -y \ containerd.io=1.4* \ - docker-ce-cli=5:19.03* \ - docker-ce=5:19.03* + docker-ce-cli=5:20.10* \ + docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io systemctl daemon-reload From 6250a9cd04a45d1b44c57492f17d3fe997ad8c01 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 1 Jul 2022 19:08:08 +0500 Subject: [PATCH 177/489] Upgrade to alpine 3.16 for machine-controller's image (#1342) Signed-off-by: Waleed Malik --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e21dccc34..1a835fe9c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . RUN make all -FROM alpine:3.12 +FROM alpine:3.16 RUN apk add --no-cache ca-certificates cdrkit From f1456cb32e1ca7345d65ebae77dc49426f5d02f0 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Sat, 2 Jul 2022 08:33:06 +0500 Subject: [PATCH 178/489] Azure: Support for accelerated networking (#1344) Signed-off-by: Waleed Malik --- .../provider/azure/create_delete_resources.go | 4 +- pkg/cloudprovider/provider/azure/provider.go | 55 ++++++++++++++----- .../provider/azure/types/types.go | 29 +++++----- 3 files changed, 59 insertions(+), 29 deletions(-) diff --git a/pkg/cloudprovider/provider/azure/create_delete_resources.go b/pkg/cloudprovider/provider/azure/create_delete_resources.go index 90ddce7ea..68931e11a 100644 --- a/pkg/cloudprovider/provider/azure/create_delete_resources.go +++ b/pkg/cloudprovider/provider/azure/create_delete_resources.go @@ -319,7 +319,7 @@ func getVirtualNetwork(ctx context.Context, c *config) (network.VirtualNetwork, return virtualNetworksClient.Get(ctx, c.VNetResourceGroup, c.VNetName, "") } -func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineUID types.UID, config *config, publicIP, publicIPv6 *network.PublicIPAddress, ipFamily util.IPFamily) (*network.Interface, error) { +func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineUID types.UID, config *config, publicIP, publicIPv6 *network.PublicIPAddress, ipFamily util.IPFamily, enableAcceleratedNetworking *bool) (*network.Interface, error) { ifClient, err := getInterfacesClient(config) if err != nil { return nil, fmt.Errorf("failed to create interfaces client: %w", err) @@ -362,6 +362,8 @@ func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineU }) } + ifSpec.InterfacePropertiesFormat.EnableAcceleratedNetworking = enableAcceleratedNetworking + if config.SecurityGroupName != "" { authorizer, err := auth.NewClientCredentialsConfig(config.ClientID, config.ClientSecret, config.TenantID).Authorizer() if err != nil { diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index b5b8e7003..667c85fbd 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -49,9 +49,10 @@ import ( ) const ( - CapabilityPremiumIO = "PremiumIO" - CapabilityUltraSSD = "UltraSSDAvailable" - CapabilityValueTrue = "True" + CapabilityPremiumIO = "PremiumIO" + CapabilityUltraSSD = "UltraSSDAvailable" + CapabilityValueTrue = "True" + capabilityAcceleratedNetworking = "AcceleratedNetworkingEnabled" machineUIDTag = "Machine-UID" @@ -100,8 +101,9 @@ type config struct { DataDiskSize int32 DataDiskSKU *compute.StorageAccountTypes - AssignPublicIP bool - Tags map[string]string + AssignPublicIP bool + EnableAcceleratedNetworking *bool + Tags map[string]string } type azureVM struct { @@ -316,6 +318,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p } c.AssignAvailabilitySet = rawCfg.AssignAvailabilitySet + c.EnableAcceleratedNetworking = rawCfg.EnableAcceleratedNetworking c.AvailabilitySet, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.AvailabilitySet) if err != nil { @@ -612,7 +615,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, err } - iface, err := createOrUpdateNetworkInterface(ctx, ifaceName(machine), machine.UID, config, publicIP, publicIPv6, ipFamily) + iface, err := createOrUpdateNetworkInterface(ctx, ifaceName(machine), machine.UID, config, publicIP, publicIPv6, ipFamily, config.EnableAcceleratedNetworking) if err != nil { return nil, fmt.Errorf("failed to generate main network interface: %w", err) } @@ -935,13 +938,8 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri return s, "azure", nil } -func validateDiskSKUs(ctx context.Context, c *config) error { +func validateDiskSKUs(ctx context.Context, c *config, sku compute.ResourceSku) error { if c.OSDiskSKU != nil || c.DataDiskSKU != nil { - sku, err := getSKU(ctx, c) - if err != nil { - return fmt.Errorf("failed to get VM SKU: %w", err) - } - if c.OSDiskSKU != nil { if _, ok := osDiskSKUs[*c.OSDiskSKU]; !ok { return fmt.Errorf("invalid OS disk SKU '%s'", *c.OSDiskSKU) @@ -972,6 +970,15 @@ func validateDiskSKUs(ctx context.Context, c *config) error { return nil } +func validateSKUCapabilities(ctx context.Context, c *config, sku compute.ResourceSku) error { + if c.EnableAcceleratedNetworking != nil && *c.EnableAcceleratedNetworking { + if !SKUHasCapability(sku, capabilityAcceleratedNetworking) { + return fmt.Errorf("VM size %q does not support accelerated networking", c.VMSize) + } + } + return nil +} + func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { c, providerConfig, err := p.getConfig(spec.ProviderSpec) if err != nil { @@ -1039,10 +1046,19 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return fmt.Errorf("failed to get subnet: %w", err) } - if err := validateDiskSKUs(ctx, c); err != nil { + sku, err := getSKU(ctx, c) + if err != nil { + return fmt.Errorf("failed to get VM SKU: %w", err) + } + + if err := validateDiskSKUs(ctx, c, sku); err != nil { return fmt.Errorf("failed to validate disk SKUs: %w", err) } + if err := validateSKUCapabilities(ctx, c, sku); err != nil { + return fmt.Errorf("failed to validate SKU capabilities: %w", err) + } + _, err = getOSImageReference(c, providerConfig.OperatingSystem) return err } @@ -1092,7 +1108,7 @@ func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Mach } if kuberneteshelper.HasFinalizer(machine, finalizerNIC) { - _, err = createOrUpdateNetworkInterface(ctx, ifaceName(machine), newUID, config, publicIP, publicIPv6, util.Unspecified) + _, err = createOrUpdateNetworkInterface(ctx, ifaceName(machine), newUID, config, publicIP, publicIPv6, util.Unspecified, config.EnableAcceleratedNetworking) if err != nil { return fmt.Errorf("failed to update UID on main network interface: %w", err) } @@ -1239,3 +1255,14 @@ func supportsDiskSKU(vmSKU compute.ResourceSku, diskSKU compute.StorageAccountTy return nil } + +func SKUHasCapability(sku compute.ResourceSku, name string) bool { + if sku.Capabilities != nil { + for _, capability := range *sku.Capabilities { + if capability.Name != nil && *capability.Name == name && *capability.Value == CapabilityValueTrue { + return true + } + } + } + return false +} diff --git a/pkg/cloudprovider/provider/azure/types/types.go b/pkg/cloudprovider/provider/azure/types/types.go index de6de2166..f5705b60c 100644 --- a/pkg/cloudprovider/provider/azure/types/types.go +++ b/pkg/cloudprovider/provider/azure/types/types.go @@ -28,20 +28,21 @@ type RawConfig struct { ClientID providerconfigtypes.ConfigVarString `json:"clientID,omitempty"` ClientSecret providerconfigtypes.ConfigVarString `json:"clientSecret,omitempty"` - Location providerconfigtypes.ConfigVarString `json:"location"` - ResourceGroup providerconfigtypes.ConfigVarString `json:"resourceGroup"` - VNetResourceGroup providerconfigtypes.ConfigVarString `json:"vnetResourceGroup"` - VMSize providerconfigtypes.ConfigVarString `json:"vmSize"` - VNetName providerconfigtypes.ConfigVarString `json:"vnetName"` - SubnetName providerconfigtypes.ConfigVarString `json:"subnetName"` - LoadBalancerSku providerconfigtypes.ConfigVarString `json:"loadBalancerSku"` - RouteTableName providerconfigtypes.ConfigVarString `json:"routeTableName"` - AvailabilitySet providerconfigtypes.ConfigVarString `json:"availabilitySet"` - AssignAvailabilitySet *bool `json:"assignAvailabilitySet"` - SecurityGroupName providerconfigtypes.ConfigVarString `json:"securityGroupName"` - Zones []string `json:"zones"` - ImagePlan *ImagePlan `json:"imagePlan,omitempty"` - ImageReference *ImageReference `json:"imageReference,omitempty"` + Location providerconfigtypes.ConfigVarString `json:"location"` + ResourceGroup providerconfigtypes.ConfigVarString `json:"resourceGroup"` + VNetResourceGroup providerconfigtypes.ConfigVarString `json:"vnetResourceGroup"` + VMSize providerconfigtypes.ConfigVarString `json:"vmSize"` + VNetName providerconfigtypes.ConfigVarString `json:"vnetName"` + SubnetName providerconfigtypes.ConfigVarString `json:"subnetName"` + LoadBalancerSku providerconfigtypes.ConfigVarString `json:"loadBalancerSku"` + RouteTableName providerconfigtypes.ConfigVarString `json:"routeTableName"` + AvailabilitySet providerconfigtypes.ConfigVarString `json:"availabilitySet"` + AssignAvailabilitySet *bool `json:"assignAvailabilitySet"` + SecurityGroupName providerconfigtypes.ConfigVarString `json:"securityGroupName"` + Zones []string `json:"zones"` + ImagePlan *ImagePlan `json:"imagePlan,omitempty"` + ImageReference *ImageReference `json:"imageReference,omitempty"` + EnableAcceleratedNetworking *bool `json:"enableAcceleratedNetworking"` ImageID providerconfigtypes.ConfigVarString `json:"imageID"` OSDiskSize int32 `json:"osDiskSize"` From 03deca3f9d6c1ccfd4faf9296043a1b165404c9a Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 5 Jul 2022 13:54:09 +0500 Subject: [PATCH 179/489] Improvements for CI (#1346) * Remove pinned AMI for AWS Signed-off-by: Waleed Malik * Update CI build image to go-1.18-node-16-kind-0.14-9 Signed-off-by: Waleed Malik * Try upto 5 previous commits for gocache Inspired by https://github.com/kubermatic/kubermatic/pull/10110 Signed-off-by: Waleed Malik * Adjust requests and limits for CI jobs Signed-off-by: Waleed Malik --- .prow/e2e-features.yaml | 36 +++++---- .prow/postsubmits.yaml | 4 +- .prow/provider-alibaba.yaml | 9 ++- .prow/provider-anexia.yaml | 9 ++- .prow/provider-aws.yaml | 81 ++++++++++--------- .prow/provider-azure.yaml | 27 ++++--- .prow/provider-digitalocean.yaml | 9 ++- .prow/provider-equinix-metal.yaml | 9 ++- .prow/provider-gcp.yaml | 9 ++- .prow/provider-hetzner.yaml | 9 ++- .prow/provider-kubevirt.yaml | 9 ++- .prow/provider-linode.yaml | 9 ++- .prow/provider-nutanix.yaml | 9 ++- .prow/provider-openstack.yaml | 18 +++-- .prow/provider-scaleway.yaml | 9 ++- .prow/provider-vmware-cloud-director.yaml | 9 ++- .prow/provider-vsphere.yaml | 27 ++++--- .prow/verify.yaml | 38 +++++++-- Makefile | 3 +- hack/ci/download-gocache.sh | 45 +++++++---- hack/ci/setup-kind-cluster.sh | 2 +- hack/ci/upload-gocache.sh | 2 +- .../machinedeployment-aws-arm-machines.yaml | 3 +- ...deployment-aws-ebs-encryption-enabled.yaml | 1 - 24 files changed, 227 insertions(+), 159 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 93a9e3e22..aa4c12907 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -16,7 +16,6 @@ presubmits: - name: pull-machine-controller-e2e-invalid-objects-get-rejected always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-aws: "true" @@ -35,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -44,13 +43,14 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-e2e-custom-ca always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-openstack: "true" @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -72,13 +72,14 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-e2e-ubuntu-upgrade always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-openstack: "true" @@ -90,7 +91,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -99,13 +100,14 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-e2e-deployment-upgrade always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-hetzner: "true" @@ -116,7 +118,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -125,5 +127,7 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index f3e9361c8..baba3fd92 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index da5dd7f80..e0cfc2ed2 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -17,7 +17,6 @@ presubmits: optional: true always_run: false decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" max_concurrency: 1 labels: @@ -30,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -39,5 +38,7 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 50825a12b..96a7ab09e 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -16,7 +16,6 @@ presubmits: - name: pull-machine-controller-e2e-anexia always_run: false decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-hetzner: "true" @@ -28,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -37,5 +36,7 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 834031126..2ea31d349 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -16,7 +16,6 @@ presubmits: - name: pull-machine-controller-e2e-aws always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-aws: "true" @@ -29,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -38,13 +37,14 @@ presubmits: privileged: true resources: requests: - memory: 4Gi - cpu: 1 + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-e2e-aws-arm always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-aws: "true" @@ -56,7 +56,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -65,13 +65,14 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-e2e-aws-ebs-encryption-enabled always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-aws: "true" @@ -83,7 +84,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,13 +93,14 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-e2e-aws-flatcar-containerd always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-aws: "true" @@ -110,7 +112,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -119,13 +121,14 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-e2e-aws-spot-instance always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-aws: "true" @@ -138,7 +141,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -147,13 +150,14 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-e2e-aws-sles always_run: false decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-aws: "true" @@ -165,7 +169,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -174,13 +178,14 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-e2e-aws-flatcar-coreos-cloud-init always_run: false decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-aws: "true" @@ -192,7 +197,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -201,13 +206,14 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-e2e-aws-centos8 always_run: false decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-aws: "true" @@ -219,7 +225,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -228,13 +234,14 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-e2e-aws-assume-role always_run: false decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-aws-assume-role: "true" @@ -246,7 +253,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -255,5 +262,7 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index d1e9f689d..6bee8b9f2 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -16,7 +16,6 @@ presubmits: - name: pull-machine-controller-e2e-azure always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-azure: "true" @@ -29,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -38,13 +37,14 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-e2e-azure-custom-image-reference always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-azure: "true" @@ -57,7 +57,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -66,14 +66,15 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-e2e-azure-redhat-satellite optional: true always_run: false decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-azure: "true" @@ -86,7 +87,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -95,5 +96,7 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 7989696f6..45f53e484 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -16,7 +16,6 @@ presubmits: - name: pull-machine-controller-e2e-digitalocean always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-digitalocean: "true" @@ -28,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -37,5 +36,7 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 9754203b5..4193213d6 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -17,7 +17,6 @@ presubmits: optional: true run_if_changed: pkg\/cloudprovider\/provider\/equinixmetal\/.* decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-hetzner: "true" @@ -29,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -38,5 +37,7 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index c2788019a..d16d63ae7 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -16,7 +16,6 @@ presubmits: - name: pull-machine-controller-e2e-gce always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-gce: "true" @@ -29,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -38,5 +37,7 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 6f52328b1..493507eec 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -16,7 +16,6 @@ presubmits: - name: pull-machine-controller-e2e-hetzner always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-hetzner: "true" @@ -27,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -36,5 +35,7 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 477f9ada2..29184045a 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -16,7 +16,6 @@ presubmits: - name: pull-machine-controller-e2e-kubevirt always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" max_concurrency: 1 labels: @@ -30,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -39,5 +38,7 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index f1b416667..2e3cf370f 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -17,7 +17,6 @@ presubmits: always_run: false optional: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-hetzner: "true" @@ -29,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -38,5 +37,7 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 6b3c68fc0..a1e28f26b 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -19,7 +19,6 @@ presubmits: # TODO uncomment this when Nutanix is in a working condition #run_if_changed: "(pkg/cloudprovider/provider/nutanix/)" decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-hetzner: "true" @@ -31,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -40,5 +39,7 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 7fec6859a..a9205eeb4 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -16,7 +16,6 @@ presubmits: - name: pull-machine-controller-e2e-openstack always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-openstack: "true" @@ -29,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -38,13 +37,14 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-e2e-openstack-project-auth always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-openstack: "true" @@ -57,7 +57,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -66,5 +66,7 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 3fe04bfc7..1a2f06e0c 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -16,7 +16,6 @@ presubmits: - name: pull-machine-controller-e2e-scaleway always_run: false decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-scaleway: "true" @@ -28,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -37,5 +36,7 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 99704f8b6..32052adcd 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -16,7 +16,6 @@ presubmits: - name: pull-machine-controller-e2e-vmware-cloud-director always_run: false decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" run_if_changed: "(pkg/cloudprovider/provider/vmwareclouddirector/|pkg/userdata)" labels: @@ -30,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -39,5 +38,7 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 10a9d9ed9..ece985542 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -16,7 +16,6 @@ presubmits: - name: pull-machine-controller-e2e-vsphere always_run: true decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-hetzner: "true" @@ -29,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -38,13 +37,14 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-e2e-vsphere-datastore-cluster always_run: false decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-vsphere: "true" @@ -57,7 +57,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -66,13 +66,14 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-e2e-vsphere-resource-pool always_run: false decorate: true - error_on_eviction: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-vsphere: "true" @@ -85,7 +86,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-5 + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -94,5 +95,7 @@ presubmits: privileged: true resources: requests: - memory: 1Gi - cpu: 500m + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 66eca1e82..43de08569 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -29,7 +29,10 @@ presubmits: - all resources: requests: - cpu: 1 + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi - name: pull-machine-controller-dependencies always_run: true @@ -46,7 +49,11 @@ presubmits: - check-dependencies resources: requests: - cpu: 800m + memory: 32Mi + cpu: 50m + limits: + memory: 256Mi + cpu: 250m - name: pull-machine-controller-lint always_run: true @@ -65,6 +72,8 @@ presubmits: requests: cpu: 800m memory: 6Gi + limits: + memory: 6Gi - name: pull-machine-controller-yamllint always_run: true @@ -81,7 +90,11 @@ presubmits: - "yamllint -c .yamllint.conf ." resources: requests: - cpu: 200m + memory: 32Mi + cpu: 50m + limits: + memory: 256Mi + cpu: 250m - name: pre-machine-controller-verify-shfmt run_if_changed: "^hack/" @@ -122,8 +135,11 @@ presubmits: - "./hack/verify-boilerplate.sh" resources: requests: - memory: 64Mi - cpu: 100m + memory: 32Mi + cpu: 50m + limits: + memory: 256Mi + cpu: 250m - name: pull-machine-controller-license-validation run_if_changed: "^go.(mod|sum)$" @@ -138,8 +154,11 @@ presubmits: - ./hack/verify-licenses.sh resources: requests: - memory: 512Mi - cpu: 1 + memory: 32Mi + cpu: 50m + limits: + memory: 256Mi + cpu: 250m - name: pull-machine-controller-test always_run: true @@ -157,4 +176,7 @@ presubmits: - test-unit resources: requests: - cpu: 800m + cpu: 3 + memory: 6Gi + limits: + memory: 6Gi diff --git a/Makefile b/Makefile index 71feb43fa..bfb27d53b 100644 --- a/Makefile +++ b/Makefile @@ -95,8 +95,7 @@ test-unit-docker: .PHONY: test-unit test-unit: - @#The `-race` flag requires CGO - CGO_ENABLED=1 go test -v -race ./... + go test -v ./... .PHONY: build-tests build-tests: diff --git a/hack/ci/download-gocache.sh b/hack/ci/download-gocache.sh index 4f5178079..650d67032 100755 --- a/hack/ci/download-gocache.sh +++ b/hack/ci/download-gocache.sh @@ -21,6 +21,8 @@ set -euo pipefail # receives a SIGINT set -o monitor +source $(dirname $0)/../lib.sh + # The gocache needs a matching go version to work, so append that to the name GO_VERSION="$(go version | awk '{ print $3 }' | sed 's/go//g')" @@ -34,7 +36,7 @@ exit_gracefully() { trap exit_gracefully EXIT if [ -z "${GOCACHE_MINIO_ADDRESS:-}" ]; then - echo "env var GOCACHE_MINIO_ADDRESS unset, can not download gocache" + echodate "env var GOCACHE_MINIO_ADDRESS unset, cannot download gocache" exit 0 fi @@ -53,25 +55,40 @@ if [[ -z "${CACHE_VERSION}" ]]; then GIT_BRANCH="master" fi -if [ -z "${PULL_NUMBER:-}" ]; then - # Special case: This is called in a Postubmit. Go one revision back, - # as there can't be a cache for the current revision - CACHE_VERSION="$(git rev-parse ${CACHE_VERSION}~1)" -fi - # normalize branch name to prevent accidental directories being created GIT_BRANCH="$(echo "$GIT_BRANCH" | sed 's#/#-#g')" ARCHIVE_NAME="${CACHE_VERSION}-${GO_VERSION}.tar" URL="${GOCACHE_MINIO_ADDRESS}/machine-controller/${GIT_BRANCH}/${ARCHIVE_NAME}" -# Do not go through the retry loop when there is nothing -if ! curl --head --silent --fail "${URL}" > /dev/null; then - echo "Remote has no gocache ${ARCHIVE_NAME}, exiting" +# Do not go through the retry loop when there is nothing, but do try the +# first few parents if no cache was found. This is helpful for retests happening +# quickly after something got merged to master and no gocache for the most +# recent commit exists yet. In this case, taking the previous commit's +# cache is better than nothing. This also helps for postsubmits, where the current +# commit (the one that got merged) cannot have a cache yet. +HAS_CACHE=false +for i in $(seq 1 5); do + # check if we have a cache for the given git revision + if curl --head --silent --fail "${URL}" > /dev/null; then + HAS_CACHE=true + break + fi + echodate "No gocache machine-controller/${GIT_BRANCH}/${ARCHIVE_NAME} available, trying previous commit as a fallback..." + + CACHE_VERSION="$(git rev-parse ${CACHE_VERSION}~1)" + ARCHIVE_NAME="${CACHE_VERSION}-${GO_VERSION}.tar" + URL="${GOCACHE_MINIO_ADDRESS}/machine-controller/${GIT_BRANCH}/${ARCHIVE_NAME}" +done +if ! $HAS_CACHE; then + echodate "Could not find any suitable gocaches, giving up." exit 0 fi -echo "Downloading and extracting gocache" -curl --fail --header "Content-Type: application/octet-stream" "${URL}" | tar -C $GOCACHE -xf - - -echo "Successfully fetched gocache into $GOCACHE" +echodate "Downloading and extracting gocache" +TEST_NAME="Download and extract gocache" +# Passing the Headers as space-separated literals doesn't seem to work +# in conjunction with the retry func, so we just put them in a file instead +echo 'Content-Type: application/octet-stream' > /tmp/headers +retry 5 curl --fail -H @/tmp/headers "${URL}" | tar -C $GOCACHE -xf - +echodate "Successfully fetched gocache into $GOCACHE" diff --git a/hack/ci/setup-kind-cluster.sh b/hack/ci/setup-kind-cluster.sh index bd8587761..d03becb3a 100755 --- a/hack/ci/setup-kind-cluster.sh +++ b/hack/ci/setup-kind-cluster.sh @@ -191,7 +191,7 @@ if [ -z "${DISABLE_CLUSTER_EXPOSER:-}" ]; then TEST_NAME="Wait for cluster exposer" echodate "Waiting for cluster exposer to be running" - retry 5 curl -s --fail http://127.0.0.1:2047/metrics -o /dev/null + retry 10 curl -s --fail http://127.0.0.1:2047/metrics -o /dev/null echodate "Cluster exposer is running" echodate "Setting up iptables rules to make nodeports available" diff --git a/hack/ci/upload-gocache.sh b/hack/ci/upload-gocache.sh index 2ff04c5ad..5db05684f 100755 --- a/hack/ci/upload-gocache.sh +++ b/hack/ci/upload-gocache.sh @@ -55,7 +55,7 @@ ARCHIVE_FILE="/tmp/${GIT_HEAD_HASH}.tar" # No compression because that needs quite a bit of CPU tar -C "$GOCACHE" -cf "$ARCHIVE_FILE" . -echo "Uploading gocache archive" +echo "Uploading gocache archive machine-controller/${GIT_BRANCH}/${GIT_HEAD_HASH}-${GO_VERSION}.tar" curl \ --fail \ --upload-file "${ARCHIVE_FILE}" \ diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml index bc81a8a15..e4c0d6375 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml @@ -34,8 +34,7 @@ spec: diskSize: 50 diskType: "gp2" ebsVolumeEncrypted: false - # TODO: Revert this to "<< AMI >>" when https://github.com/kubermatic/kubermatic/issues/10022 is marked as resolved. - ami: "ami-07d0e9bbaa6dad756" + ami: "<< AMI >>" securityGroupIDs: - "sg-a2c195ca" tags: diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml index a1bd27bed..ba06debe1 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml @@ -34,7 +34,6 @@ spec: diskSize: 50 diskType: "gp2" ebsVolumeEncrypted: true - ami: "<< AMI >>" securityGroupIDs: - "sg-a2c195ca" tags: From d1f0a51169c1e269e66762e18e778558f9e6af6d Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 5 Jul 2022 17:29:10 +0500 Subject: [PATCH 180/489] Remove support for Kubernetes 1.21 (#1343) Signed-off-by: Waleed Malik --- README.md | 1 - pkg/userdata/amzn2/provider_test.go | 47 +- .../amzn2/testdata/kubelet-v1.21-aws.yaml | 436 ---------------- ...l.yaml => kubelet-v1.23-aws-external.yaml} | 4 +- .../amzn2/testdata/kubelet-v1.23-aws.yaml | 2 +- ...aml => kubelet-v1.23-vsphere-mirrors.yaml} | 4 +- ....yaml => kubelet-v1.23-vsphere-proxy.yaml} | 4 +- ...sphere.yaml => kubelet-v1.23-vsphere.yaml} | 4 +- pkg/userdata/centos/provider_test.go | 57 +-- .../centos/testdata/kubelet-v1.21-aws.yaml | 446 ---------------- ...l.yaml => kubelet-v1.23-aws-external.yaml} | 4 +- ...aml => kubelet-v1.23-vsphere-mirrors.yaml} | 4 +- ....yaml => kubelet-v1.23-vsphere-proxy.yaml} | 4 +- ...sphere.yaml => kubelet-v1.23-vsphere.yaml} | 4 +- pkg/userdata/flatcar/provider_test.go | 64 +-- .../flatcar/testdata/cloud-init_v1.21.10.yaml | 478 ----------------- pkg/userdata/flatcar/testdata/containerd.yaml | 5 +- .../flatcar/testdata/ignition_v1.21.10.json | 1 - pkg/userdata/helper/common_test.go | 1 - .../download_binaries_v1.21.10.golden | 17 - ...temd_unit_version-v1.21.10-external.golden | 39 -- ...ublet_systemd_unit_version-v1.21.10.golden | 38 -- pkg/userdata/rhel/provider_test.go | 47 +- .../rhel/testdata/kubelet-v1.21-aws.yaml | 481 ------------------ pkg/userdata/rockylinux/provider_test.go | 51 +- .../testdata/kubelet-v1.21-aws.yaml | 441 ---------------- ...l.yaml => kubelet-v1.23-aws-external.yaml} | 4 +- ...aml => kubelet-v1.23-vsphere-mirrors.yaml} | 4 +- ....yaml => kubelet-v1.23-vsphere-proxy.yaml} | 4 +- ...sphere.yaml => kubelet-v1.23-vsphere.yaml} | 4 +- pkg/userdata/sles/provider_test.go | 1 - .../sles/testdata/version-1.21.10.yaml | 425 ---------------- pkg/userdata/ubuntu/provider_test.go | 3 +- pkg/userdata/ubuntu/testdata/nutanix.yaml | 4 +- .../ubuntu/testdata/version-1.21.10.yaml | 448 ---------------- test/e2e/provisioning/all_e2e_test.go | 18 +- test/e2e/provisioning/helper.go | 1 - 37 files changed, 110 insertions(+), 3490 deletions(-) delete mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml rename pkg/userdata/amzn2/testdata/{kubelet-v1.21-aws-external.yaml => kubelet-v1.23-aws-external.yaml} (98%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.21-vsphere-mirrors.yaml => kubelet-v1.23-vsphere-mirrors.yaml} (99%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.21-vsphere-proxy.yaml => kubelet-v1.23-vsphere-proxy.yaml} (99%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.21-vsphere.yaml => kubelet-v1.23-vsphere.yaml} (98%) delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml rename pkg/userdata/centos/testdata/{kubelet-v1.21-aws-external.yaml => kubelet-v1.23-aws-external.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.21-vsphere-mirrors.yaml => kubelet-v1.23-vsphere-mirrors.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.21-vsphere-proxy.yaml => kubelet-v1.23-vsphere-proxy.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.21-vsphere.yaml => kubelet-v1.23-vsphere.yaml} (99%) delete mode 100644 pkg/userdata/flatcar/testdata/cloud-init_v1.21.10.yaml delete mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.21.10.json delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.21.10.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10.golden delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml rename pkg/userdata/rockylinux/testdata/{kubelet-v1.21-aws-external.yaml => kubelet-v1.23-aws-external.yaml} (98%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.21-vsphere-mirrors.yaml => kubelet-v1.23-vsphere-mirrors.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.21-vsphere-proxy.yaml => kubelet-v1.23-vsphere-proxy.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.21-vsphere.yaml => kubelet-v1.23-vsphere.yaml} (98%) delete mode 100644 pkg/userdata/sles/testdata/version-1.21.10.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.21.10.yaml diff --git a/README.md b/README.md index 9674c1d0d..2627af999 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,6 @@ Currently supported K8S versions are: - 1.24 - 1.23 - 1.22 -- 1.21 ## What does not work diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index 9cd2057b2..baea9d9d0 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -100,40 +100,49 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.21-aws", + name: "kubelet-v1.22-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.22.5", + }, + }, + }, + { + name: "kubelet-v1.23-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.8", + Kubelet: "1.23.5", }, }, }, { - name: "kubelet-v1.21-aws-external", + name: "kubelet-v1.23-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.8", + Kubelet: "1.23.5", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.21-vsphere", + name: "kubelet-v1.23-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.8", + Kubelet: "1.23.5", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.21-vsphere-proxy", + name: "kubelet-v1.23-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.8", + Kubelet: "1.23.5", }, }, cloudProviderName: stringPtr("vsphere"), @@ -143,11 +152,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.21-vsphere-mirrors", + name: "kubelet-v1.23-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.8", + Kubelet: "1.23.5", }, }, cloudProviderName: stringPtr("vsphere"), @@ -156,24 +165,6 @@ func TestUserDataGeneration(t *testing.T) { registryMirrors: "https://registry.docker-cn.com/", pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, - { - name: "kubelet-v1.22-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.5", - }, - }, - }, - { - name: "kubelet-v1.23-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.0", - }, - }, - }, { name: "kubelet-v1.24-aws", spec: clusterv1alpha1.MachineSpec{ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml deleted file mode 100644 index 7a8b853b9..000000000 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml +++ /dev/null @@ -1,436 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml similarity index 98% rename from pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml index d08b2f73a..77275d458 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml @@ -139,7 +139,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -225,8 +225,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml index 4733cab46..10cf2cebe 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml @@ -139,7 +139,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-mirrors.yaml index e2fcbf6f3..bb9ac745e 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -152,7 +152,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -242,8 +242,6 @@ write_files: --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-proxy.yaml index 5c96b174e..9c36c7abd 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -152,7 +152,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -242,8 +242,6 @@ write_files: --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere.yaml similarity index 98% rename from pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere.yaml index 4c4c07bb7..c6eb43a78 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.8}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -233,8 +233,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index 37f36be41..61f5ce8f5 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -100,89 +100,80 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.21-aws", + name: "kubelet-v1.22-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.10", + Kubelet: "1.22.7", }, }, }, { - name: "kubelet-v1.21-aws-external", + name: "kubelet-v1.23-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.10", + Kubelet: "1.23.5", }, }, - externalCloudProvider: true, }, { - name: "kubelet-v1.21-vsphere", + name: "kubelet-v1.23-nutanix", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.10", + Kubelet: "1.23.5", }, }, - cloudProviderName: stringPtr("vsphere"), + cloudProviderName: stringPtr("nutanix"), }, { - name: "kubelet-v1.21-vsphere-proxy", + name: "kubelet-v1.23-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.10", + Kubelet: "1.23.5", }, }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128/", - noProxy: "192.168.1.0", - insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", + externalCloudProvider: true, }, { - name: "kubelet-v1.21-vsphere-mirrors", + name: "kubelet-v1.23-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.10", + Kubelet: "1.23.5", }, }, cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128/", - noProxy: "192.168.1.0", - registryMirrors: "https://registry.docker-cn.com/", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.22-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.7", - }, - }, - }, - { - name: "kubelet-v1.23-aws", + name: "kubelet-v1.23-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ Kubelet: "1.23.5", }, }, + cloudProviderName: stringPtr("vsphere"), + httpProxy: "http://192.168.100.100:3128/", + noProxy: "192.168.1.0", + insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", + pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.23-nutanix", + name: "kubelet-v1.23-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ Kubelet: "1.23.5", }, }, - cloudProviderName: stringPtr("nutanix"), + cloudProviderName: stringPtr("vsphere"), + httpProxy: "http://192.168.100.100:3128/", + noProxy: "192.168.1.0", + registryMirrors: "https://registry.docker-cn.com/", + pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { name: "kubelet-v1.24-aws", diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml deleted file mode 100644 index 4f7bee9e1..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws.yaml +++ /dev/null @@ -1,446 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml index fa03994b4..da8f203ec 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -235,8 +235,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 3c7b56718..11c411561 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -162,7 +162,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -252,8 +252,6 @@ write_files: --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml index 7dd0ddc5a..8cc2eb137 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -162,7 +162,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -252,8 +252,6 @@ write_files: --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml index b325de29d..aee1a886d 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml @@ -154,7 +154,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -243,8 +243,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index 18688dd67..cc82c03a9 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -121,37 +121,6 @@ func TestUserDataGeneration(t *testing.T) { t.Parallel() tests := []userDataTestCase{ - { - name: "ignition_v1.21.10", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", - SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, - CAPublicKey: "ssh-rsa AAABBB", - Network: &providerconfigtypes.NetworkConfig{ - CIDR: "192.168.81.4/24", - Gateway: "192.168.81.1", - DNS: providerconfigtypes.DNSConfig{ - Servers: []string{"8.8.8.8"}, - }, - }, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.21.10", - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - osConfig: &Config{ - DisableAutoUpdate: true, - ProvisioningUtility: Ignition, - }, - }, { name: "ignition_v1.22.7", providerSpec: &providerconfigtypes.Config{ @@ -276,37 +245,6 @@ func TestUserDataGeneration(t *testing.T) { ProvisioningUtility: Ignition, }, }, - { - name: "cloud-init_v1.21.10", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "anexia", - SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, - CAPublicKey: "ssh-rsa AAABBB", - Network: &providerconfigtypes.NetworkConfig{ - CIDR: "192.168.81.4/24", - Gateway: "192.168.81.1", - DNS: providerconfigtypes.DNSConfig{ - Servers: []string{"8.8.8.8"}, - }, - }, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.21.10", - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "anexia", - config: "{anexia-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - osConfig: &Config{ - DisableAutoUpdate: true, - ProvisioningUtility: CloudInit, - }, - }, { name: "cloud-init_v1.22.7", providerSpec: &providerconfigtypes.Config{ @@ -413,7 +351,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "v1.21.10", + Kubelet: "v1.24.0", }, }, ccProvider: &fakeCloudConfigProvider{}, diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.10.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.21.10.yaml deleted file mode 100644 index a1b39a561..000000000 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.21.10.yaml +++ /dev/null @@ -1,478 +0,0 @@ -#cloud-config - -users: -- name: core - ssh_authorized_keys: - - ssh-rsa AAABBB - - ssh-rsa CCCDDD - - -coreos: - units: - - name: static-nic.network - content: | - [Match] - # Because of difficulty predicting specific NIC names on different cloud providers, - # we only support static addressing on VSphere. There should be a single NIC attached - # that we will match by name prefix 'en' which denotes ethernet devices. - Name=en* - - [Network] - DHCP=no - Address=192.168.81.4/24 - Gateway=192.168.81.1 - DNS=8.8.8.8 - - - name: update-engine.service - command: stop - mask: true - - name: locksmithd.service - command: stop - mask: true - - name: download-script.service - enable: true - command: start - content: | - [Unit] - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/download.sh - [Install] - WantedBy=multi-user.target - - - name: kubelet-healthcheck.service - enable: true - command: start - drop-ins: - - name: 40-download.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - - - name: nodeip.service - enable: true - command: start - content: | - [Unit] - Description=Setup Kubelet Node IP Env - Requires=network-online.target - After=network-online.target - - [Service] - ExecStart=/opt/bin/setup_net_env.sh - RemainAfterExit=yes - Type=oneshot - [Install] - WantedBy=multi-user.target - - - name: kubelet.service - enable: true - command: start - drop-ins: - - name: 10-nodeip.conf - content: | - [Service] - EnvironmentFile=/etc/kubernetes/nodeip.conf - - name: 40-download.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=anexia \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - - - name: apply-sysctl-settings.service - enable: true - command: start - content: | - [Unit] - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - ExecStart=/opt/bin/apply_sysctl_settings.sh - [Install] - WantedBy=multi-user.target - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - permissions: "0644" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/etc/kubernetes/kubelet.conf" - permissions: "0644" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /opt/load-kernel-modules.sh - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: /etc/sysctl.d/k8s.conf - permissions: "0644" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: /etc/kubernetes/bootstrap-kubelet.conf - permissions: "0400" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: /etc/kubernetes/cloud-config - permissions: "0400" - content: | - {anexia-config:true} - -- path: /etc/kubernetes/pki/ca.crt - permissions: "0644" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - -- path: /etc/hostname - permissions: "0600" - content: 'node1' - -- path: /etc/ssh/sshd_config - permissions: "0600" - user: root - content: | - # Use most defaults for sshd configuration. - Subsystem sftp internal-sftp - ClientAliveInterval 180 - UseDNS no - UsePAM yes - PrintLastLog no # handled by PAM - PrintMotd no # handled by PAM - PasswordAuthentication no - ChallengeResponseAuthentication no - -- path: /opt/bin/download.sh - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - systemctl daemon-reload - systemctl enable --now docker - - systemctl disable download-script.service - -- path: /opt/bin/apply_sysctl_settings.sh - permissions: "0755" - user: root - content: | - #!/bin/bash - set -xeuo pipefail - sysctl --system - systemctl disable apply-sysctl-settings.service - -- path: "/etc/ssh/trusted-user-ca-keys.pem" - content: | - ssh-rsa AAABBB - -- path: "/etc/ssh/sshd_config" - content: | - TrustedUserCAKeys /etc/ssh/trusted-user-ca-keys.pem - CASignatureAlgorithms ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,ssh-ed25519,rsa-sha2-512,rsa-sha2-256,ssh-rsa - append: true - -- path: /etc/docker/daemon.json - permissions: "0644" - user: root - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/crictl.yaml - permissions: "0644" - user: root - content: | - runtime-endpoint: unix:///run/containerd/containerd.sock diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index 3f98342f8..bcd1eedbd 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -110,9 +110,6 @@ coreos: --lock-file=/tmp/kubelet.lock \ --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] @@ -399,7 +396,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" + KUBE_VERSION="${KUBE_VERSION:-v1.24.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.21.10.json b/pkg/userdata/flatcar/testdata/ignition_v1.21.10.json deleted file mode 100644 index b40d43f74..000000000 --- a/pkg/userdata/flatcar/testdata/ignition_v1.21.10.json +++ /dev/null @@ -1 +0,0 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.21.10%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/common_test.go b/pkg/userdata/helper/common_test.go index 1c1d61897..082a49bc6 100644 --- a/pkg/userdata/helper/common_test.go +++ b/pkg/userdata/helper/common_test.go @@ -26,7 +26,6 @@ var update = flag.Bool("update", false, "update testdata files") var ( versions = []*semver.Version{ - semver.MustParse("v1.21.10"), semver.MustParse("v1.22.7"), semver.MustParse("v1.23.5"), semver.MustParse("v1.24.0"), diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.21.10.golden b/pkg/userdata/helper/testdata/download_binaries_v1.21.10.golden deleted file mode 100644 index 75ed990ec..000000000 --- a/pkg/userdata/helper/testdata/download_binaries_v1.21.10.golden +++ /dev/null @@ -1,17 +0,0 @@ -mkdir -p /opt/bin/ -mkdir -p /var/lib/calico -mkdir -p /etc/kubernetes/manifests -mkdir -p /etc/cni/net.d -mkdir -p /opt/cni/bin -if [ ! -f /opt/cni/bin/loopback ]; then - curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - -fi -if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.21.10/bin/linux/amd64/kubelet - chmod +x /opt/bin/kubelet -fi - -if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh -fi diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10-external.golden deleted file mode 100644 index 435cefdf8..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10-external.golden +++ /dev/null @@ -1,39 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10.golden deleted file mode 100644 index e1c0b6225..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.21.10.golden +++ /dev/null @@ -1,38 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index 82c1bcae6..25051f24f 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -99,15 +99,6 @@ func TestUserDataGeneration(t *testing.T) { t.Parallel() tests := []userDataTestCase{ - { - name: "kubelet-v1.21-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.10", - }, - }, - }, { name: "kubelet-v1.22-aws", spec: clusterv1alpha1.MachineSpec{ @@ -136,25 +127,6 @@ func TestUserDataGeneration(t *testing.T) { }, externalCloudProvider: true, }, - { - name: "kubelet-v1.24-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.0", - }, - }, - }, - { - name: "kubelet-v1.24-aws-external", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.0", - }, - }, - externalCloudProvider: true, - }, { name: "kubelet-v1.23-vsphere", spec: clusterv1alpha1.MachineSpec{ @@ -193,6 +165,25 @@ func TestUserDataGeneration(t *testing.T) { registryMirrors: "https://registry.docker-cn.com/", pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, + { + name: "kubelet-v1.24-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.24.0", + }, + }, + }, + { + name: "kubelet-v1.24-aws-external", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.24.0", + }, + }, + externalCloudProvider: true, + }, { name: "kubelet-v1.22-nutanix", spec: clusterv1alpha1.MachineSpec{ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml deleted file mode 100644 index ccaf019ea..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.21-aws.yaml +++ /dev/null @@ -1,481 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rockylinux/provider_test.go b/pkg/userdata/rockylinux/provider_test.go index 3927c846f..e709a2ee0 100644 --- a/pkg/userdata/rockylinux/provider_test.go +++ b/pkg/userdata/rockylinux/provider_test.go @@ -100,40 +100,49 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.21-aws", + name: "kubelet-v1.22-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.10", + Kubelet: "1.22.7", }, }, }, { - name: "kubelet-v1.21-aws-external", + name: "kubelet-v1.23-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.10", + Kubelet: "1.23.5", + }, + }, + }, + { + name: "kubelet-v1.23-aws-external", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.23.5", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.21-vsphere", + name: "kubelet-v1.23-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.10", + Kubelet: "1.23.5", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.21-vsphere-proxy", + name: "kubelet-v1.23-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.10", + Kubelet: "1.23.5", }, }, cloudProviderName: stringPtr("vsphere"), @@ -143,11 +152,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.21-vsphere-mirrors", + name: "kubelet-v1.23-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.10", + Kubelet: "1.23.5", }, }, cloudProviderName: stringPtr("vsphere"), @@ -157,22 +166,14 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.22-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.7", - }, - }, - }, - { - name: "kubelet-v1.23-aws", + name: "kubelet-v1.23-nutanix", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ Kubelet: "1.23.5", }, }, + cloudProviderName: stringPtr("nutanix"), }, { name: "kubelet-v1.24-aws", @@ -183,16 +184,6 @@ func TestUserDataGeneration(t *testing.T) { }, }, }, - { - name: "kubelet-v1.23-nutanix", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", - }, - }, - cloudProviderName: stringPtr("nutanix"), - }, } defaultCloudProvider := &fakeCloudConfigProvider{ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml deleted file mode 100644 index 733f76cd2..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws.yaml +++ /dev/null @@ -1,441 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml similarity index 98% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml index 379993a03..1621348e7 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -230,8 +230,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml index a9f0f2121..5524b8cde 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -157,7 +157,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -247,8 +247,6 @@ write_files: --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml index 36f5e4acf..931fdde4f 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -157,7 +157,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -247,8 +247,6 @@ write_files: --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml similarity index 98% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml index 2a373b371..e701f5963 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.21-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -238,8 +238,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/sles/provider_test.go b/pkg/userdata/sles/provider_test.go index 302344db9..ba589e583 100644 --- a/pkg/userdata/sles/provider_test.go +++ b/pkg/userdata/sles/provider_test.go @@ -126,7 +126,6 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.21.10"), semver.MustParse("v1.22.7"), semver.MustParse("v1.23.5"), semver.MustParse("v1.24.0"), diff --git a/pkg/userdata/sles/testdata/version-1.21.10.yaml b/pkg/userdata/sles/testdata/version-1.21.10.yaml deleted file mode 100644 index 2c781afee..000000000 --- a/pkg/userdata/sles/testdata/version-1.21.10.yaml +++ /dev/null @@ -1,425 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 47914c90b..8f0ebd10b 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -127,7 +127,6 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.21.10"), semver.MustParse("v1.22.7"), semver.MustParse("v1.23.5"), semver.MustParse("v1.24.0"), @@ -489,7 +488,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.21.10", + Kubelet: "1.23.5", }, }, ccProvider: &fakeCloudConfigProvider{ diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index f49afbc6d..58c16ae1f 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -152,7 +152,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.21.10}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -235,8 +235,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml b/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml deleted file mode 100644 index a40f49324..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.21.10.yaml +++ /dev/null @@ -1,448 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index c291f6885..6c3a5f9e5 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -79,6 +79,8 @@ const ( nutanixManifest = "./testdata/machinedeployment-nutanix.yaml" ) +const defaultKubernetesVersion = "1.23.5" + var testRunIdentifier = flag.String("identifier", "local", "The unique identifier for this test run") func TestInvalidObjectsGetRejected(t *testing.T) { @@ -357,7 +359,7 @@ func TestOpenstackProjectAuthProvisioningE2E(t *testing.T) { name: "MachineDeploy with project auth vars", osName: "ubuntu", containerRuntime: "containerd", - kubernetesVersion: "1.21.8", + kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } testScenario(t, scenario, *testRunIdentifier, params, OSManifestProjectAuth, false) @@ -429,7 +431,7 @@ func TestAWSAssumeRoleProvisioningE2E(t *testing.T) { name: "AWS with AssumeRole", osName: "ubuntu", containerRuntime: "docker", - kubernetesVersion: "1.22.5", + kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } testScenario(t, scenario, *testRunIdentifier, params, AWSManifest, false) @@ -538,7 +540,7 @@ func TestAWSFlatcarContainerdProvisioningE2E(t *testing.T) { name: "flatcar with containerd in AWS", osName: "flatcar", containerRuntime: "containerd", - kubernetesVersion: "1.22.5", + kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } testScenario(t, scenario, *testRunIdentifier, params, AWSManifest, false) @@ -588,7 +590,7 @@ func TestAWSEbsEncryptionEnabledProvisioningE2E(t *testing.T) { name: "AWS with ebs encryption enabled", osName: "ubuntu", containerRuntime: "containerd", - kubernetesVersion: "v1.21.8", + kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } testScenario(t, scenario, fmt.Sprintf("aws-%s", *testRunIdentifier), params, AWSEBSEncryptedManifest, false) @@ -677,7 +679,7 @@ func TestAzureRedhatSatelliteProvisioningE2E(t *testing.T) { name: "Azure redhat satellite server subscription", osName: "rhel", containerRuntime: "docker", - kubernetesVersion: "1.21.8", + kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } @@ -877,7 +879,7 @@ func TestVsphereResourcePoolProvisioningE2E(t *testing.T) { name: "vSphere resource pool provisioning", osName: "flatcar", containerRuntime: "docker", - kubernetesVersion: "1.22.5", + kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } @@ -993,7 +995,7 @@ func TestUbuntuProvisioningWithUpgradeE2E(t *testing.T) { name: "Ubuntu upgrade", osName: "ubuntu", containerRuntime: "docker", - kubernetesVersion: "1.22.5", + kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } @@ -1018,7 +1020,7 @@ func TestDeploymentControllerUpgradesMachineE2E(t *testing.T) { name: "MachineDeployment upgrade", osName: "ubuntu", containerRuntime: "docker", - kubernetesVersion: "1.21.8", + kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateUpdateAndDelete, } testScenario(t, scenario, *testRunIdentifier, params, HZManifest, false) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 446b6d068..532b0abce 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,7 +33,6 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.21.10"), semver.MustParse("v1.22.7"), semver.MustParse("v1.23.5"), semver.MustParse("v1.24.0"), From 5c8d3b44f270b6d39c3cbda991b9c4f39e37b77a Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 6 Jul 2022 20:44:16 +0500 Subject: [PATCH 181/489] vcd: ipAllocationMode should be an optional field (#1348) Signed-off-by: Waleed Malik --- pkg/cloudprovider/provider/vmwareclouddirector/types/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go b/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go index c8c5aa3de..188e3c2bd 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go @@ -46,7 +46,7 @@ type RawConfig struct { // Network configuration. Network providerconfigtypes.ConfigVarString `json:"network"` - IPAllocationMode IPAllocationMode `json:"ipAllocationMode"` + IPAllocationMode IPAllocationMode `json:"ipAllocationMode,omitempty"` // Compute configuration. CPUs int64 `json:"cpus"` From 32c8ec0998445a664938d2f7783a9d12168adf7b Mon Sep 17 00:00:00 2001 From: Pratik Deoghare Date: Thu, 14 Jul 2022 09:37:50 +0200 Subject: [PATCH 182/489] set --node-ip depending on external CCM availaibilty in dualstack (#1336) * paramterize kubelet flags template by node-ip * set kubelet --node-ip flag * remove --node-ip flag for openstack in dualstack config Issue: https://github.com/kubermatic/kubermatic/issues/9959 * set --node-ip,--cloud-provider flags for openstack, digitalocean * refactor * . * add comments about --node-ip flag settings in dualstack * rename variables for clarity * remove unspecified case * move docs to func level * . * fix testdata files * remove unnecessary check for --cloud-provider * decide setting -node-ip based on ipFamily, cloudprovider, in-tree-ness * move withNodeIPFlag near CloudProviderFlags * fix comments; simplify code --- hack/run-machine-controller.sh | 2 +- pkg/userdata/amzn2/provider.go | 4 +- pkg/userdata/centos/provider.go | 4 +- pkg/userdata/flatcar/provider.go | 6 +- pkg/userdata/helper/helper.go | 21 +- pkg/userdata/helper/kubelet.go | 64 ++- pkg/userdata/helper/kubelet_test.go | 3 + pkg/userdata/rhel/provider.go | 4 +- pkg/userdata/rockylinux/provider.go | 4 +- pkg/userdata/sles/provider.go | 4 +- pkg/userdata/ubuntu/provider.go | 4 +- pkg/userdata/ubuntu/provider_test.go | 57 +++ .../testdata/digitalocean-dualstack.yaml | 452 ++++++++++++++++++ .../ubuntu/testdata/openstack-dualstack.yaml | 452 ++++++++++++++++++ 14 files changed, 1049 insertions(+), 32 deletions(-) create mode 100644 pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml create mode 100644 pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml diff --git a/hack/run-machine-controller.sh b/hack/run-machine-controller.sh index 69cc68125..192a52c46 100755 --- a/hack/run-machine-controller.sh +++ b/hack/run-machine-controller.sh @@ -29,7 +29,7 @@ $(dirname $0)/../machine-controller \ -worker-count=50 \ -logtostderr \ -v=6 \ - -cluster-dns=172.16.0.10 \ + -cluster-dns=169.254.20.10 \ -enable-profiling \ -metrics-address=0.0.0.0:8080 \ -health-probe-address=0.0.0.0:8085 \ diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index f79ac612c..10e5fb480 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -114,7 +114,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { KubeletVersion: kubeletVersion.String(), Kubeconfig: kubeconfigString, KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), + NodeIPScript: userdatahelper.SetupNodeIPEnvScript(pconfig.Network.GetIPFamily()), ExtraKubeletFlags: crEngine.KubeletFlags(), ContainerRuntimeScript: crScript, ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), @@ -254,7 +254,7 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index 5a96b70de..3943bf0a8 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -114,7 +114,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { KubeletVersion: kubeletVersion.String(), Kubeconfig: kubeconfigString, KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), + NodeIPScript: userdatahelper.SetupNodeIPEnvScript(pconfig.Network.GetIPFamily()), ExtraKubeletFlags: crEngine.KubeletFlags(), ContainerRuntimeScript: crScript, ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), @@ -272,7 +272,7 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index 471ce53ec..183b7e033 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -120,7 +120,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { KubeletVersion: kubeletVersion.String(), Kubeconfig: kubeconfigString, KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), + NodeIPScript: userdatahelper.SetupNodeIPEnvScript(pconfig.Network.GetIPFamily()), ExtraKubeletFlags: crEngine.KubeletFlags(), ContainerRuntimeScript: crScript, ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), @@ -301,7 +301,7 @@ systemd: Requires=download-script.service After=download-script.service contents: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags false | indent 8 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags false | indent 8 }} storage: files: @@ -623,7 +623,7 @@ coreos: Requires=download-script.service After=download-script.service content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags false | indent 6 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags false | indent 6 }} - name: apply-sysctl-settings.service enable: true diff --git a/pkg/userdata/helper/helper.go b/pkg/userdata/helper/helper.go index 291513e20..63155f26c 100644 --- a/pkg/userdata/helper/helper.go +++ b/pkg/userdata/helper/helper.go @@ -21,6 +21,8 @@ import ( "fmt" "strings" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" + "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" ) @@ -146,14 +148,29 @@ NO_PROXY=%s no_proxy=%s`, proxy, proxy, proxy, proxy, noProxy, noProxy) } -func SetupNodeIPEnvScript() string { +func SetupNodeIPEnvScript(ipFamily util.IPFamily) string { + const defaultIfcIPv4 = `DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+")` + + var defaultIfcIP string + switch ipFamily { + case util.IPv4: + defaultIfcIP = defaultIfcIPv4 + case util.IPv6: + defaultIfcIP = `DEFAULT_IFC_IP=$(ip -o -6 route get 1:: | grep -oP "src \K\S+")` + case util.DualStack: + defaultIfcIP = `DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") +DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") +DEFAULT_IFC_IP=$DEFAULT_IFC_IP,$DEFAULT_IFC_IP6` + default: + defaultIfcIP = defaultIfcIPv4 + } return `#!/usr/bin/env bash echodate() { echo "[$(date -Is)]" "$@" } # get the default interface IP address -DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") +` + defaultIfcIP + ` # get the full hostname FULL_HOSTNAME=$(hostname -f) diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index 07f6691da..2a3ad59d5 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -26,6 +26,7 @@ import ( "github.com/Masterminds/semver/v3" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -39,15 +40,18 @@ const ( defaultKubeletContainerLogMaxSize = "100Mi" ) -const ( - kubeletFlagsTpl = `--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ +func kubeletFlagsTpl(withNodeIP bool) string { + flagsTemplate := `--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ ---cert-dir=/etc/kubernetes/pki \ +--cert-dir=/etc/kubernetes/pki \` + + flagsTemplate += ` {{- if or (.CloudProvider) (.IsExternal) }} {{ cloudProviderFlags .CloudProvider .IsExternal }} \ -{{- end }} -{{- if and (.Hostname) (ne .CloudProvider "aws") }} +{{- end }}` + + flagsTemplate += `{{- if and (.Hostname) (ne .CloudProvider "aws") }} --hostname-override={{ .Hostname }} \ {{- else if and (eq .CloudProvider "aws") (.IsExternal) }} --hostname-override=${KUBELET_HOSTNAME} \ @@ -62,9 +66,17 @@ const ( {{- end }} {{- range .ExtraKubeletFlags }} {{ . }} \ -{{- end }} +{{- end }}` + + if withNodeIP { + flagsTemplate += ` --node-ip ${KUBELET_NODE_IP}` + } + return flagsTemplate +} + +const ( kubeletSystemdUnitTpl = `[Unit] After={{ .ContainerRuntime }}.service Requires={{ .ContainerRuntime }}.service @@ -89,7 +101,7 @@ ExecStartPre=/bin/bash /opt/disable-swap.sh {{ end }} ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ -{{ kubeletFlags .KubeletVersion .CloudProvider .Hostname .ClusterDNSIPs .IsExternal .PauseImage .InitialTaints .ExtraKubeletFlags | indent 2 }} +{{ kubeletFlags .KubeletVersion .CloudProvider .Hostname .ClusterDNSIPs .IsExternal .IPFamily .PauseImage .InitialTaints .ExtraKubeletFlags | indent 2 }} [Install] WantedBy=multi-user.target` @@ -123,20 +135,32 @@ var kubeletTLSCipherSuites = []string{ "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", } +func withNodeIPFlag(ipFamily util.IPFamily, cloudProvider string, external bool) bool { + // If external or in-tree CCM is in use we don't need to set --node-ip + // as the cloud provider will know what IPs to return. + if ipFamily == util.DualStack { + if external || cloudProvider != "" { + return false + } + } + return true +} + // CloudProviderFlags returns --cloud-provider and --cloud-config flags. -func CloudProviderFlags(cpName string, external bool) (string, error) { +func CloudProviderFlags(cpName string, external bool) string { if cpName == "" && !external { - return "", nil + return "" } if external { - return "--cloud-provider=external", nil + return `--cloud-provider=external` } - return fmt.Sprintf(cpFlags, cpName), nil + + return fmt.Sprintf(cpFlags, cpName) } // KubeletSystemdUnit returns the systemd unit for the kubelet. -func KubeletSystemdUnit(containerRuntime, kubeletVersion, cloudProvider, hostname string, dnsIPs []net.IP, external bool, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string, disableSwap bool) (string, error) { +func KubeletSystemdUnit(containerRuntime, kubeletVersion, cloudProvider, hostname string, dnsIPs []net.IP, external bool, ipFamily util.IPFamily, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string, disableSwap bool) (string, error) { tmpl, err := template.New("kubelet-systemd-unit").Funcs(TxtFuncMap()).Parse(kubeletSystemdUnitTpl) if err != nil { return "", fmt.Errorf("failed to parse kubelet-systemd-unit template: %w", err) @@ -149,6 +173,7 @@ func KubeletSystemdUnit(containerRuntime, kubeletVersion, cloudProvider, hostnam Hostname string ClusterDNSIPs []net.IP IsExternal bool + IPFamily util.IPFamily PauseImage string InitialTaints []corev1.Taint ExtraKubeletFlags []string @@ -160,6 +185,7 @@ func KubeletSystemdUnit(containerRuntime, kubeletVersion, cloudProvider, hostnam Hostname: hostname, ClusterDNSIPs: dnsIPs, IsExternal: external, + IPFamily: ipFamily, PauseImage: pauseImage, InitialTaints: initialTaints, ExtraKubeletFlags: extraKubeletFlags, @@ -279,8 +305,16 @@ func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGate } // KubeletFlags returns the kubelet flags. -func KubeletFlags(version, cloudProvider, hostname string, dnsIPs []net.IP, external bool, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string) (string, error) { - tmpl, err := template.New("kubelet-flags").Funcs(TxtFuncMap()).Parse(kubeletFlagsTpl) +// --node-ip and --cloud-provider kubelet flags conflict in the dualstack setup. +// In general, it is not expected to need to use --node-ip with external CCMs, +// as the cloud provider is expected to know the correct IPs to return. +// For details read kubernetes/sig-networking channel discussion +// https://kubernetes.slack.com/archives/C09QYUH5W/p1654003958331739 +func KubeletFlags(version, cloudProvider, hostname string, dnsIPs []net.IP, external bool, ipFamily util.IPFamily, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string) (string, error) { + withNodeIPFlag := withNodeIPFlag(ipFamily, cloudProvider, external) + + tmpl, err := template.New("kubelet-flags").Funcs(TxtFuncMap()). + Parse(kubeletFlagsTpl(withNodeIPFlag)) if err != nil { return "", fmt.Errorf("failed to parse kubelet-flags template: %w", err) } @@ -328,6 +362,7 @@ func KubeletFlags(version, cloudProvider, hostname string, dnsIPs []net.IP, exte ClusterDNSIPs []net.IP KubeletVersion string IsExternal bool + IPFamily util.IPFamily PauseImage string InitialTaints string ExtraKubeletFlags []string @@ -337,6 +372,7 @@ func KubeletFlags(version, cloudProvider, hostname string, dnsIPs []net.IP, exte ClusterDNSIPs: dnsIPs, KubeletVersion: version, IsExternal: external, + IPFamily: ipFamily, PauseImage: pauseImage, InitialTaints: strings.Join(initialTaintsArgs, ","), ExtraKubeletFlags: kubeletFlags, diff --git a/pkg/userdata/helper/kubelet_test.go b/pkg/userdata/helper/kubelet_test.go index 5b7326a1c..18b02effc 100644 --- a/pkg/userdata/helper/kubelet_test.go +++ b/pkg/userdata/helper/kubelet_test.go @@ -23,6 +23,7 @@ import ( "github.com/Masterminds/semver/v3" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" testhelper "github.com/kubermatic/machine-controller/pkg/test" corev1 "k8s.io/api/core/v1" @@ -36,6 +37,7 @@ type kubeletFlagTestCase struct { hostname string cloudProvider string external bool + ipFamily util.IPFamily pauseImage string initialTaints []corev1.Taint extraFlags []string @@ -117,6 +119,7 @@ func TestKubeletSystemdUnit(t *testing.T) { test.hostname, test.dnsIPs, test.external, + test.ipFamily, test.pauseImage, test.initialTaints, test.extraFlags, diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 75efa5d45..46f3b3046 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -114,7 +114,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { KubeletVersion: kubeletVersion.String(), Kubeconfig: kubeconfigString, KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), + NodeIPScript: userdatahelper.SetupNodeIPEnvScript(pconfig.Network.GetIPFamily()), ExtraKubeletFlags: crEngine.KubeletFlags(), ContainerRuntimeScript: crScript, ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), @@ -273,7 +273,7 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" diff --git a/pkg/userdata/rockylinux/provider.go b/pkg/userdata/rockylinux/provider.go index 450bde849..930984309 100644 --- a/pkg/userdata/rockylinux/provider.go +++ b/pkg/userdata/rockylinux/provider.go @@ -114,7 +114,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { KubeletVersion: kubeletVersion.String(), Kubeconfig: kubeconfigString, KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), + NodeIPScript: userdatahelper.SetupNodeIPEnvScript(pconfig.Network.GetIPFamily()), ExtraKubeletFlags: crEngine.KubeletFlags(), ContainerRuntimeScript: crScript, ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), @@ -264,7 +264,7 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" diff --git a/pkg/userdata/sles/provider.go b/pkg/userdata/sles/provider.go index 7a7130d80..41fbf06a5 100644 --- a/pkg/userdata/sles/provider.go +++ b/pkg/userdata/sles/provider.go @@ -108,7 +108,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { KubeletVersion: kubeletVersion.String(), Kubeconfig: kubeconfigString, KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), + NodeIPScript: userdatahelper.SetupNodeIPEnvScript(pconfig.Network.GetIPFamily()), ExtraKubeletFlags: crEngine.KubeletFlags(), ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), ContainerRuntimeConfig: crConfig, @@ -216,7 +216,7 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - path: "/etc/systemd/system/kubelet.service.d/extras.conf" content: | diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index 4fb0920dd..5a83a8a1b 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -114,7 +114,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { KubeletVersion: kubeletVersion.String(), Kubeconfig: kubeconfigString, KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(), + NodeIPScript: userdatahelper.SetupNodeIPEnvScript(pconfig.Network.GetIPFamily()), ExtraKubeletFlags: crEngine.KubeletFlags(), ContainerRuntimeScript: crScript, ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), @@ -263,7 +263,7 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} +{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - path: "/etc/systemd/system/kubelet.service.d/extras.conf" content: | diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 8f0ebd10b..14c230c61 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -31,6 +31,7 @@ import ( clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/apis/plugin" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/containerruntime" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" testhelper "github.com/kubermatic/machine-controller/pkg/test" @@ -296,6 +297,62 @@ func TestUserDataGeneration(t *testing.T) { DistUpgradeOnBoot: false, }, }, + { + name: "openstack-dualstack", + providerSpec: &providerconfigtypes.Config{ + CloudProvider: "openstack", + SSHPublicKeys: []string{"ssh-rsa AAABBB"}, + Network: &providerconfigtypes.NetworkConfig{ + IPFamily: util.DualStack, + }, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: defaultVersion, + }, + }, + ccProvider: &fakeCloudConfigProvider{ + name: "openstack", + config: "{openstack-config:true}", + err: nil, + }, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, + kubernetesCACert: "CACert", + osConfig: &Config{ + DistUpgradeOnBoot: false, + }, + externalCloudProvider: true, + }, + { + name: "digitalocean-dualstack", + providerSpec: &providerconfigtypes.Config{ + CloudProvider: "digitalocean", + SSHPublicKeys: []string{"ssh-rsa AAABBB"}, + Network: &providerconfigtypes.NetworkConfig{ + IPFamily: util.DualStack, + }, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: defaultVersion, + }, + }, + ccProvider: &fakeCloudConfigProvider{ + config: "{digitalocean-config:true}", + err: nil, + }, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, + kubernetesCACert: "CACert", + osConfig: &Config{ + DistUpgradeOnBoot: false, + }, + }, { name: "openstack-overwrite-cloud-config", providerSpec: &providerconfigtypes.Config{ diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml new file mode 100644 index 000000000..d76ac8ad9 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml @@ -0,0 +1,452 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {digitalocean-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") + DEFAULT_IFC_IP=$DEFAULT_IFC_IP,$DEFAULT_IFC_IP6 + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + - 10.10.10.11 + - 10.10.10.12 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml new file mode 100644 index 000000000..77904a297 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml @@ -0,0 +1,452 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {openstack-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") + DEFAULT_IFC_IP=$DEFAULT_IFC_IP,$DEFAULT_IFC_IP6 + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + - 10.10.10.11 + - 10.10.10.12 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service From 8cb7e09b74b743cc5aab3eaf83e53b6bc8d6221f Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Mon, 18 Jul 2022 17:09:17 +0200 Subject: [PATCH 183/489] Rename vSphere os image templates (#1364) * rename vSphere os image templates Signed-off-by: Moath Qasim * use vsphere legacy preset Signed-off-by: Moath Qasim --- .prow/provider-vsphere.yaml | 2 +- test/e2e/provisioning/helper.go | 11 +++++++++++ .../machinedeployment-vsphere-datastore-cluster.yaml | 2 +- .../machinedeployment-vsphere-resource-pool.yaml | 3 +-- .../testdata/machinedeployment-vsphere-static-ip.yaml | 8 ++++---- .../testdata/machinedeployment-vsphere.yaml | 2 +- 6 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index ece985542..104d807c8 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -20,7 +20,7 @@ presubmits: labels: preset-hetzner: "true" preset-e2e-ssh: "true" - preset-vsphere: "true" + preset-vsphere-legacy: "true" preset-rhel: "true" preset-goproxy: "true" preset-kind-volume-mounts: "true" diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 532b0abce..25a2b747c 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -55,6 +55,14 @@ var ( string(providerconfigtypes.OperatingSystemFlatcar): "machine-controller-e2e-flatcar-stable-2983", string(providerconfigtypes.OperatingSystemRockyLinux): "machine-controller-e2e-rockylinux", } + + vSphereOSImageTemplates = map[string]string{ + string(providerconfigtypes.OperatingSystemCentOS): "machine-controller-e2e-centos", + string(providerconfigtypes.OperatingSystemFlatcar): "machine-controller-e2e-flatcar", + string(providerconfigtypes.OperatingSystemRHEL): "machine-controller-e2e-rhel", + string(providerconfigtypes.OperatingSystemRockyLinux): "machine-controller-e2e-rockylinux", + string(providerconfigtypes.OperatingSystemUbuntu): "machine-controller-e2e-ubuntu", + } ) type scenario struct { @@ -226,6 +234,9 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar // only used by OpenStack scenarios scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_IMAGE >>=%s", openStackImages[testCase.osName])) + // only use by vSphere scenarios + scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_Image_Template >>=%s", vSphereOSImageTemplates[testCase.osName])) + // default kubeconfig to the hardcoded path at which `make e2e-cluster` creates its new kubeconfig gopath := os.Getenv("GOPATH") projectDir := filepath.Join(gopath, "src/github.com/kubermatic/machine-controller") diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml index e22c05001..3242f90e0 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml @@ -24,7 +24,7 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "vsphere" cloudProviderSpec: - templateVMName: 'machine-controller-e2e-<< OS_NAME >>' + templateVMName: '<< OS_Image_Template >>' username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' datacenter: 'dc-1' diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml index c2b42657f..a54021366 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml @@ -24,13 +24,12 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "vsphere" cloudProviderSpec: - templateVMName: 'machine-controller-e2e-<< OS_NAME >>' + templateVMName: '<< OS_Image_Template >>' username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' datacenter: 'dc-1' folder: '/dc-1/vm/e2e-tests' password: << VSPHERE_PASSWORD >> - # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically datastoreCluster: 'dsc-1' resourcePool: 'e2e-resource-pool' cpus: 2 diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml index e46cafe48..eac0bdcfa 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml @@ -24,14 +24,14 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "vsphere" cloudProviderSpec: - templateVMName: '<< OS_NAME >>-template' + templateVMName: '<< OS_Image_Template >>' username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' - datacenter: 'Customer-A' - folder: '/Customer-A/vm/e2e-tests' + datacenter: 'dc-1' + folder: '/dc-1/vm/e2e-tests' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically - datastore: datastore1 + datastore: HS-FreeNAS allowInsecure: true cpus: 2 MemoryMB: 2048 diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml index 49101efc5..921d00669 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml @@ -24,7 +24,7 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "vsphere" cloudProviderSpec: - templateVMName: 'machine-controller-e2e-<< OS_NAME >>' + templateVMName: '<< OS_Image_Template >>' username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' datacenter: 'dc-1' From 6630c09bdfda78f792b4a7360bd68d355b4e25ee Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Mon, 18 Jul 2022 22:30:11 +0200 Subject: [PATCH 184/489] disable azure tests temporarily (#1365) Signed-off-by: Moath Qasim --- test/e2e/provisioning/all_e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 6c3a5f9e5..2626809f1 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -610,7 +610,7 @@ func TestAzureProvisioningE2E(t *testing.T) { t.Fatal("unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") } - selector := Not(OsSelector("sles", "amzn2")) + selector := Not(OsSelector("sles", "amzn2", "rhel", "rockylinux", "flatcar")) // act params := []string{ fmt.Sprintf("<< AZURE_TENANT_ID >>=%s", azureTenantID), From 47275727a1861c5090089e2f401c78e81a232f13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Tue, 19 Jul 2022 08:20:56 +0200 Subject: [PATCH 185/489] Equinix: use m3.small.x86 for CentOS E2E tests (#1363) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- test/e2e/provisioning/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 25a2b747c..cefad450f 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -213,7 +213,7 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar if strings.Contains(cloudProvider, string(providerconfigtypes.CloudProviderEquinixMetal)) { switch testCase.osName { case string(providerconfigtypes.OperatingSystemCentOS): - scenarioParams = append(scenarioParams, fmt.Sprintf("<< INSTANCE_TYPE >>=%s", "c3.small.x86")) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< INSTANCE_TYPE >>=%s", "m3.small.x86")) scenarioParams = append(scenarioParams, fmt.Sprintf("<< METRO_CODE >>=%s", "AM")) case string(providerconfigtypes.OperatingSystemFlatcar): scenarioParams = append(scenarioParams, fmt.Sprintf("<< INSTANCE_TYPE >>=%s", "c3.small.x86")) From 40a5eb90958f0723d018b8ca411a597e4468cf13 Mon Sep 17 00:00:00 2001 From: Rastislav Szabo Date: Wed, 20 Jul 2022 15:25:29 +0200 Subject: [PATCH 186/489] Add IPFamily options in vSphere cloud config API (#1366) * Add IPFamily options in vSphere cloudconfig API Signed-off-by: Rastislav Szabo * Make ip-family optional in vsphere cloud-config Signed-off-by: Rastislav Szabo --- .../provider/vsphere/types/cloudconfig.go | 8 +++++ .../vsphere/types/cloudconfig_test.go | 30 +++++++++++++++++++ .../types/testdata/3-dual-stack.golden | 29 ++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 pkg/cloudprovider/provider/vsphere/types/testdata/3-dual-stack.golden diff --git a/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go b/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go index 073262a8b..9e9f997bf 100644 --- a/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go +++ b/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go @@ -36,6 +36,9 @@ working-dir = {{ .Global.WorkingDir | iniEscape }} datacenter = {{ .Global.Datacenter | iniEscape }} datastore = {{ .Global.DefaultDatastore | iniEscape }} server = {{ .Global.VCenterIP | iniEscape }} +{{- if .Global.IPFamily }} +ip-family = {{ .Global.IPFamily | iniEscape }} +{{- end }} [Disk] scsicontrollertype = {{ .Disk.SCSIControllerType | iniEscape }} @@ -53,6 +56,9 @@ user = {{ $vc.User | iniEscape }} password = {{ $vc.Password | iniEscape }} port = {{ $vc.VCenterPort }} datacenters = {{ $vc.Datacenters | iniEscape }} +{{- if $vc.IPFamily }} +ip-family = {{ $vc.IPFamily | iniEscape }} +{{- end }} {{ end }} ` ) @@ -79,6 +85,7 @@ type GlobalOpts struct { DefaultDatastore string `gcfg:"datastore"` VCenterIP string `gcfg:"server"` ClusterID string `gcfg:"cluster-id"` + IPFamily string `gcfg:"ip-family"` // NOTE: supported only in case of out-of-tree CCM } type VirtualCenterConfig struct { @@ -86,6 +93,7 @@ type VirtualCenterConfig struct { Password string `gcfg:"password"` VCenterPort string `gcfg:"port"` Datacenters string `gcfg:"datacenters"` + IPFamily string `gcfg:"ip-family"` // NOTE: supported only in case of out-of-tree CCM } // CloudConfig is used to read and store information from the cloud configuration file. diff --git a/pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go b/pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go index f63c60cbd..399f31231 100644 --- a/pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go +++ b/pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go @@ -87,6 +87,36 @@ func TestCloudConfigToString(t *testing.T) { }, }, }, + { + name: "3-dual-stack", + config: &CloudConfig{ + Global: GlobalOpts{ + User: "admin", + Password: "password", + InsecureFlag: true, + IPFamily: "ipv4,ipv6", + }, + Workspace: WorkspaceOpts{ + VCenterIP: "https://127.0.0.1:8443/", + ResourcePoolPath: "/some-resource-pool", + DefaultDatastore: "Datastore", + Folder: "some-folder", + Datacenter: "Datacenter", + }, + Disk: DiskOpts{ + SCSIControllerType: "pvscsi", + }, + VirtualCenter: map[string]*VirtualCenterConfig{ + "vc1": { + User: "1-some-user", + Password: "1-some-password", + VCenterPort: "443", + Datacenters: "1-foo", + IPFamily: "ipv4,ipv6", + }, + }, + }, + }, } for _, test := range tests { diff --git a/pkg/cloudprovider/provider/vsphere/types/testdata/3-dual-stack.golden b/pkg/cloudprovider/provider/vsphere/types/testdata/3-dual-stack.golden new file mode 100644 index 000000000..88343530b --- /dev/null +++ b/pkg/cloudprovider/provider/vsphere/types/testdata/3-dual-stack.golden @@ -0,0 +1,29 @@ +[Global] +user = "admin" +password = "password" +port = "" +insecure-flag = true +working-dir = "" +datacenter = "" +datastore = "" +server = "" +ip-family = "ipv4,ipv6" + +[Disk] +scsicontrollertype = "pvscsi" + +[Workspace] +server = "https://127.0.0.1:8443/" +datacenter = "Datacenter" +folder = "some-folder" +default-datastore = "Datastore" +resourcepool-path = "/some-resource-pool" + + +[VirtualCenter "vc1"] +user = "1-some-user" +password = "1-some-password" +port = 443 +datacenters = "1-foo" +ip-family = "ipv4,ipv6" + From 86c60d725de5440977ac5a69159b807cd1c92886 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 21 Jul 2022 15:44:27 +0500 Subject: [PATCH 187/489] Fix unmarshalling for configvarstring (#1367) Signed-off-by: Waleed Malik --- pkg/providerconfig/types/types.go | 12 +++++++++++- pkg/providerconfig/types/types_test.go | 8 +++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index b9ddbbcac..b4e12ad73 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + "strconv" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" @@ -226,7 +227,16 @@ func (configVarString *ConfigVarString) UnmarshalJSON(b []byte) error { if !bytes.HasPrefix(b, []byte("{")) { b = bytes.TrimPrefix(b, []byte(`"`)) b = bytes.TrimSuffix(b, []byte(`"`)) - configVarString.Value = string(b) + + // `Unquote` expects the input string to be inside quotation marks. + // Since we can have a string without any quotations, in which case `TrimPrefix` and + // `TrimSuffix` will be noop. We explicitly add quotation marks to the input string + // to make sure that `Unquote` never fails. + s, err := strconv.Unquote("\"" + string(b) + "\"") + if err != nil { + return err + } + configVarString.Value = s return nil } // This type must have the same fields as ConfigVarString but not diff --git a/pkg/providerconfig/types/types_test.go b/pkg/providerconfig/types/types_test.go index 7b0d8601a..d1d7252bd 100644 --- a/pkg/providerconfig/types/types_test.go +++ b/pkg/providerconfig/types/types_test.go @@ -21,7 +21,7 @@ import ( "reflect" "testing" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" ) @@ -160,6 +160,7 @@ func TestConfigVarBoolMarshalling(t *testing.T) { func TestConfigVarStringMarshallingAndUnmarshalling(t *testing.T) { testCases := []ConfigVarString{ {Value: "val"}, + {Value: "spe Date: Fri, 22 Jul 2022 20:33:33 +0200 Subject: [PATCH 188/489] Fix logic for finding node by ProviderID (#1351) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix logic for finding node by ProviderID Signed-off-by: Marko Mudrinić * Extend the Instance interface with ProviderID() function Signed-off-by: Marko Mudrinić * Implement ProviderID function for cloud providers Signed-off-by: Marko Mudrinić * Add more unit tests for getNode and fix linter errors Signed-off-by: Marko Mudrinić * Retry if instance doesn't have IP addresses Signed-off-by: Marko Mudrinić --- pkg/cloudprovider/instance/instance.go | 2 + .../provider/alibaba/provider.go | 5 ++ pkg/cloudprovider/provider/anexia/instance.go | 5 ++ pkg/cloudprovider/provider/aws/provider.go | 11 +++ pkg/cloudprovider/provider/azure/provider.go | 8 ++ .../provider/baremetal/provider.go | 5 ++ .../provider/digitalocean/provider.go | 4 + .../provider/equinixmetal/provider.go | 4 + pkg/cloudprovider/provider/fake/provider.go | 4 + pkg/cloudprovider/provider/gce/instance.go | 4 + .../provider/hetzner/provider.go | 4 + .../provider/kubevirt/provider.go | 4 + pkg/cloudprovider/provider/linode/provider.go | 5 ++ .../provider/nutanix/provider.go | 5 ++ .../provider/openstack/provider.go | 4 + .../provider/scaleway/provider.go | 5 ++ .../provider/vmwareclouddirector/provider.go | 5 ++ .../provider/vsphere/provider.go | 11 ++- pkg/controller/machine/machine_controller.go | 60 +++++++++++--- pkg/controller/machine/machine_test.go | 79 +++++++++++++++---- 20 files changed, 206 insertions(+), 28 deletions(-) diff --git a/pkg/cloudprovider/instance/instance.go b/pkg/cloudprovider/instance/instance.go index 64299ada4..f97c327a9 100644 --- a/pkg/cloudprovider/instance/instance.go +++ b/pkg/cloudprovider/instance/instance.go @@ -24,6 +24,8 @@ type Instance interface { Name() string // ID returns the instance identifier. ID() string + // ProviderID returns the expected providerID for the instance + ProviderID() string // Addresses returns a list of addresses associated with the instance. Addresses() map[string]v1.NodeAddressType // Status returns the instance status. diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index 64cd3339a..3b4f773f9 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -86,6 +86,11 @@ func (a *alibabaInstance) ID() string { return a.instance.InstanceId } +// TODO: Implement once we start supporting Alibaba CCM. +func (a *alibabaInstance) ProviderID() string { + return "" +} + func (a *alibabaInstance) Addresses() map[string]v1.NodeAddressType { primaryIPAddresses := map[string]v1.NodeAddressType{} for _, networkInterface := range a.instance.NetworkInterfaces.NetworkInterface { diff --git a/pkg/cloudprovider/provider/anexia/instance.go b/pkg/cloudprovider/provider/anexia/instance.go index 80c404d38..c8cf6400c 100644 --- a/pkg/cloudprovider/provider/anexia/instance.go +++ b/pkg/cloudprovider/provider/anexia/instance.go @@ -45,6 +45,11 @@ func (ai *anexiaInstance) ID() string { return ai.info.Identifier } +// TODO(xmudrii): Implement this. +func (ai *anexiaInstance) ProviderID() string { + return "" +} + func (ai *anexiaInstance) Addresses() map[string]v1.NodeAddressType { addresses := map[string]v1.NodeAddressType{} diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index a6436caef..bc81d8bea 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -1004,6 +1004,17 @@ func (d *awsInstance) ID() string { return aws.StringValue(d.instance.InstanceId) } +func (d *awsInstance) ProviderID() string { + if d.instance.InstanceId == nil { + return "" + } + if d.instance.Placement.AvailabilityZone == nil { + return "aws:///" + *d.instance.InstanceId + } + + return "aws:///" + *d.instance.Placement.AvailabilityZone + "/" + *d.instance.InstanceId +} + func (d *awsInstance) Addresses() map[string]v1.NodeAddressType { addresses := map[string]v1.NodeAddressType{ aws.StringValue(d.instance.PublicIpAddress): v1.NodeExternalIP, diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 667c85fbd..ecf60fc54 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -124,6 +124,14 @@ func (vm *azureVM) Name() string { return *vm.vm.Name } +func (vm *azureVM) ProviderID() string { + if vm.vm.ID == nil { + return "" + } + + return "azure://" + *vm.vm.ID +} + func (vm *azureVM) Status() instance.Status { return vm.status } diff --git a/pkg/cloudprovider/provider/baremetal/provider.go b/pkg/cloudprovider/provider/baremetal/provider.go index e62e757ff..1203cb786 100644 --- a/pkg/cloudprovider/provider/baremetal/provider.go +++ b/pkg/cloudprovider/provider/baremetal/provider.go @@ -53,6 +53,11 @@ func (b bareMetalServer) ID() string { return b.server.GetID() } +// TODO: Tinkerbell doesn't have a CCM. +func (b bareMetalServer) ProviderID() string { + return "" +} + func (b bareMetalServer) Addresses() map[string]corev1.NodeAddressType { return map[string]corev1.NodeAddressType{ b.server.GetIPAddress(): corev1.NodeInternalIP, diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index 1c5ba1f58..fbb121f6f 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -503,6 +503,10 @@ func (d *doInstance) ID() string { return strconv.Itoa(d.droplet.ID) } +func (d *doInstance) ProviderID() string { + return fmt.Sprintf("digitalocean://%d", d.droplet.ID) +} + func (d *doInstance) Addresses() map[string]v1.NodeAddressType { addresses := map[string]v1.NodeAddressType{} for _, n := range d.droplet.Networks.V4 { diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index 13de4e467..0fd67b6c8 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -399,6 +399,10 @@ func (s *metalDevice) ID() string { return s.device.ID } +func (s *metalDevice) ProviderID() string { + return "equinixmetal://" + s.device.ID +} + func (s *metalDevice) Addresses() map[string]v1.NodeAddressType { // returns addresses in CIDR format addresses := map[string]v1.NodeAddressType{} diff --git a/pkg/cloudprovider/provider/fake/provider.go b/pkg/cloudprovider/provider/fake/provider.go index b3d116d59..c1d0c7c56 100644 --- a/pkg/cloudprovider/provider/fake/provider.go +++ b/pkg/cloudprovider/provider/fake/provider.go @@ -48,6 +48,10 @@ func (f CloudProviderInstance) ID() string { return "" } +func (f CloudProviderInstance) ProviderID() string { + return "" +} + func (f CloudProviderInstance) Addresses() map[string]corev1.NodeAddressType { return nil } diff --git a/pkg/cloudprovider/provider/gce/instance.go b/pkg/cloudprovider/provider/gce/instance.go index f53967ae4..1d61d4bae 100644 --- a/pkg/cloudprovider/provider/gce/instance.go +++ b/pkg/cloudprovider/provider/gce/instance.go @@ -60,6 +60,10 @@ func (gi *googleInstance) ID() string { return strconv.FormatUint(gi.ci.Id, 10) } +func (gi *googleInstance) ProviderID() string { + return fmt.Sprintf("gce://%s/%s/%s", gi.projectID, gi.zone, gi.ci.Name) +} + // Addresses implements instance.Instance. func (gi *googleInstance) Addresses() map[string]v1.NodeAddressType { addrs := map[string]v1.NodeAddressType{} diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 076514949..81dcb476d 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -533,6 +533,10 @@ func (s *hetznerServer) ID() string { return strconv.Itoa(s.server.ID) } +func (s *hetznerServer) ProviderID() string { + return fmt.Sprintf("hcloud://%d", s.server.ID) +} + func (s *hetznerServer) Addresses() map[string]v1.NodeAddressType { addresses := map[string]v1.NodeAddressType{} for _, fips := range s.server.PublicNet.FloatingIPs { diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 030fce973..27175f320 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -161,6 +161,10 @@ func (k *kubeVirtServer) ID() string { return string(k.vmi.UID) } +func (k *kubeVirtServer) ProviderID() string { + return "kubevirt://" + k.vmi.Name +} + func (k *kubeVirtServer) Addresses() map[string]corev1.NodeAddressType { addresses := map[string]corev1.NodeAddressType{} for _, kvInterface := range k.vmi.Status.Interfaces { diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index 2738c7d01..030fe4585 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -403,6 +403,11 @@ func (d *linodeInstance) ID() string { return strconv.Itoa(d.linode.ID) } +// TODO: Implement once we start supporting Linode CCM. +func (d *linodeInstance) ProviderID() string { + return "" +} + func (d *linodeInstance) Addresses() map[string]v1.NodeAddressType { addresses := map[string]v1.NodeAddressType{} for _, n := range d.linode.IPv4 { diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 697a7c363..70c2b701e 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -85,6 +85,11 @@ func (nutanixServer Server) ID() string { return nutanixServer.id } +// NB: Nutanix doesn't have a CCM. +func (nutanixServer Server) ProviderID() string { + return "" +} + func (nutanixServer Server) Addresses() map[string]corev1.NodeAddressType { return nutanixServer.addresses } diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 8b591c1c6..705b5f724 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -934,6 +934,10 @@ func (d *osInstance) ID() string { return d.server.ID } +func (d *osInstance) ProviderID() string { + return "openstack:///" + d.server.ID +} + func (d *osInstance) Addresses() map[string]corev1.NodeAddressType { addresses := map[string]corev1.NodeAddressType{} for _, networkAddresses := range d.server.Addresses { diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go index cb69420ce..1066f5adb 100644 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ b/pkg/cloudprovider/provider/scaleway/provider.go @@ -379,6 +379,11 @@ func (s *scwServer) ID() string { return s.server.ID } +// TODO: Implement once we start supporting Scaleway CCM. +func (s *scwServer) ProviderID() string { + return "" +} + func (s *scwServer) Addresses() map[string]corev1.NodeAddressType { addresses := map[string]corev1.NodeAddressType{} if s.server.PrivateIP != nil { diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go index 2414c1a39..473bf4329 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -120,6 +120,11 @@ func (s Server) ID() string { return s.id } +// TODO: Implement once we start supporting vCloud Director CCM. +func (s Server) ProviderID() string { + return "" +} + func (s Server) Addresses() map[string]corev1.NodeAddressType { return s.addresses } diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 1d47efeb7..a16f58b62 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -82,6 +82,7 @@ var _ instance.Instance = &Server{} type Server struct { name string id string + uuid string status instance.Status addresses map[string]corev1.NodeAddressType } @@ -94,6 +95,10 @@ func (vsphereServer Server) ID() string { return vsphereServer.id } +func (vsphereServer Server) ProviderID() string { + return "vsphere://" + vsphereServer.uuid +} + func (vsphereServer Server) Addresses() map[string]corev1.NodeAddressType { return vsphereServer.addresses } @@ -372,7 +377,7 @@ func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, fmt.Errorf("error when waiting for vm powerOn task: %w", err) } - return Server{name: virtualMachine.Name(), status: instance.StatusRunning, id: virtualMachine.Reference().Value}, nil + return Server{name: virtualMachine.Name(), status: instance.StatusRunning, id: virtualMachine.Reference().Value, uuid: virtualMachine.UUID(ctx)}, nil } func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { @@ -502,7 +507,7 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, da } // We must return here because the vendored code for determining if the guest // utils are running yields an NPD when using with an instance that is not running - return Server{name: virtualMachine.Name(), status: instance.StatusUnknown}, nil + return Server{name: virtualMachine.Name(), status: instance.StatusUnknown, uuid: virtualMachine.UUID(ctx)}, nil } // virtualMachine.IsToolsRunning panics when executed on a VM that is not powered on @@ -530,7 +535,7 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, da klog.V(3).Infof("Can't fetch the IP addresses for machine %s, the VMware guest utils are not running yet. This might take a few minutes", machine.Spec.Name) } - return Server{name: virtualMachine.Name(), status: instance.StatusRunning, addresses: addresses, id: virtualMachine.Reference().Value}, nil + return Server{name: virtualMachine.Name(), status: instance.StatusRunning, addresses: addresses, id: virtualMachine.Reference().Value, uuid: virtualMachine.UUID(ctx)}, nil } func (p *provider) MigrateUID(_ context.Context, _ *clusterv1alpha1.Machine, _ ktypes.UID) error { diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 39f4065e4..9bd7472a4 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -903,6 +903,23 @@ func (r *Reconciler) ensureInstanceExistsForMachine( addresses := providerInstance.Addresses() eventMessage := fmt.Sprintf("Found instance at cloud provider, addresses: %v", addresses) r.recorder.Event(machine, corev1.EventTypeNormal, "InstanceFound", eventMessage) + // It might happen that we got here, but we still don't have IP addresses + // for the instance. In that case it doesn't make sense to proceed because: + // * if we match Node by ProviderID, Machine will get NodeOwnerRef, but + // there will be no IP address on that Machine object. Since we + // successfully set NodeOwnerRef, Machine will not be reconciled again, + // so it will never get IP addresses. This breaks the NodeCSRApprover + // workflow because NodeCSRApprover cannot validate certificates without + // IP addresses, resulting in a broken Node + // * if we can't match Node by ProviderID, fallback to matching by IP + // address will not have any result because we still don't have IP + // addresses for that instance + // Considering that, we just retry after 15 seconds, hoping that we'll + // get IP addresses by then. + if len(addresses) == 0 { + return &reconcile.Result{RequeueAfter: 15 * time.Second}, nil + } + machineAddresses := []corev1.NodeAddress{} for address, addressType := range addresses { machineAddresses = append(machineAddresses, corev1.NodeAddress{Address: address, Type: addressType}) @@ -1065,19 +1082,13 @@ func (r *Reconciler) getNode(ctx context.Context, instance instance.Instance, pr return nil, false, err } - // We trim leading slashes in raw ID, since we always want three slashes in full ID - providerID := fmt.Sprintf("%s:///%s", provider, strings.TrimLeft(instance.ID(), "/")) for _, node := range nodes.Items { - if provider == providerconfigtypes.CloudProviderAzure { - // Azure IDs are case-insensitive - if strings.EqualFold(node.Spec.ProviderID, providerID) { - return node.DeepCopy(), true, nil - } - } else { - if node.Spec.ProviderID == providerID { - return node.DeepCopy(), true, nil - } + // Try to find Node by providerID. Should work if CCM is deployed. + if node := findNodeByProviderID(instance, provider, nodes.Items); node != nil { + klog.V(4).Infof("Found node %q by providerID", node.Name) + return node, true, nil } + // If we were unable to find Node by ProviderID, fallback to IP address matching. // This usually happens if there's no CCM deployed in the cluster. // @@ -1104,6 +1115,7 @@ func (r *Reconciler) getNode(ctx context.Context, instance instance.Instance, pr continue } if nodeAddress.Address == instanceAddress { + klog.V(4).Infof("Found node %q by IP address", node.Name) return node.DeepCopy(), true, nil } } @@ -1112,6 +1124,32 @@ func (r *Reconciler) getNode(ctx context.Context, instance instance.Instance, pr return nil, false, nil } +func findNodeByProviderID(instance instance.Instance, provider providerconfigtypes.CloudProvider, nodes []corev1.Node) *corev1.Node { + providerID := instance.ProviderID() + if providerID == "" { + return nil + } + + for _, node := range nodes { + if strings.EqualFold(node.Spec.ProviderID, providerID) { + return node.DeepCopy() + } + + // AWS has two different providerID notations: + // * aws://// + // * aws:/// + // The first case is handled above, while the second here is handled here. + if provider == providerconfigtypes.CloudProviderAWS { + pid := strings.Split(node.Spec.ProviderID, "aws:///") + if len(pid) == 2 && pid[1] == instance.ID() { + return node.DeepCopy() + } + } + } + + return nil +} + func (r *Reconciler) ReadinessChecks(ctx context.Context) map[string]healthcheck.Check { return map[string]healthcheck.Check{ "valid-info-kubeconfig": func() error { diff --git a/pkg/controller/machine/machine_test.go b/pkg/controller/machine/machine_test.go index d528ec4f6..c24d5ae32 100644 --- a/pkg/controller/machine/machine_test.go +++ b/pkg/controller/machine/machine_test.go @@ -48,10 +48,11 @@ func init() { } type fakeInstance struct { - name string - id string - addresses map[string]corev1.NodeAddressType - status instance.Status + name string + id string + providerID string + addresses map[string]corev1.NodeAddressType + status instance.Status } func (i *fakeInstance) Name() string { @@ -62,6 +63,10 @@ func (i *fakeInstance) ID() string { return i.id } +func (i *fakeInstance) ProviderID() string { + return i.providerID +} + func (i *fakeInstance) Status() instance.Status { return i.status } @@ -70,11 +75,7 @@ func (i *fakeInstance) Addresses() map[string]corev1.NodeAddressType { return i.addresses } -func getTestNode(id, provider string) corev1.Node { - providerID := "" - if provider != "" { - providerID = fmt.Sprintf("%s:///%s", provider, id) - } +func getTestNode(id, providerID string) corev1.Node { return corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("node%s", id), @@ -98,10 +99,10 @@ func getTestNode(id, provider string) corev1.Node { } func TestController_GetNode(t *testing.T) { - node1 := getTestNode("1", "aws") - node2 := getTestNode("2", "openstack") + node1 := getTestNode("1", "aws:///i-1") + node2 := getTestNode("2", "openstack:///test") node3 := getTestNode("3", "") - node4 := getTestNode("4", "hetzner") + node4 := getTestNode("4", "hcloud://123") nodeList := []*corev1.Node{&node1, &node2, &node3, &node4} tests := []struct { @@ -134,7 +135,7 @@ func TestController_GetNode(t *testing.T) { resNode: &node1, exists: true, err: nil, - instance: &fakeInstance{id: "1", addresses: map[string]corev1.NodeAddressType{"": ""}}, + instance: &fakeInstance{id: "1", addresses: map[string]corev1.NodeAddressType{"": ""}, providerID: "aws:///i-1"}, }, { name: "node found by internal ip", @@ -182,7 +183,7 @@ func TestController_GetNode(t *testing.T) { resNode: &node4, exists: true, err: nil, - instance: &fakeInstance{id: "4", addresses: map[string]corev1.NodeAddressType{"": ""}}, + instance: &fakeInstance{id: "4", addresses: map[string]corev1.NodeAddressType{"": ""}, providerID: "hcloud://123"}, }, } @@ -647,3 +648,53 @@ func TestControllerDeleteNodeForMachine(t *testing.T) { }) } } + +func TestControllerFindNodeByProviderID(t *testing.T) { + tests := []struct { + name string + instance instance.Instance + provider providerconfigtypes.CloudProvider + nodes []corev1.Node + expectedNode bool + }{ + { + name: "aws providerID type 1", + instance: &fakeInstance{id: "99", providerID: "aws:///some-zone/i-99"}, + provider: providerconfigtypes.CloudProviderAWS, + nodes: []corev1.Node{ + getTestNode("1", "random"), + getTestNode("2", "aws:///some-zone/i-99"), + }, + expectedNode: true, + }, + { + name: "aws providerID type 2", + instance: &fakeInstance{id: "99", providerID: "aws:///i-99"}, + provider: providerconfigtypes.CloudProviderAWS, + nodes: []corev1.Node{ + getTestNode("1", "aws:///i-99"), + getTestNode("2", "random"), + }, + expectedNode: true, + }, + { + name: "azure providerID", + instance: &fakeInstance{id: "99", providerID: "azure:///test/test"}, + provider: providerconfigtypes.CloudProviderAWS, + nodes: []corev1.Node{ + getTestNode("1", "random"), + getTestNode("2", "azure:///test/test"), + }, + expectedNode: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + node := findNodeByProviderID(test.instance, test.provider, test.nodes) + if (node != nil) != test.expectedNode { + t.Errorf("expected %t, but got %t", test.expectedNode, (node != nil)) + } + }) + } +} From cc0e5e962c9bb26d85d6dd6e7485b1f8da55a9f6 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Mon, 25 Jul 2022 09:59:28 +0200 Subject: [PATCH 189/489] fix typos (#1371) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix typos * Update pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go Co-authored-by: Marko Mudrinić Co-authored-by: Marko Mudrinić --- cmd/machine-controller/main.go | 4 ++-- examples/openstack-machinedeployment.yaml | 2 +- .../cluster/v1alpha1/migrations/migrations.go | 2 +- .../provider/anexia/provider_test.go | 2 +- .../provider/anexia/types/errors.go | 2 +- pkg/cloudprovider/provider/aws/provider.go | 10 ++++----- .../baremetal/plugins/tinkerbell/driver.go | 2 +- pkg/cloudprovider/provider/nutanix/client.go | 2 +- .../provider/scaleway/provider.go | 4 ++-- pkg/controller/machine/kubeconfig_test.go | 22 +++++++++---------- pkg/controller/machine/machine_test.go | 4 ++-- pkg/userdata/amzn2/provider_test.go | 4 ++-- pkg/userdata/centos/provider_test.go | 4 ++-- pkg/userdata/rhel/provider_test.go | 4 ++-- pkg/userdata/rockylinux/provider_test.go | 4 ++-- test/e2e/provisioning/all_e2e_test.go | 8 +++---- test/e2e/provisioning/deploymentscenario.go | 2 +- test/e2e/provisioning/verify.go | 8 +++---- 18 files changed, 45 insertions(+), 45 deletions(-) diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index fe5a52183..1f4f8f9c1 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -152,7 +152,7 @@ func main() { flag.StringVar(&healthProbeAddress, "health-probe-address", "127.0.0.1:8085", "The address on which the liveness check on /healthz and readiness check on /readyz will be available") flag.StringVar(&metricsAddress, "metrics-address", "127.0.0.1:8080", "The address on which Prometheus metrics will be available under /metrics") flag.StringVar(&name, "name", "", "When set, the controller will only process machines with the label \"machine.k8s.io/controller\": name") - flag.StringVar(&joinClusterTimeout, "join-cluster-timeout", "", "when set, machines that have an owner and do not join the cluster within the configured duration will be deleted, so the owner re-creats them") + flag.StringVar(&joinClusterTimeout, "join-cluster-timeout", "", "when set, machines that have an owner and do not join the cluster within the configured duration will be deleted, so the owner re-creates them") flag.StringVar(&bootstrapTokenServiceAccountName, "bootstrap-token-service-account-name", "", "When set use the service account token from this SA as bootstrap token instead of creating a temporary one. Passed in namespace/name format") flag.BoolVar(&profiling, "enable-profiling", false, "when set, enables the endpoints on the http server under /debug/pprof/") flag.DurationVar(&skipEvictionAfter, "skip-eviction-after", 2*time.Hour, "Skips the eviction if a machine is not gone after the specified duration.") @@ -378,7 +378,7 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { } // Migrate providerConfig field to providerSpec field. - if err := migrations.MigrateProviderConfigToProviderSpecIfNecesary(ctx, bs.opt.cfg, client); err != nil { + if err := migrations.MigrateProviderConfigToProviderSpecIfNecessary(ctx, bs.opt.cfg, client); err != nil { return fmt.Errorf("migration of providerConfig field to providerSpec field failed: %w", err) } diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index cc7d4e23c..a5bc3eedc 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -129,7 +129,7 @@ spec: image: "Ubuntu 18.04 amd64" flavor: "m1.small" # UUID of the server group - # used to configure affinity or anti-affinity of the VM instaces relative to hypervisor + # used to configure affinity or anti-affinity of the VM instances relative to hypervisor serverGroup: "" securityGroups: - configMapKeyRef: diff --git a/pkg/apis/cluster/v1alpha1/migrations/migrations.go b/pkg/apis/cluster/v1alpha1/migrations/migrations.go index 0d1f712fa..284bfa18a 100644 --- a/pkg/apis/cluster/v1alpha1/migrations/migrations.go +++ b/pkg/apis/cluster/v1alpha1/migrations/migrations.go @@ -51,7 +51,7 @@ import ( ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) -func MigrateProviderConfigToProviderSpecIfNecesary(ctx context.Context, config *restclient.Config, client ctrlruntimeclient.Client) error { +func MigrateProviderConfigToProviderSpecIfNecessary(ctx context.Context, config *restclient.Config, client ctrlruntimeclient.Client) error { klog.Infof("Starting to migrate providerConfigs to providerSpecs") dynamicClient, err := dynamicclient.NewForConfig(config) if err != nil { diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index d03feda8a..8592f5764 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -79,7 +79,7 @@ func TestAnexiaProvider(t *testing.T) { } if providerStatus.InstanceID != TestIdentifier { - t.Errorf("Excpected InstanceID to be set") + t.Error("Expected InstanceID to be set") } }) diff --git a/pkg/cloudprovider/provider/anexia/types/errors.go b/pkg/cloudprovider/provider/anexia/types/errors.go index 28d3dd7bb..65f7ab6d2 100644 --- a/pkg/cloudprovider/provider/anexia/types/errors.go +++ b/pkg/cloudprovider/provider/anexia/types/errors.go @@ -29,7 +29,7 @@ func (r MultiErrors) Error() string { for i, err := range r { errString[i] = fmt.Sprintf("Error %d: %s", i, err) } - return fmt.Sprintf("Multiple errors occoured:\n%s", strings.Join(errString, "\n")) + return fmt.Sprintf("Multiple errors occurred:\n%s", strings.Join(errString, "\n")) } func NewMultiError(errs ...error) error { diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index bc81d8bea..ee28c0570 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -1137,7 +1137,7 @@ func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) e } type ec2Credentials struct { - acccessKeyID string + accessKeyID string secretAccessKey string region string assumeRoleARN string @@ -1155,7 +1155,7 @@ func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) e // Very simple and very stupid machineEc2Credentials[fmt.Sprintf("%s/%s/%s/%s/%s", config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID)] = ec2Credentials{ - acccessKeyID: config.AccessKeyID, + accessKeyID: config.AccessKeyID, secretAccessKey: config.SecretAccessKey, region: config.Region, assumeRoleARN: config.AssumeRoleARN, @@ -1165,7 +1165,7 @@ func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) e allReservations := []*ec2.Reservation{} for _, cred := range machineEc2Credentials { - ec2Client, err := getEC2client(cred.acccessKeyID, cred.secretAccessKey, cred.region, cred.assumeRoleARN, cred.assumeRoleExternalID) + ec2Client, err := getEC2client(cred.accessKeyID, cred.secretAccessKey, cred.region, cred.assumeRoleARN, cred.assumeRoleExternalID) if err != nil { machineErrors = append(machineErrors, fmt.Errorf("failed to get EC2 client: %w", err)) continue @@ -1180,7 +1180,7 @@ func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) e for _, machine := range machines.Items { metricInstancesForMachines.WithLabelValues(fmt.Sprintf("%s/%s", machine.Namespace, machine.Name)).Set( - getIntanceCountForMachine(machine, allReservations)) + getInstanceCountForMachine(machine, allReservations)) } if len(machineErrors) > 0 { @@ -1190,7 +1190,7 @@ func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) e return nil } -func getIntanceCountForMachine(machine clusterv1alpha1.Machine, reservations []*ec2.Reservation) float64 { +func getInstanceCountForMachine(machine clusterv1alpha1.Machine, reservations []*ec2.Reservation) float64 { var count float64 for _, reservation := range reservations { for _, i := range reservation.Instances { diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go index 356593248..8c08ff8c4 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go @@ -172,7 +172,7 @@ func (d *driver) ProvisionServer(ctx context.Context, uid types.UID, cfg *plugin } if _, err := d.workflowClient.Create(ctx, workflowTemplate.Id, hw.GetID()); err != nil { - return nil, fmt.Errorf("failed to provisioing server id %s running template id %s: %w", workflowTemplate.Id, hw.GetID(), err) + return nil, fmt.Errorf("failed to provision server id %s running template id %s: %w", workflowTemplate.Id, hw.GetID(), err) } return &hw, nil diff --git a/pkg/cloudprovider/provider/nutanix/client.go b/pkg/cloudprovider/provider/nutanix/client.go index e2ef6c9b3..0ebf070e4 100644 --- a/pkg/cloudprovider/provider/nutanix/client.go +++ b/pkg/cloudprovider/provider/nutanix/client.go @@ -57,7 +57,7 @@ func GetClientSet(config *Config) (*ClientSet, error) { } if config.Password == "" { - return nil, errors.New("no password specificed") + return nil, errors.New("no password specified") } if config.Endpoint == "" { diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go index 1066f5adb..6a2aa4887 100644 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ b/pkg/cloudprovider/provider/scaleway/provider.go @@ -420,12 +420,12 @@ func (s *scwServer) Status() cloudInstance.Status { // if the given error doesn't qualify the error passed as // an argument will be returned. func scalewayErrToTerminalError(err error) error { - var deinedErr *scw.PermissionsDeniedError + var deniedErr *scw.PermissionsDeniedError var invalidArgErr *scw.InvalidArgumentsError var outOfStackErr *scw.OutOfStockError var quotaErr *scw.QuotasExceededError - if errors.As(err, &deinedErr) { + if errors.As(err, &deniedErr) { return cloudprovidererrors.TerminalError{ Reason: common.InvalidConfigurationMachineError, Message: "A request has been rejected due to invalid credentials which were taken from the MachineSpec", diff --git a/pkg/controller/machine/kubeconfig_test.go b/pkg/controller/machine/kubeconfig_test.go index 731080762..a98b702a1 100644 --- a/pkg/controller/machine/kubeconfig_test.go +++ b/pkg/controller/machine/kubeconfig_test.go @@ -31,20 +31,20 @@ import ( func TestUpdateSecretExpirationAndGetToken(t *testing.T) { tests := []struct { - initialExperirationTime time.Time - shouldRenew bool + initialExpirationTime time.Time + shouldRenew bool }{ { - initialExperirationTime: time.Now().Add(1 * time.Hour), - shouldRenew: false, + initialExpirationTime: time.Now().Add(1 * time.Hour), + shouldRenew: false, }, { - initialExperirationTime: time.Now().Add(25 * time.Minute), - shouldRenew: true, + initialExpirationTime: time.Now().Add(25 * time.Minute), + shouldRenew: true, }, { - initialExperirationTime: time.Now().Add(-25 * time.Minute), - shouldRenew: true, + initialExpirationTime: time.Now().Add(-25 * time.Minute), + shouldRenew: true, }, } @@ -58,7 +58,7 @@ func TestUpdateSecretExpirationAndGetToken(t *testing.T) { data := map[string][]byte{} data[tokenSecretKey] = []byte("tokenSecret") data[tokenIDKey] = []byte("tokenID") - data[expirationKey] = []byte(testCase.initialExperirationTime.Format(time.RFC3339)) + data[expirationKey] = []byte(testCase.initialExpirationTime.Format(time.RFC3339)) secret.Data = data reconciler.client = ctrlruntimefake. NewClientBuilder(). @@ -79,12 +79,12 @@ func TestUpdateSecretExpirationAndGetToken(t *testing.T) { } if testCase.shouldRenew && - bytes.Equal(updatedSecret.Data[expirationKey], []byte(testCase.initialExperirationTime.Format(time.RFC3339))) { + bytes.Equal(updatedSecret.Data[expirationKey], []byte(testCase.initialExpirationTime.Format(time.RFC3339))) { t.Errorf("Error, token secret did not update but was expected to!") } if !testCase.shouldRenew && - !bytes.Equal(updatedSecret.Data[expirationKey], []byte(testCase.initialExperirationTime.Format(time.RFC3339))) { + !bytes.Equal(updatedSecret.Data[expirationKey], []byte(testCase.initialExpirationTime.Format(time.RFC3339))) { t.Errorf("Error, token secret was expected to get updated, but did not happen!") } diff --git a/pkg/controller/machine/machine_test.go b/pkg/controller/machine/machine_test.go index c24d5ae32..0fc1f5ccf 100644 --- a/pkg/controller/machine/machine_test.go +++ b/pkg/controller/machine/machine_test.go @@ -258,7 +258,7 @@ func TestControllerDeletesMachinesOnJoinTimeout(t *testing.T) { joinTimeoutConfig: durationPtr(10 * time.Minute), }, { - name: "machine older than joinClusterTimout gets deleted", + name: "machine older than joinClusterTimeout gets deleted", creationTimestamp: metav1.Time{Time: time.Now().Add(-20 * time.Minute)}, hasNode: false, ownerReferences: []metav1.OwnerReference{{Name: "owner", Kind: "MachineSet"}}, @@ -266,7 +266,7 @@ func TestControllerDeletesMachinesOnJoinTimeout(t *testing.T) { joinTimeoutConfig: durationPtr(10 * time.Minute), }, { - name: "machine older than joinClusterTimout does not get deleted when ownerReference.Kind != MachineSet", + name: "machine older than joinClusterTimeout does not get deleted when ownerReference.Kind != MachineSet", creationTimestamp: metav1.Time{Time: time.Now().Add(-20 * time.Minute)}, hasNode: false, ownerReferences: []metav1.OwnerReference{{Name: "owner", Kind: "Cat"}}, diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index baea9d9d0..816e2752f 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -202,10 +202,10 @@ func TestUserDataGeneration(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - emtpyProviderSpec := clusterv1alpha1.ProviderSpec{ + emptyProviderSpec := clusterv1alpha1.ProviderSpec{ Value: &runtime.RawExtension{}, } - test.spec.ProviderSpec = emtpyProviderSpec + test.spec.ProviderSpec = emptyProviderSpec var cloudProvider *fakeCloudConfigProvider if test.cloudProviderName != nil { cloudProvider = &fakeCloudConfigProvider{ diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index 61f5ce8f5..68e8cb472 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -212,10 +212,10 @@ func TestUserDataGeneration(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - emtpyProviderSpec := clusterv1alpha1.ProviderSpec{ + emptyProviderSpec := clusterv1alpha1.ProviderSpec{ Value: &runtime.RawExtension{}, } - test.spec.ProviderSpec = emtpyProviderSpec + test.spec.ProviderSpec = emptyProviderSpec var cloudProvider *fakeCloudConfigProvider if test.cloudProviderName != nil { cloudProvider = &fakeCloudConfigProvider{ diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index 25051f24f..018a06f4d 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -232,10 +232,10 @@ func TestUserDataGeneration(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - emtpyProviderSpec := clusterv1alpha1.ProviderSpec{ + emptyProviderSpec := clusterv1alpha1.ProviderSpec{ Value: &runtime.RawExtension{}, } - test.spec.ProviderSpec = emtpyProviderSpec + test.spec.ProviderSpec = emptyProviderSpec var cloudProvider *fakeCloudConfigProvider if test.cloudProviderName != nil { cloudProvider = &fakeCloudConfigProvider{ diff --git a/pkg/userdata/rockylinux/provider_test.go b/pkg/userdata/rockylinux/provider_test.go index e709a2ee0..de1fc01e8 100644 --- a/pkg/userdata/rockylinux/provider_test.go +++ b/pkg/userdata/rockylinux/provider_test.go @@ -212,10 +212,10 @@ func TestUserDataGeneration(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - emtpyProviderSpec := clusterv1alpha1.ProviderSpec{ + emptyProviderSpec := clusterv1alpha1.ProviderSpec{ Value: &runtime.RawExtension{}, } - test.spec.ProviderSpec = emtpyProviderSpec + test.spec.ProviderSpec = emptyProviderSpec var cloudProvider *fakeCloudConfigProvider if test.cloudProviderName != nil { cloudProvider = &fakeCloudConfigProvider{ diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 2626809f1..df998945b 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -117,7 +117,7 @@ func TestCustomCAsAreApplied(t *testing.T) { osNetwork := os.Getenv("OS_NETWORK_NAME") if osAuthURL == "" || osUsername == "" || osPassword == "" || osDomain == "" || osRegion == "" || osTenant == "" { - t.Fatal("unable to run test suite, all of OS_AUTH_URL, OS_USERNAME, OS_PASSOWRD, OS_REGION, and OS_TENANT OS_DOMAIN must be set!") + t.Fatal("unable to run test suite, all of OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_REGION, and OS_TENANT OS_DOMAIN must be set!") } params := []string{ @@ -311,7 +311,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { osNetwork := os.Getenv("OS_NETWORK_NAME") if osAuthURL == "" || osUsername == "" || osPassword == "" || osDomain == "" || osRegion == "" || osTenant == "" { - t.Fatal("unable to run test suite, all of OS_AUTH_URL, OS_USERNAME, OS_PASSOWRD, OS_REGION, and OS_TENANT OS_DOMAIN must be set!") + t.Fatal("unable to run test suite, all of OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_REGION, and OS_TENANT OS_DOMAIN must be set!") } params := []string{ @@ -342,7 +342,7 @@ func TestOpenstackProjectAuthProvisioningE2E(t *testing.T) { osNetwork := os.Getenv("OS_NETWORK_NAME") if osAuthURL == "" || osUsername == "" || osPassword == "" || osDomain == "" || osRegion == "" || osProject == "" { - t.Fatal("unable to run test suite, all of OS_AUTH_URL, OS_USERNAME, OS_PASSOWRD, OS_REGION, and OS_TENANT OS_DOMAIN must be set!") + t.Fatal("unable to run test suite, all of OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_REGION, and OS_TENANT OS_DOMAIN must be set!") } params := []string{ @@ -978,7 +978,7 @@ func TestUbuntuProvisioningWithUpgradeE2E(t *testing.T) { osNetwork := os.Getenv("OS_NETWORK_NAME") if osAuthURL == "" || osUsername == "" || osPassword == "" || osDomain == "" || osRegion == "" || osTenant == "" { - t.Fatal("unable to run test suite, all of OS_AUTH_URL, OS_USERNAME, OS_PASSOWRD, OS_REGION, and OS_TENANT OS_DOMAIN must be set!") + t.Fatal("unable to run test suite, all of OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_REGION, and OS_TENANT OS_DOMAIN must be set!") } params := []string{ diff --git a/test/e2e/provisioning/deploymentscenario.go b/test/e2e/provisioning/deploymentscenario.go index 0ccec451e..825f81f15 100644 --- a/test/e2e/provisioning/deploymentscenario.go +++ b/test/e2e/provisioning/deploymentscenario.go @@ -51,7 +51,7 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s klog.Infof("Waiting for second MachineSet to appear after updating MachineDeployment %s", machineDeployment.Name) var machineSets []clusterv1alpha1.MachineSet if err := wait.Poll(5*time.Second, timeout, func() (bool, error) { - machineSets, err = getMachingMachineSets(machineDeployment, client) + machineSets, err = getMatchingMachineSets(machineDeployment, client) if err != nil { return false, err } diff --git a/test/e2e/provisioning/verify.go b/test/e2e/provisioning/verify.go index 6a2f323b2..a474d37e4 100644 --- a/test/e2e/provisioning/verify.go +++ b/test/e2e/provisioning/verify.go @@ -272,7 +272,7 @@ func assureNodeForMachineDeployment(machineDeployment *clusterv1alpha1.MachineDe } for _, machine := range machines { - // Azure doesn't seem to easely expose the private IP address, there is only a PublicIPAddressClient in the sdk + // Azure doesn't seem to easily expose the private IP address, there is only a PublicIPAddressClient in the sdk providerConfig, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to get provider config: %w", err) @@ -338,7 +338,7 @@ func readAndModifyManifest(pathToManifest string, keyValuePairs []string) (strin // getMatchingMachines returns all machines that are owned by the passed machineDeployment. func getMatchingMachines(machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Client) ([]clusterv1alpha1.Machine, error) { - matchingMachineSets, err := getMachingMachineSets(machineDeployment, client) + matchingMachineSets, err := getMatchingMachineSets(machineDeployment, client) if err != nil { return nil, err } @@ -369,8 +369,8 @@ func getMatchingMachinesForMachineset(machineSet *clusterv1alpha1.MachineSet, cl return matchingMachines, nil } -// getMachingMachineSets returns all machineSets that are owned by the passed machineDeployment. -func getMachingMachineSets(machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Reader) ([]clusterv1alpha1.MachineSet, error) { +// getMatchingMachineSets returns all machineSets that are owned by the passed machineDeployment. +func getMatchingMachineSets(machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Reader) ([]clusterv1alpha1.MachineSet, error) { // Ensure we actually have an object from the KubeAPI and not just the result of the yaml parsing, as the latter // can not be the owner of anything due to missing UID. if machineDeployment.ResourceVersion == "" { From 6232612e8ee6fc53c20a125415f8764fd4e81517 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 27 Jul 2022 18:07:44 +0500 Subject: [PATCH 190/489] Enable OperatingSystemManager in E2E tests (#1369) * Enable OSM in E2E tests Signed-off-by: Waleed Malik * Update OSM image Signed-off-by: Waleed Malik --- .prow/provider-aws.yaml | 32 + examples/machine-controller.yaml | 31 + examples/operating-system-manager.yaml | 1415 +++++++++++++++++ go.mod | 5 +- go.sum | 4 +- hack/ci/run-e2e-tests.sh | 1 + hack/ci/setup-machine-controller-in-kind.sh | 28 + hack/run-machine-controller.sh | 3 +- .../machinedeployments_validation.go | 3 +- pkg/admission/machines.go | 1 + pkg/cloudprovider/provider/anexia/provider.go | 2 +- .../provider/vmwareclouddirector/helper.go | 11 +- .../provider/vmwareclouddirector/provider.go | 2 +- pkg/controller/machine/bootstrap.go | 4 +- pkg/providerconfig/types.go | 3 +- pkg/providerconfig/types_test.go | 2 +- pkg/userdata/flatcar/flatcar.go | 18 +- test/e2e/provisioning/all_e2e_test.go | 28 +- .../testdata/machinedeployment-anexia.yaml | 6 +- 19 files changed, 1572 insertions(+), 27 deletions(-) create mode 100644 examples/operating-system-manager.yaml diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 2ea31d349..1cc78bcc3 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -42,6 +42,38 @@ presubmits: limits: memory: 6Gi + - name: pull-machine-controller-e2e-aws-legacy-userdata + always_run: true + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-aws: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-rhel: "true" + preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" + spec: + containers: + - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + env: + - name: OPERATING_SYSTEM_MANAGER + value: "false" + command: + - "./hack/ci/run-e2e-tests.sh" + args: + - "TestAWSProvisioningE2E" + securityContext: + privileged: true + resources: + requests: + memory: 6Gi + cpu: 2 + limits: + memory: 6Gi + - name: pull-machine-controller-e2e-aws-arm always_run: true decorate: true diff --git a/examples/machine-controller.yaml b/examples/machine-controller.yaml index fe5a3409e..b724ba5e8 100644 --- a/examples/machine-controller.yaml +++ b/examples/machine-controller.yaml @@ -218,6 +218,7 @@ spec: - -cluster-dns=10.10.10.10 - -metrics-address=0.0.0.0:8080 - -health-probe-address=0.0.0.0:8085 + - -use-osm=true # Machines that fail to join the cluster within this timeout and # are owned by a MachineSet will get deleted so the MachineSet # controller re-creates them @@ -260,6 +261,8 @@ spec: - /usr/local/bin/webhook - -logtostderr - -v=6 + - -use-osm=true + - -namespace=kube-system - -listen-address=0.0.0.0:9876 volumeMounts: - name: machine-controller-admission-cert @@ -566,6 +569,34 @@ subjects: name: machine-controller namespace: kube-system --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: machine-controller-webhook + namespace: kube-system +rules: + - apiGroups: + - operatingsystemmanager.k8c.io + resources: + - operatingsystemprofiles + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: machine-controller-webhook + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: machine-controller-webhook +subjects: + - kind: ServiceAccount + name: machine-controller + namespace: kube-system +--- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml new file mode 100644 index 000000000..2640b5cdb --- /dev/null +++ b/examples/operating-system-manager.yaml @@ -0,0 +1,1415 @@ +# Source: https://github.com/kubermatic/operating-system-manager/tree/v0.5.0/deploy +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: operatingsystemconfigs.operatingsystemmanager.k8c.io +spec: + group: operatingsystemmanager.k8c.io + names: + kind: OperatingSystemConfig + listKind: OperatingSystemConfigList + plural: operatingsystemconfigs + shortNames: + - osc + singular: operatingsystemconfig + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: OperatingSystemConfig is the object that represents the OperatingSystemConfig + properties: + apiVersion: + description: + "APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + kind: + description: + "Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + metadata: + type: object + spec: + description: + OperatingSystemConfigSpec represents the operating system + configuration spec. + properties: + bootstrapConfig: + description: + BootstrapConfig is used for initial configuration of + machine and to fetch the kubernetes secret that contains the provisioning + config. + properties: + files: + description: + Files is a list of files that should exist in the + instance + items: + description: + File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: + Inline is a struct that contains information + about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding (e.g. + base64). + type: string + required: + - data + type: object + type: object + path: + description: + Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + modules: + description: + CloudInitModules contains the supported cloud-init + modules + properties: + bootcmd: + description: + BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string + type: array + rh_subscription: + additionalProperties: + type: string + description: + RHSubscription registers a Red Hat system either + by username and password or activation and org + type: object + runcmd: + description: + RunCMD Run arbitrary commands at a rc.local like + level with output to the console. + items: + type: string + type: array + yum_repo_dir: + description: + "YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d" + type: string + yum_repos: + additionalProperties: + additionalProperties: + type: string + type: object + description: + YumRepos adds yum repository configuration to + the system. + type: object + type: object + units: + description: + Units a list of the systemd unit files which will + run on the instance + items: + description: + Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. + items: + description: + DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object + type: array + enable: + description: + Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: + Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. + type: string + required: + - name + type: object + type: array + userSSHKeys: + description: UserSSHKeys is a list of attached user ssh keys + items: + type: string + type: array + type: object + cloudProvider: + description: + CloudProvider represent the cloud provider that support + the given operating system version + properties: + name: + description: Name represents the name of the supported cloud provider + enum: + - aws + - azure + - digitalocean + - gce + - hetzner + - kubevirt + - linode + - nutanix + - openstack + - equinixmetal + - vsphere + - fake + - alibaba + - anexia + - scaleway + - baremetal + - external + - vmware-cloud-director + type: string + spec: + description: + Spec represents the os/image reference in the supported + cloud provider + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - name + type: object + osName: + description: "OSType represent the operating system name e.g: ubuntu" + enum: + - flatcar + - rhel + - centos + - ubuntu + - sles + - amzn2 + - rockylinux + type: string + osVersion: + description: OSVersion the version of the operating system + type: string + provisioningConfig: + description: + ProvisioningConfig is used for provisioning the worker + node. + properties: + files: + description: + Files is a list of files that should exist in the + instance + items: + description: + File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: + Inline is a struct that contains information + about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding (e.g. + base64). + type: string + required: + - data + type: object + type: object + path: + description: + Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + modules: + description: + CloudInitModules contains the supported cloud-init + modules + properties: + bootcmd: + description: + BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string + type: array + rh_subscription: + additionalProperties: + type: string + description: + RHSubscription registers a Red Hat system either + by username and password or activation and org + type: object + runcmd: + description: + RunCMD Run arbitrary commands at a rc.local like + level with output to the console. + items: + type: string + type: array + yum_repo_dir: + description: + "YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d" + type: string + yum_repos: + additionalProperties: + additionalProperties: + type: string + type: object + description: + YumRepos adds yum repository configuration to + the system. + type: object + type: object + units: + description: + Units a list of the systemd unit files which will + run on the instance + items: + description: + Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. + items: + description: + DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object + type: array + enable: + description: + Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: + Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. + type: string + required: + - name + type: object + type: array + userSSHKeys: + description: UserSSHKeys is a list of attached user ssh keys + items: + type: string + type: array + type: object + required: + - bootstrapConfig + - cloudProvider + - osName + - osVersion + - provisioningConfig + type: object + required: + - spec + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.0 + creationTimestamp: null + name: operatingsystemprofiles.operatingsystemmanager.k8c.io +spec: + group: operatingsystemmanager.k8c.io + names: + kind: OperatingSystemProfile + listKind: OperatingSystemProfileList + plural: operatingsystemprofiles + shortNames: + - osp + singular: operatingsystemprofile + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: OperatingSystemProfile is the object that represents the OperatingSystemProfile + properties: + apiVersion: + description: + "APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + kind: + description: + "Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + metadata: + type: object + spec: + description: + OperatingSystemProfileSpec represents the operating system + configuration spec. + properties: + bootstrapConfig: + description: + BootstrapConfig is used for initial configuration of + machine and to fetch the kubernetes secret that contains the provisioning + config. + properties: + files: + description: + Files is a list of files that should exist in the + instance + items: + description: + File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: + Inline is a struct that contains information + about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding (e.g. + base64). + type: string + required: + - data + type: object + type: object + path: + description: + Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + modules: + description: + CloudInitModules field contains the optional cloud-init + modules which are supported by OSM + properties: + bootcmd: + description: + BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string + type: array + rh_subscription: + additionalProperties: + type: string + description: + RHSubscription registers a Red Hat system either + by username and password or activation and org + type: object + runcmd: + description: + RunCMD Run arbitrary commands at a rc.local like + level with output to the console. + items: + type: string + type: array + yum_repo_dir: + description: + "YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d" + type: string + yum_repos: + additionalProperties: + additionalProperties: + type: string + type: object + description: + YumRepos adds yum repository configuration to + the system. + type: object + type: object + supportedContainerRuntimes: + description: + SupportedContainerRuntimes represents the container + runtimes supported by the given OS + items: + description: + ContainerRuntimeSpec aggregates information about + a specific container runtime + properties: + files: + description: + Files to add to the main files list when the + containerRuntime is selected + items: + description: + File is a file that should get written to + the host's file system. The content can either be inlined + or referenced from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: + Inline is a struct that contains + information about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding + (e.g. base64). + type: string + required: + - data + type: object + type: object + path: + description: + Path is the path of the file system where + the file should get written to. + type: string + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. + Should be in decimal base and without any leading + zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + name: + description: Name of the Container runtime + enum: + - docker + - containerd + type: string + templates: + additionalProperties: + type: string + description: + Templates to add to the available templates + when the containerRuntime is selected + type: object + required: + - files + - name + type: object + type: array + templates: + additionalProperties: + type: string + description: Templates to be included in units and files + type: object + units: + description: + Units a list of the systemd unit files which will + run on the instance + items: + description: + Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. + items: + description: + DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object + type: array + enable: + description: + Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: + Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. + type: string + required: + - name + type: object + type: array + type: object + osName: + description: "OSType represent the operating system name e.g: ubuntu" + enum: + - flatcar + - rhel + - centos + - ubuntu + - sles + - amzn2 + - rockylinux + type: string + osVersion: + description: OSVersion the version of the operating system + type: string + provisioningConfig: + description: + ProvisioningConfig is used for provisioning the worker + node. + properties: + files: + description: + Files is a list of files that should exist in the + instance + items: + description: + File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: + Inline is a struct that contains information + about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding (e.g. + base64). + type: string + required: + - data + type: object + type: object + path: + description: + Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + modules: + description: + CloudInitModules field contains the optional cloud-init + modules which are supported by OSM + properties: + bootcmd: + description: + BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string + type: array + rh_subscription: + additionalProperties: + type: string + description: + RHSubscription registers a Red Hat system either + by username and password or activation and org + type: object + runcmd: + description: + RunCMD Run arbitrary commands at a rc.local like + level with output to the console. + items: + type: string + type: array + yum_repo_dir: + description: + "YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d" + type: string + yum_repos: + additionalProperties: + additionalProperties: + type: string + type: object + description: + YumRepos adds yum repository configuration to + the system. + type: object + type: object + supportedContainerRuntimes: + description: + SupportedContainerRuntimes represents the container + runtimes supported by the given OS + items: + description: + ContainerRuntimeSpec aggregates information about + a specific container runtime + properties: + files: + description: + Files to add to the main files list when the + containerRuntime is selected + items: + description: + File is a file that should get written to + the host's file system. The content can either be inlined + or referenced from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: + Inline is a struct that contains + information about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding + (e.g. base64). + type: string + required: + - data + type: object + type: object + path: + description: + Path is the path of the file system where + the file should get written to. + type: string + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. + Should be in decimal base and without any leading + zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + name: + description: Name of the Container runtime + enum: + - docker + - containerd + type: string + templates: + additionalProperties: + type: string + description: + Templates to add to the available templates + when the containerRuntime is selected + type: object + required: + - files + - name + type: object + type: array + templates: + additionalProperties: + type: string + description: Templates to be included in units and files + type: object + units: + description: + Units a list of the systemd unit files which will + run on the instance + items: + description: + Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. + items: + description: + DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object + type: array + enable: + description: + Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: + Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. + type: string + required: + - name + type: object + type: array + type: object + supportedCloudProviders: + description: + SupportedCloudProviders represent the cloud providers + that support the given operating system version + items: + description: + CloudProviderSpec contains the os/image reference for + a specific supported cloud provider + properties: + name: + description: + Name represents the name of the supported cloud + provider + enum: + - aws + - azure + - digitalocean + - gce + - hetzner + - kubevirt + - linode + - nutanix + - openstack + - equinixmetal + - vsphere + - fake + - alibaba + - anexia + - scaleway + - baremetal + - external + - vmware-cloud-director + type: string + spec: + description: + Spec represents the os/image reference in the supported + cloud provider + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - name + type: object + type: array + version: + description: Version is the version of the operating System Profile + pattern: v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$ + type: string + required: + - bootstrapConfig + - osName + - osVersion + - provisioningConfig + - supportedCloudProviders + - version + type: object + required: + - spec + type: object + served: true + storage: true +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: operating-system-manager-selfsigned-issuer + namespace: kube-system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: operating-system-manager-serving-cert + namespace: kube-system +spec: + dnsNames: + - "operating-system-manager-webhook.kube-system.svc" + - "operating-system-manager-webhook.kube-system.svc.cluster.local" + issuerRef: + kind: Issuer + name: operating-system-manager-selfsigned-issuer + secretName: webhook-server-cert +--- +apiVersion: v1 +kind: Namespace +metadata: + name: cloud-init-settings +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: cloud-init-getter + namespace: cloud-init-settings +--- +apiVersion: v1 +kind: Secret +type: kubernetes.io/service-account-token +metadata: + name: cloud-init-getter-token + namespace: cloud-init-settings + annotations: + kubernetes.io/service-account.name: "cloud-init-getter" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cloud-init-getter + namespace: cloud-init-settings +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cloud-init-getter + namespace: cloud-init-settings +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cloud-init-getter +subjects: + - kind: ServiceAccount + name: cloud-init-getter + namespace: cloud-init-settings +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: operating-system-manager-webhook + namespace: kube-system +--- +apiVersion: v1 +kind: Service +metadata: + name: operating-system-manager-webhook + namespace: kube-system +spec: + ports: + - name: 443-9443 + port: 443 + protocol: TCP + targetPort: 9443 + selector: + app: operating-system-manager-webhook + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: operating-system-manager-webhook + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: operating-system-manager-webhook + template: + metadata: + labels: + app: operating-system-manager-webhook + spec: + serviceAccountName: operating-system-manager-webhook + containers: + # TODO: Update this to a semver tag before release. + # - image: quay.io/kubermatic/operating-system-manager:952f1cd42007d6770b8e2805b0e18247377b14bb # Ref: https://github.com/kubermatic/operating-system-manager/commit/952f1cd42007d6770b8e2805b0e18247377b14bb + - image: quay.io/kubermatic/operating-system-manager:952f1cd42007d6770b8e2805b0e18247377b14bb + imagePullPolicy: IfNotPresent + name: webhook + command: + - /usr/local/bin/webhook + - -logtostderr + - -v=6 + - -namespace=kube-system + volumeMounts: + - name: operating-system-manager-admission-cert + mountPath: /tmp/k8s-webhook-server/serving-certs + readOnly: true + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + memory: 256Mi + cpu: 100m + limits: + memory: 256Mi + cpu: 100m + volumes: + - name: operating-system-manager-admission-cert + secret: + defaultMode: 420 + secretName: webhook-server-cert +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: operatingsystemmanager.k8c.io + annotations: + cert-manager.io/inject-ca-from: kube-system/operating-system-manager-serving-cert +webhooks: + - name: operatingsystemprofiles.operatingsystemmanager.k8c.io + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: ["v1", "v1beta1"] + rules: + - apiGroups: + - "operatingsystemmanager.k8c.io" + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - operatingsystemprofiles + clientConfig: + service: + namespace: kube-system + name: operating-system-manager-webhook + path: /operatingsystemprofile + - name: operatingsystemconfigs.operatingsystemmanager.k8c.io + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: ["v1", "v1beta1"] + rules: + - apiGroups: + - "operatingsystemmanager.k8c.io" + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - operatingsystemconfigs + clientConfig: + service: + namespace: kube-system + name: operating-system-manager-webhook + path: /operatingsystemconfig +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: operating-system-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: operating-system-manager + namespace: kube-system +rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - apiGroups: + - "" + resources: + - configmaps + - secrets + verbs: + - create + - update + - list + - get + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - "*" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: operating-system-manager + namespace: cloud-init-settings +rules: + # Secrets access is required for managing provisioning configurations + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - create + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: operating-system-manager + namespace: kube-public +rules: + - apiGroups: + - "" + resources: + - configmaps + resourceNames: + - cluster-info + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: operating-system-manager + namespace: default +rules: + - apiGroups: + - "" + resources: + - endpoints + resourceNames: + - kubernetes + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: operating-system-manager + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: operating-system-manager +subjects: + - kind: ServiceAccount + name: operating-system-manager +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: operating-system-manager + namespace: cloud-init-settings +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: operating-system-manager +subjects: + - kind: ServiceAccount + name: operating-system-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: operating-system-manager + namespace: kube-public +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: operating-system-manager +subjects: + - kind: ServiceAccount + name: operating-system-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: operating-system-manager + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: operating-system-manager +subjects: + - kind: ServiceAccount + name: operating-system-manager + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:operating-system-manager +rules: + - apiGroups: + - operatingsystemmanager.k8c.io + resources: + - operatingsystemprofiles + - operatingsystemconfigs + verbs: + - "*" + - apiGroups: + - cluster.k8s.io + resources: + - machinedeployments + verbs: + - get + - list + - watch + - patch + - update + # Secrets and configmaps are needed for the bootstrap token creation and when a ref is used for a + # value in the machineSpec + - apiGroups: + - "" + resources: + - secrets + - configmaps + verbs: + - list + - get + - watch + - apiGroups: + - apps + resources: + - deployments + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:operating-system-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:operating-system-manager +subjects: + - kind: ServiceAccount + name: operating-system-manager + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: operating-system-manager + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: operating-system-manager + template: + metadata: + annotations: + "prometheus.io/scrape": "true" + "prometheus.io/port": "8080" + "prometheus.io/path": "/metrics" + labels: + app: operating-system-manager + spec: + serviceAccountName: operating-system-manager + containers: + # - image: quay.io/kubermatic/operating-system-manager:952f1cd42007d6770b8e2805b0e18247377b14bb # Ref: https://github.com/kubermatic/operating-system-manager/commit/952f1cd42007d6770b8e2805b0e18247377b14bb + - image: quay.io/kubermatic/operating-system-manager:952f1cd42007d6770b8e2805b0e18247377b14bb + imagePullPolicy: IfNotPresent + name: operating-system-manager + command: + - /usr/local/bin/osm-controller + - -logtostderr + - -v=5 + - -worker-count=5 + - -cluster-dns=10.10.10.10 + - -metrics-address=0.0.0.0:8080 + - -health-probe-address=0.0.0.0:8085 + - -namespace=kube-system + - -container-runtime=containerd + ports: + - containerPort: 8085 + livenessProbe: + httpGet: + path: /readyz + port: 8085 + initialDelaySeconds: 5 + periodSeconds: 5 + readinessProbe: + httpGet: + path: /healthz + port: 8085 + periodSeconds: 5 + resources: + requests: + memory: 256Mi + cpu: 100m + limits: + memory: 256Mi + cpu: 100m diff --git a/go.mod b/go.mod index c21b40bb2..ba74e39bf 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,10 @@ require ( google.golang.org/grpc v1.45.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 - k8c.io/operating-system-manager v0.5.0 + // TODO: Update this to a semver tag before release. + // Ref: https://github.com/kubermatic/operating-system-manager/commit/952f1cd42007d6770b8e2805b0e18247377b14bb + // Please ensure that you update the image tags in `examples/operating-system-manager.yaml` as well. + k8c.io/operating-system-manager v0.5.1-0.20220727101546-952f1cd42007 k8s.io/api v0.24.2 k8s.io/apiextensions-apiserver v0.24.2 k8s.io/apimachinery v0.24.2 diff --git a/go.sum b/go.sum index a42511c89..a6d5e4720 100644 --- a/go.sum +++ b/go.sum @@ -1521,8 +1521,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8c.io/operating-system-manager v0.5.0 h1:HRPPhJG27gl5T/HpGXHrzdbONVmdi1UnASZpKln1N04= -k8c.io/operating-system-manager v0.5.0/go.mod h1:8Q1xpjJomTG9X6lfx/y3+yGHCackHtqxuYEk0TIPMfA= +k8c.io/operating-system-manager v0.5.1-0.20220727101546-952f1cd42007 h1:a6+IbIxdm63VNW5D1jvTLJBlcPy3KviyQWh6nJzogv4= +k8c.io/operating-system-manager v0.5.1-0.20220727101546-952f1cd42007/go.mod h1:8Q1xpjJomTG9X6lfx/y3+yGHCackHtqxuYEk0TIPMfA= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.24.2 h1:g518dPU/L7VRLxWfcadQn2OnsiGWVOadTLpdnqgY2OI= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= diff --git a/hack/ci/run-e2e-tests.sh b/hack/ci/run-e2e-tests.sh index ba4140807..95b4687aa 100755 --- a/hack/ci/run-e2e-tests.sh +++ b/hack/ci/run-e2e-tests.sh @@ -33,6 +33,7 @@ trap cleanup EXIT export GIT_HEAD_HASH="$(git rev-parse HEAD)" export MC_VERSION="${GIT_HEAD_HASH}" +export OPERATING_SYSTEM_MANAGER="${OPERATING_SYSTEM_MANAGER:-true}" TEST_NAME="Pre-warm Go build cache" echodate "Attempting to pre-warm Go build cache" diff --git a/hack/ci/setup-machine-controller-in-kind.sh b/hack/ci/setup-machine-controller-in-kind.sh index 3ba2a8c87..d34d5b934 100755 --- a/hack/ci/setup-machine-controller-in-kind.sh +++ b/hack/ci/setup-machine-controller-in-kind.sh @@ -22,6 +22,7 @@ if [ -z "${KIND_CLUSTER_NAME:-}" ]; then fi export MC_VERSION="${MC_VERSION:-$(git rev-parse HEAD)}" +export OPERATING_SYSTEM_MANAGER="${OPERATING_SYSTEM_MANAGER:-true}" # Build the Docker image for machine-controller beforeDockerBuild=$(nowms) @@ -45,10 +46,37 @@ if [ ! -f machine-controller-deployed ]; then # This is required for running e2e tests in KIND url="-override-bootstrap-kubelet-apiserver=$MASTER_URL" sed -i "s;-node-csr-approver=true;$url;g" examples/machine-controller.yaml + + # Ensure that we update `use-osm` flag if OSM is disabled + if [[ "$OPERATING_SYSTEM_MANAGER" == "false" ]]; then + sed -i "s;-use-osm=true;-use-osm=false;g" examples/machine-controller.yaml + fi + make deploy touch machine-controller-deployed fi +if [[ "$OPERATING_SYSTEM_MANAGER" == "true" ]]; then + # cert-manager is required by OSM for generating TLS Certificates + echodate "Installing cert-manager" + ( + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.7.1/cert-manager.yaml + # Wait for cert-manager to be ready + kubectl -n cert-manager rollout status deploy/cert-manager + kubectl -n cert-manager rollout status deploy/cert-manager-cainjector + kubectl -n cert-manager rollout status deploy/cert-manager-webhook + ) + + echodate "Installing operating-system-manager" + ( + # This is required for running e2e tests in KIND + url="-override-bootstrap-kubelet-apiserver=$MASTER_URL" + sed -i "s;-container-runtime=containerd;$url;g" examples/operating-system-manager.yaml + sed -i -e 's/-worker-count=5/-worker-count=50/g' examples/operating-system-manager.yaml + kubectl apply -f examples/operating-system-manager.yaml + ) +fi + sleep 10 retry 10 check_all_deployments_ready kube-system diff --git a/hack/run-machine-controller.sh b/hack/run-machine-controller.sh index 192a52c46..7718af663 100755 --- a/hack/run-machine-controller.sh +++ b/hack/run-machine-controller.sh @@ -21,7 +21,7 @@ set -e MC_KUBECONFIG=${MC_KUBECONFIG:-$(dirname $0)/../.kubeconfig} # If you want to use the default kubeconfig `export MC_KUBECONFIG=$KUBECONFIG` -# `-use-osm` flag can be specified if https://github.com/kubermatic/operating-system-manager is used to manage user data. +# `-use-osm` flag can be removed to use legacy userdata that is generated by machine-controller. make -C $(dirname $0)/.. build-machine-controller $(dirname $0)/../machine-controller \ @@ -33,4 +33,5 @@ $(dirname $0)/../machine-controller \ -enable-profiling \ -metrics-address=0.0.0.0:8080 \ -health-probe-address=0.0.0.0:8085 \ + -use-osm=true \ -node-container-runtime=containerd diff --git a/pkg/admission/machinedeployments_validation.go b/pkg/admission/machinedeployments_validation.go index 57442b589..e7b13c2b5 100644 --- a/pkg/admission/machinedeployments_validation.go +++ b/pkg/admission/machinedeployments_validation.go @@ -157,7 +157,8 @@ func ensureOSPAnnotation(md *v1alpha1.MachineDeployment, providerConfig provider // Annotation not specified, populate default OSP annotation switch providerConfig.OperatingSystem { case providerconfigtypes.OperatingSystemUbuntu, providerconfigtypes.OperatingSystemCentOS, providerconfigtypes.OperatingSystemFlatcar, - providerconfigtypes.OperatingSystemAmazonLinux2: + providerconfigtypes.OperatingSystemAmazonLinux2, providerconfigtypes.OperatingSystemRockyLinux, providerconfigtypes.OperatingSystemSLES, + providerconfigtypes.OperatingSystemRHEL: md.Annotations[osmresources.MachineDeploymentOSPAnnotation] = fmt.Sprintf(ospNamePattern, providerConfig.OperatingSystem) return nil diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index 295ebb135..1e8972fec 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -160,6 +160,7 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec providerConfig.OperatingSystem, providerConfig.CloudProvider, providerConfig.OperatingSystemSpec, + ad.useOSM, ) if err != nil { return err diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 7d99885e2..7d451e84b 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -228,7 +228,7 @@ func getIPAddress(ctx context.Context, client anxclient.Client) (string, error) klog.Info("reusing already provisioned ip", "IP", status.ReservedIP) return status.ReservedIP, nil } - klog.Info(fmt.Sprintf("Creating a new IP for machine ''%s", reconcileContext.Machine.Name)) + klog.Info(fmt.Sprintf("Creating a new IP for machine %q", reconcileContext.Machine.Name)) addrAPI := anxaddr.NewAPI(client) config := reconcileContext.Config res, err := addrAPI.ReserveRandom(ctx, anxaddr.ReserveRandom{ diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go index 48dcdc8f1..1b21f0ef9 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go @@ -28,7 +28,6 @@ import ( vcdapitypes "github.com/vmware/go-vcloud-director/v2/types/v56" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" "k8s.io/utils/pointer" ) @@ -238,14 +237,20 @@ func recomposeComputeAndDisk(config *Config, vm *govcd.VM) (*govcd.VM, error) { return vm, nil } -func setUserData(userdata string, vm *govcd.VM, providerConfig *providerconfigtypes.Config) error { +func setUserData(userdata string, vm *govcd.VM, isFlatcar bool) error { userdataBase64 := base64.StdEncoding.EncodeToString([]byte(userdata)) props := map[string]string{ - "user-data": userdataBase64, "disk.enableUUID": "1", "instance-id": vm.VM.Name, } + if isFlatcar { + props["guestinfo.ignition.config.data"] = userdataBase64 + props["guestinfo.ignition.config.data.encoding"] = "base64" + } else { + props["user-data"] = userdataBase64 + } + vmProperties := &vcdapitypes.ProductSectionList{ ProductSection: &vcdapitypes.ProductSection{ Info: "Custom properties", diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go index 473bf4329..9c06152d1 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -251,7 +251,7 @@ func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, // 5. Before powering on the VM, configure customization to attach userdata with the VM // update guest properties. - err = setUserData(userdata, vm, providerConfig) + err = setUserData(userdata, vm, providerConfig.OperatingSystem == providerconfigtypes.OperatingSystemFlatcar) if err != nil { return nil, err } diff --git a/pkg/controller/machine/bootstrap.go b/pkg/controller/machine/bootstrap.go index e8e81d0bb..68ea4a656 100644 --- a/pkg/controller/machine/bootstrap.go +++ b/pkg/controller/machine/bootstrap.go @@ -17,6 +17,7 @@ limitations under the License. package controller import ( + "net/url" "regexp" "strings" @@ -30,7 +31,8 @@ func getOSMBootstrapUserdata(machineName string, bootstrapSecret corev1.Secret) // We have to inject the hostname i.e. machine name. bootstrapConfig = strings.ReplaceAll(bootstrapConfig, hostnamePlaceholder, machineName) - + // Data is HTML Encoded for ignition. + bootstrapConfig = strings.ReplaceAll(bootstrapConfig, url.QueryEscape(hostnamePlaceholder), url.QueryEscape(machineName)) return cleanupTemplateOutput(bootstrapConfig) } diff --git a/pkg/providerconfig/types.go b/pkg/providerconfig/types.go index 27e73c6a5..0b0879a80 100644 --- a/pkg/providerconfig/types.go +++ b/pkg/providerconfig/types.go @@ -184,6 +184,7 @@ func DefaultOperatingSystemSpec( osys providerconfigtypes.OperatingSystem, cloudProvider providerconfigtypes.CloudProvider, operatingSystemSpec runtime.RawExtension, + operatingSystemManagerEnabled bool, ) (runtime.RawExtension, error) { switch osys { case providerconfigtypes.OperatingSystemAmazonLinux2: @@ -191,7 +192,7 @@ func DefaultOperatingSystemSpec( case providerconfigtypes.OperatingSystemCentOS: return centos.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemFlatcar: - return flatcar.DefaultConfigForCloud(operatingSystemSpec, cloudProvider), nil + return flatcar.DefaultConfigForCloud(operatingSystemSpec, cloudProvider, operatingSystemManagerEnabled), nil case providerconfigtypes.OperatingSystemRHEL: return rhel.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemSLES: diff --git a/pkg/providerconfig/types_test.go b/pkg/providerconfig/types_test.go index 40930062b..13deb05ae 100644 --- a/pkg/providerconfig/types_test.go +++ b/pkg/providerconfig/types_test.go @@ -30,7 +30,7 @@ func TestDefaultOperatingSystemSpec(t *testing.T) { for _, osys := range providerconfigtypes.AllOperatingSystems { osys := osys t.Run(string(osys), func(t *testing.T) { - operatingSystemSpec, err := DefaultOperatingSystemSpec(osys, "", runtime.RawExtension{}) + operatingSystemSpec, err := DefaultOperatingSystemSpec(osys, "", runtime.RawExtension{}, true) if err != nil { t.Error("no error expected") diff --git a/pkg/userdata/flatcar/flatcar.go b/pkg/userdata/flatcar/flatcar.go index cdb5166e9..c76c63a35 100644 --- a/pkg/userdata/flatcar/flatcar.go +++ b/pkg/userdata/flatcar/flatcar.go @@ -44,19 +44,29 @@ type Config struct { } func DefaultConfig(operatingSystemSpec runtime.RawExtension) runtime.RawExtension { - return DefaultConfigForCloud(operatingSystemSpec, "") + // Webhook has already performed the defaulting at this point. So the value for + // cloudProvider and operatingSystemManagerEnabled parameters are insignificant. + return DefaultConfigForCloud(operatingSystemSpec, "", true) } -func DefaultConfigForCloud(operatingSystemSpec runtime.RawExtension, cloudProvider types.CloudProvider) runtime.RawExtension { +func DefaultConfigForCloud(operatingSystemSpec runtime.RawExtension, cloudProvider types.CloudProvider, operatingSystemManagerEnabled bool) runtime.RawExtension { + // If userdata is being used from machine-controller and selected cloud provider is AWS then we + // force cloud-init. Because AWS has a very low cap for the maximum size of user-data. In case of ignition, + // we always exceed that limit which prevents new ec2 instances from being created. osSpec := Config{} - if operatingSystemSpec.Raw != nil { _ = json.Unmarshal(operatingSystemSpec.Raw, &osSpec) } - if cloudProvider == types.CloudProviderAWS { + // In case of OSM this is not required. + if cloudProvider == types.CloudProviderAWS && !operatingSystemManagerEnabled { osSpec.ProvisioningUtility = CloudInit } + // Always default to ignition if no value was provided + if osSpec.ProvisioningUtility == "" { + osSpec.ProvisioningUtility = Ignition + } + operatingSystemSpec.Raw, _ = json.Marshal(osSpec) return operatingSystemSpec diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index df998945b..c19c979c6 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -324,7 +324,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { fmt.Sprintf("<< NETWORK_NAME >>=%s", osNetwork), } - selector := Not(OsSelector("sles", "rhel", "amzn2")) + selector := Not(OsSelector("sles", "amzn2")) runScenarios(t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -390,6 +390,12 @@ func TestDigitalOceanProvisioningE2E(t *testing.T) { func TestAWSProvisioningE2E(t *testing.T) { t.Parallel() + provisioningUtility := flatcar.Ignition + // `OPERATING_SYSTEM_MANAGER` will be false when legacy machine-controller userdata should be used for E2E tests. + if v := os.Getenv("OPERATING_SYSTEM_MANAGER"); v == "false" { + provisioningUtility = flatcar.CloudInit + } + // test data awsKeyID := os.Getenv("AWS_E2E_TESTS_KEY_ID") awsSecret := os.Getenv("AWS_E2E_TESTS_SECRET") @@ -402,8 +408,9 @@ func TestAWSProvisioningE2E(t *testing.T) { // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), fmt.Sprintf("<< AWS_SECRET_ACCESS_KEY >>=%s", awsSecret), - fmt.Sprintf("<< PROVISIONING_UTILITY >>=%s", flatcar.CloudInit), + fmt.Sprintf("<< PROVISIONING_UTILITY >>=%s", provisioningUtility), } + runScenarios(t, selector, params, AWSManifest, fmt.Sprintf("aws-%s", *testRunIdentifier)) } @@ -424,7 +431,7 @@ func TestAWSAssumeRoleProvisioningE2E(t *testing.T) { // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), fmt.Sprintf("<< AWS_SECRET_ACCESS_KEY >>=%s", awsSecret), - fmt.Sprintf("<< PROVISIONING_UTILITY >>=%s", flatcar.CloudInit), + fmt.Sprintf("<< PROVISIONING_UTILITY >>=%s", flatcar.Ignition), } scenario := scenario{ @@ -452,7 +459,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), fmt.Sprintf("<< AWS_SECRET_ACCESS_KEY >>=%s", awsSecret), - fmt.Sprintf("<< PROVISIONING_UTILITY >>=%s", flatcar.CloudInit), + fmt.Sprintf("<< PROVISIONING_UTILITY >>=%s", flatcar.Ignition), } runScenarios(t, selector, params, AWSSpotInstanceManifest, fmt.Sprintf("aws-%s", *testRunIdentifier)) } @@ -533,7 +540,7 @@ func TestAWSFlatcarContainerdProvisioningE2E(t *testing.T) { params := []string{ fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), fmt.Sprintf("<< AWS_SECRET_ACCESS_KEY >>=%s", awsSecret), - fmt.Sprintf("<< PROVISIONING_UTILITY >>=%s", flatcar.CloudInit), + fmt.Sprintf("<< PROVISIONING_UTILITY >>=%s", flatcar.Ignition), } scenario := scenario{ @@ -1030,13 +1037,20 @@ func TestAnexiaProvisioningE2E(t *testing.T) { t.Parallel() token := os.Getenv("ANEXIA_TOKEN") - if token == "" { - t.Fatal("unable to run the test suite, ANEXIA_TOKEN environment variable cannot be empty") + vlanID := os.Getenv("ANEXIA_VLAN_ID") + templateID := os.Getenv("ANEXIA_TEMPLATE_ID") + locationID := os.Getenv("ANEXIA_LOCATION_ID") + + if token == "" || vlanID == "" || templateID == "" || locationID == "" { + t.Fatal("unable to run test suite, all of ANEXIA_TOKEN, ANEXIA_VLAN_ID, ANEXIA_TEMPLATE_ID, and ANEXIA_LOCATION_ID must be set!") } selector := OsSelector("flatcar") params := []string{ fmt.Sprintf("<< ANEXIA_TOKEN >>=%s", token), + fmt.Sprintf("<< ANEXIA_VLAN_ID >>=%s", vlanID), + fmt.Sprintf("<< ANEXIA_TEMPLATE_ID >>=%s", templateID), + fmt.Sprintf("<< ANEXIA_LOCATION_ID >>=%s", locationID), } runScenarios(t, selector, params, anexiaManifest, fmt.Sprintf("anexia-%s", *testRunIdentifier)) diff --git a/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml b/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml index 64fcc7aa3..cca5f3dce 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml @@ -25,9 +25,9 @@ spec: cloudProvider: anexia cloudProviderSpec: token: "<< ANEXIA_TOKEN >>" - vlanID: "e37d7134ab934f5683fabcc72d28e036" - templateID: "12c28aa7-604d-47e9-83fb-5f1d1f1837b3" - locationID: "52b5f6b2fd3a4a7eaaedf1a7c019e9ea" + vlanID: "<< ANEXIA_VLAN_ID >>" + templateID: "<< ANEXIA_TEMPLATE_ID >>" + locationID: "<< ANEXIA_LOCATION_ID >>" cpus: 2 memory: 2048 diskSize: 60 From fc27c593bac3adce42d5da7210b5a7dcdef7eecf Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Fri, 29 Jul 2022 01:32:35 +0300 Subject: [PATCH 191/489] Enable --resolv-conf=/run/systemd/resolve/resolv.conf kubelet flag for Flatcar (#1379) * Add missing --resolv-conf to kubelet Signed-off-by: Artiom Diomin * Update fixtures Signed-off-by: Artiom Diomin --- pkg/userdata/flatcar/provider.go | 8 ++++++++ pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml | 4 ++++ pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml | 4 ++++ pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml | 4 ++++ pkg/userdata/flatcar/testdata/containerd.yaml | 4 ++++ pkg/userdata/flatcar/testdata/ignition_v1.22.7.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.23.5.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.24.0.json | 2 +- 8 files changed, 27 insertions(+), 3 deletions(-) diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index 183b7e033..8e89415bb 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -295,6 +295,10 @@ systemd: contents: | [Service] EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: resolv.conf + contents: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - name: 40-download.conf contents: | [Unit] @@ -617,6 +621,10 @@ coreos: content: | [Service] EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: resolv.conf + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - name: 40-download.conf content: | [Unit] diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml index 33c02cf25..e899ddd45 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml @@ -88,6 +88,10 @@ coreos: content: | [Service] EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: resolv.conf + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - name: 40-download.conf content: | [Unit] diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml index fabfc11fc..a222e8af1 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml @@ -88,6 +88,10 @@ coreos: content: | [Service] EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: resolv.conf + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - name: 40-download.conf content: | [Unit] diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml index 941d78668..b1593ad03 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml @@ -88,6 +88,10 @@ coreos: content: | [Service] EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: resolv.conf + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - name: 40-download.conf content: | [Unit] diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index bcd1eedbd..577086ee0 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -73,6 +73,10 @@ coreos: content: | [Service] EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: resolv.conf + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - name: 40-download.conf content: | [Unit] diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json b/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json index a7d6cc773..34834bf0b 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.7%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.7%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json b/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json index 80f86b6a2..806865ad8 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json index c678da203..62a14b9df 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file From e93d0f3c6c78002bb4717556a7e87859d4926a13 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 29 Jul 2022 14:05:51 +0500 Subject: [PATCH 192/489] Add label on machine to differentiate between the tool used for managing machine configurations (#1382) Signed-off-by: Waleed Malik --- pkg/admission/machines.go | 9 +++++++++ pkg/controller/util/machine.go | 3 +++ 2 files changed, 12 insertions(+) diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index 1e8972fec..21e4f0111 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -27,6 +27,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider" + controllerutil "github.com/kubermatic/machine-controller/pkg/controller/util" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -97,6 +98,14 @@ func (ad *admissionData) mutateMachines(ctx context.Context, ar admissionv1.Admi common.SetOSLabel(&machine.Spec, string(providerConfig.OperatingSystem)) } + // Set LegacyMachineControllerUserDataLabel to false if OSM was used for managing the machine configuration. + if ad.useOSM { + if machine.Labels == nil { + machine.Labels = make(map[string]string) + } + machine.Labels[controllerutil.LegacyMachineControllerUserDataLabel] = "false" + } + return createAdmissionResponse(machineOriginal, &machine) } diff --git a/pkg/controller/util/machine.go b/pkg/controller/util/machine.go index a8e9b090a..06afef450 100644 --- a/pkg/controller/util/machine.go +++ b/pkg/controller/util/machine.go @@ -26,6 +26,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +// LegacyMachineControllerUserDataLabel is set to true when machine-controller is used for managing machine configuration. +const LegacyMachineControllerUserDataLabel = "machine.clusters.k8s.io/legacy-machine-controller-user-data" + func GetMachineDeploymentNameAndRevisionForMachine(ctx context.Context, machine *clusterv1alpha1.Machine, c client.Client) (string, string, error) { var ( machineSetName string From d4f1f10ea635a6d4cade03c2244cdf338893f5b6 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 29 Jul 2022 15:44:52 +0500 Subject: [PATCH 193/489] Update OSM to v0.6.0 (#1383) Signed-off-by: Waleed Malik --- examples/operating-system-manager.yaml | 7 ++----- go.mod | 4 +--- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index 2640b5cdb..150871828 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -1076,9 +1076,7 @@ spec: spec: serviceAccountName: operating-system-manager-webhook containers: - # TODO: Update this to a semver tag before release. - # - image: quay.io/kubermatic/operating-system-manager:952f1cd42007d6770b8e2805b0e18247377b14bb # Ref: https://github.com/kubermatic/operating-system-manager/commit/952f1cd42007d6770b8e2805b0e18247377b14bb - - image: quay.io/kubermatic/operating-system-manager:952f1cd42007d6770b8e2805b0e18247377b14bb + - image: quay.io/kubermatic/operating-system-manager:v0.6.0 imagePullPolicy: IfNotPresent name: webhook command: @@ -1379,8 +1377,7 @@ spec: spec: serviceAccountName: operating-system-manager containers: - # - image: quay.io/kubermatic/operating-system-manager:952f1cd42007d6770b8e2805b0e18247377b14bb # Ref: https://github.com/kubermatic/operating-system-manager/commit/952f1cd42007d6770b8e2805b0e18247377b14bb - - image: quay.io/kubermatic/operating-system-manager:952f1cd42007d6770b8e2805b0e18247377b14bb + - image: quay.io/kubermatic/operating-system-manager:v0.6.0 imagePullPolicy: IfNotPresent name: operating-system-manager command: diff --git a/go.mod b/go.mod index ba74e39bf..6ece1c6ae 100644 --- a/go.mod +++ b/go.mod @@ -43,10 +43,8 @@ require ( google.golang.org/grpc v1.45.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 - // TODO: Update this to a semver tag before release. - // Ref: https://github.com/kubermatic/operating-system-manager/commit/952f1cd42007d6770b8e2805b0e18247377b14bb // Please ensure that you update the image tags in `examples/operating-system-manager.yaml` as well. - k8c.io/operating-system-manager v0.5.1-0.20220727101546-952f1cd42007 + k8c.io/operating-system-manager v0.6.0 k8s.io/api v0.24.2 k8s.io/apiextensions-apiserver v0.24.2 k8s.io/apimachinery v0.24.2 diff --git a/go.sum b/go.sum index a6d5e4720..b138f343f 100644 --- a/go.sum +++ b/go.sum @@ -1521,8 +1521,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8c.io/operating-system-manager v0.5.1-0.20220727101546-952f1cd42007 h1:a6+IbIxdm63VNW5D1jvTLJBlcPy3KviyQWh6nJzogv4= -k8c.io/operating-system-manager v0.5.1-0.20220727101546-952f1cd42007/go.mod h1:8Q1xpjJomTG9X6lfx/y3+yGHCackHtqxuYEk0TIPMfA= +k8c.io/operating-system-manager v0.6.0 h1:c+WJOV+BlW9NgSi7/QCNKCTXVwcW89s3PlWQDqQBRhA= +k8c.io/operating-system-manager v0.6.0/go.mod h1:8Q1xpjJomTG9X6lfx/y3+yGHCackHtqxuYEk0TIPMfA= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.24.2 h1:g518dPU/L7VRLxWfcadQn2OnsiGWVOadTLpdnqgY2OI= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= From ef73c1b8f334dd924310065209ea89a3d8c8b719 Mon Sep 17 00:00:00 2001 From: Mara Sophie Grosch Date: Mon, 1 Aug 2022 14:07:48 +0200 Subject: [PATCH 194/489] Implement ProviderID() for Anexia (#1385) Signed-off-by: Mara Sophie Grosch --- pkg/cloudprovider/provider/anexia/instance.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/cloudprovider/provider/anexia/instance.go b/pkg/cloudprovider/provider/anexia/instance.go index c8cf6400c..cc9a9e607 100644 --- a/pkg/cloudprovider/provider/anexia/instance.go +++ b/pkg/cloudprovider/provider/anexia/instance.go @@ -45,9 +45,8 @@ func (ai *anexiaInstance) ID() string { return ai.info.Identifier } -// TODO(xmudrii): Implement this. func (ai *anexiaInstance) ProviderID() string { - return "" + return ai.ID() } func (ai *anexiaInstance) Addresses() map[string]v1.NodeAddressType { From 73327a59cfdd95ccd0a5ac2852b45a5a16ae7c0f Mon Sep 17 00:00:00 2001 From: Sachin Tiptur <56350000+sachintiptur@users.noreply.github.com> Date: Mon, 1 Aug 2022 19:31:44 +0200 Subject: [PATCH 195/489] Fixed IPv6 string in machine object to align with hetzner ccm (#1390) Signed-off-by: Sachin Tiptur --- pkg/cloudprovider/provider/hetzner/provider.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 81dcb476d..23c4dddfe 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -546,7 +546,12 @@ func (s *hetznerServer) Addresses() map[string]v1.NodeAddressType { addresses[privateNetwork.IP.String()] = v1.NodeInternalIP } addresses[s.server.PublicNet.IPv4.IP.String()] = v1.NodeExternalIP - addresses[s.server.PublicNet.IPv6.IP.String()] = v1.NodeExternalIP + // For a given IPv6 network of 2001:db8:1234::/64, the instance address is 2001:db8:1234::1 + // Reference: https://github.com/hetznercloud/hcloud-cloud-controller-manager/blob/v1.12.1/hcloud/instances.go#L165-167 + if !s.server.PublicNet.IPv6.IP.IsUnspecified() { + s.server.PublicNet.IPv6.IP[len(s.server.PublicNet.IPv6.IP)-1] |= 0x01 + addresses[s.server.PublicNet.IPv6.IP.String()] = v1.NodeExternalIP + } return addresses } From a7237249d5dc671f96719d97529da56caee022ba Mon Sep 17 00:00:00 2001 From: Sankalp Rangare Date: Wed, 3 Aug 2022 12:59:40 +0200 Subject: [PATCH 196/489] fix kubevirt secondary disk naming (#1389) Signed-off-by: Sankalp Rangare --- .../provider/kubevirt/provider.go | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 27175f320..63c8ba9c5 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -140,6 +140,7 @@ type NodeAffinityPreset struct { } type SecondaryDisks struct { + Name string Size resource.Quantity StorageClassName string } @@ -282,7 +283,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p config.DNSConfig = rawConfig.VirtualMachine.DNSConfig } config.SecondaryDisks = make([]SecondaryDisks, 0, len(rawConfig.VirtualMachine.Template.SecondaryDisks)) - for _, sd := range rawConfig.VirtualMachine.Template.SecondaryDisks { + for i, sd := range rawConfig.VirtualMachine.Template.SecondaryDisks { sdSizeString, err := p.configVarResolver.GetConfigVarStringValue(sd.Size) if err != nil { return nil, nil, fmt.Errorf(`failed to parse "secondaryDisks.size" field: %w`, err) @@ -297,6 +298,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.storageClass" field: %w`, err) } config.SecondaryDisks = append(config.SecondaryDisks, SecondaryDisks{ + Name: fmt.Sprintf("secondarydisk%d", i), Size: pvc, StorageClassName: scString, }) @@ -526,6 +528,8 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, dataVolumeName = machine.Name annotations map[string]string ) + // Add machineName as prefix to secondaryDisks. + addPrefixToSecondaryDisk(c.SecondaryDisks, dataVolumeName) if pc.OperatingSystem == providerconfigtypes.OperatingSystemFlatcar { annotations = map[string]string{ @@ -533,12 +537,6 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, } } - // we need this check until this issue is resolved: - // https://github.com/kubevirt/containerized-data-importer/issues/895 - if len(dataVolumeName) > 63 { - return nil, fmt.Errorf("dataVolumeName size %v, is bigger than 63 characters", len(dataVolumeName)) - } - defaultBridgeNetwork, err := defaultBridgeNetwork() if err != nil { return nil, fmt.Errorf("could not compute a random MAC address") @@ -667,9 +665,9 @@ func getVMDisks(config *Config) []kubevirtv1.Disk { DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: "virtio"}}, }, } - for i := range config.SecondaryDisks { + for _, sd := range config.SecondaryDisks { disks = append(disks, kubevirtv1.Disk{ - Name: "secondarydisk" + strconv.Itoa(i), + Name: sd.Name, DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: "virtio"}}, }) } @@ -707,12 +705,12 @@ func getVMVolumes(config *Config, dataVolumeName string, userDataSecretName stri }, }, } - for i := range config.SecondaryDisks { + for _, sd := range config.SecondaryDisks { volumes = append(volumes, kubevirtv1.Volume{ - Name: "secondarydisk" + strconv.Itoa(i), + Name: sd.Name, VolumeSource: kubevirtv1.VolumeSource{ DataVolume: &kubevirtv1.DataVolumeSource{ - Name: "secondarydisk" + strconv.Itoa(i), + Name: sd.Name, }}, }) } @@ -741,10 +739,10 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. }, }, } - for i, sd := range config.SecondaryDisks { + for _, sd := range config.SecondaryDisks { dataVolumeTemplates = append(dataVolumeTemplates, kubevirtv1.DataVolumeTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Name: "secondarydisk" + strconv.Itoa(i), + Name: sd.Name, }, Spec: cdiv1beta1.DataVolumeSpec{ PVC: &corev1.PersistentVolumeClaimSpec{ @@ -874,3 +872,9 @@ func hostnameAffinityTerm(matchKey, matchValue string) []corev1.PodAffinityTerm }, } } + +func addPrefixToSecondaryDisk(secondaryDisks []SecondaryDisks, prefix string) { + for i := range secondaryDisks { + secondaryDisks[i].Name = fmt.Sprintf("%s-%s", prefix, secondaryDisks[i].Name) + } +} From 7dbd142f06e711b106bb44cdcb7443da8d6e3c5a Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Thu, 4 Aug 2022 13:29:42 +0200 Subject: [PATCH 197/489] Add `publicIPSKU` field to Azure provider (#1396) * Add `publicIPSKU` field to Azure provider Signed-off-by: Marvin Beckers * Increase ignored complexity Signed-off-by: Marvin Beckers --- .golangci.yml | 2 +- pkg/cloudprovider/provider/azure/provider.go | 45 ++++++++++++++++++- .../provider/azure/types/types.go | 1 + 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 5425522a1..ede7af007 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -58,7 +58,7 @@ issues: - "cyclomatic complexity 31 of func `main` is high" - 'cyclomatic complexity 34 of func `\(\*provider\)\.getConfig` is high' - 'cyclomatic complexity 31 of func `\(\*provider\)\.Validate` is high' - - 'cyclomatic complexity 31 of func `\(\*provider\)\.Create` is high' + - 'cyclomatic complexity 32 of func `\(\*provider\)\.Create` is high' # SA1019: node.Spec.ConfigSource is deprecated: Previously used to specify the source of the node's configuration for the DynamicKubeletConfig feature. # This feature is removed from Kubelets as of 1.24 and will be fully removed in 1.26. +optional # We still support setting dynamic kubelet config feature in machine-controller. Hence, ignoring this error. diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index ecf60fc54..a68813d8e 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -24,6 +24,8 @@ import ( "strings" "sync" "time" + "unicode" + "unicode/utf8" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" @@ -102,6 +104,7 @@ type config struct { DataDiskSKU *compute.StorageAccountTypes AssignPublicIP bool + PublicIPSKU *network.PublicIPAddressSkuName EnableAcceleratedNetworking *bool Tags map[string]string } @@ -325,6 +328,10 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p return nil, nil, fmt.Errorf("failed to get the value of \"assignPublicIP\" field, error = %w", err) } + if rawCfg.PublicIPSKU != nil { + c.PublicIPSKU = ipSkuPtr(*rawCfg.PublicIPSKU) + } + c.AssignAvailabilitySet = rawCfg.AssignAvailabilitySet c.EnableAcceleratedNetworking = rawCfg.EnableAcceleratedNetworking @@ -583,7 +590,10 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, ipFamily := providerCfg.Network.GetIPFamily() sku := network.PublicIPAddressSkuNameBasic - if ipFamily == util.DualStack { + + if config.PublicIPSKU != nil { + sku = *config.PublicIPSKU + } else if ipFamily == util.DualStack { // 1. Cannot specify basic sku PublicIp for an IPv6 network interface ipConfiguration. // 2. Different basic sku and standard sku public Ip resources in availability set is not allowed. // 1 & 2 means we have to use standard sku in dual-stack configuration. @@ -593,6 +603,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, // basic sku. sku = network.PublicIPAddressSkuNameStandard } + var publicIP, publicIPv6 *network.PublicIPAddress if config.AssignPublicIP { if err = data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { @@ -1036,6 +1047,23 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return fmt.Errorf(util.ErrUnknownNetworkFamily, f) } + if c.PublicIPSKU != nil { + valid := false + for _, sku := range network.PossiblePublicIPAddressSkuNameValues() { + if sku == *c.PublicIPSKU { + valid = true + } + } + + if !valid { + return fmt.Errorf("unknown public IP address SKU: %s", *c.PublicIPSKU) + } + + if providerConfig.Network.GetIPFamily() == util.DualStack && *c.PublicIPSKU == network.PublicIPAddressSkuNameBasic { + return fmt.Errorf("cannot use %s public IP address SKU with dualstack", network.PublicIPAddressSkuNameBasic) + } + } + vmClient, err := getVMClient(c) if err != nil { return fmt.Errorf("failed to (create) vm client: %w", err) @@ -1194,6 +1222,21 @@ func storageTypePtr(storageType string) *compute.StorageAccountTypes { return &storage } +func ipSkuPtr(ipSKU string) *network.PublicIPAddressSkuName { + // the correct Azure API representation is capitalized, so we do that even if the original input was all lowercase + sku := network.PublicIPAddressSkuName(upperFirst(ipSKU)) + return &sku +} + +func upperFirst(str string) string { + if str == "" { + return "" + } + + r, n := utf8.DecodeRuneInString(str) + return string(unicode.ToUpper(r)) + str[n:] +} + // supportsDiskSKU validates some disk SKU types against the chosen VM SKU / VM type. func supportsDiskSKU(vmSKU compute.ResourceSku, diskSKU compute.StorageAccountTypes, zones []string) error { // sanity check to make sure the Azure API did not return something bad diff --git a/pkg/cloudprovider/provider/azure/types/types.go b/pkg/cloudprovider/provider/azure/types/types.go index f5705b60c..a7b32de6a 100644 --- a/pkg/cloudprovider/provider/azure/types/types.go +++ b/pkg/cloudprovider/provider/azure/types/types.go @@ -50,6 +50,7 @@ type RawConfig struct { DataDiskSize int32 `json:"dataDiskSize"` DataDiskSKU *string `json:"dataDiskSKU,omitempty"` AssignPublicIP providerconfigtypes.ConfigVarBool `json:"assignPublicIP"` + PublicIPSKU *string `json:"publicIPSKU,omitempty"` Tags map[string]string `json:"tags,omitempty"` } From f1aca7106c557142cc1d5e81713f37794e4a243f Mon Sep 17 00:00:00 2001 From: csengerszabo Date: Mon, 8 Aug 2022 15:48:20 +0200 Subject: [PATCH 198/489] Unified PR template update (#1398) Signed-off-by: csengerszabo --- .github/PULL_REQUEST_TEMPLATE.md | 35 ++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 163cca41d..f035f3716 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,11 +1,29 @@ **What this PR does / why we need it**: -**Which issue(s) this PR fixes** *(optional, in `fixes #` format, will close the issue(s) when PR gets merged)*: +**Which issue(s) this PR fixes**: + Fixes # +**What type of PR is this?** + + **Special notes for your reviewer**: -**Optional Release Note**: +**Does this PR introduce a user-facing change? Then add your Release Note here**: +```documentation + +``` From a47be5575d5937482e83697525ec1ea38a70edb7 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Mon, 8 Aug 2022 16:59:20 +0200 Subject: [PATCH 199/489] fix sles description (#1399) Signed-off-by: Moath Qasim --- pkg/cloudprovider/provider/aws/provider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index ee28c0570..8b96c6b4d 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -153,13 +153,13 @@ var ( providerconfigtypes.OperatingSystemSLES: { awstypes.CPUArchitectureX86_64: { // Be as precise as possible - otherwise we might get a nightly dev build - description: "SUSE Linux Enterprise Server 15 SP1 (HVM, 64-bit, SSD-Backed)", + description: "SUSE Linux Enterprise Server 15 SP3 (HVM, 64-bit, SSD-Backed)", // The AWS marketplace ID from SLES owner: "013907871322", }, awstypes.CPUArchitectureARM64: { // Be as precise as possible - otherwise we might get a nightly dev build - description: "SUSE Linux Enterprise Server 15 SP1 (HVM, 64-bit, SSD-Backed)", + description: "SUSE Linux Enterprise Server 15 SP3 (HVM, 64-bit, SSD-Backed)", // The AWS marketplace ID from SLES owner: "013907871322", }, From fd0e157f3f661371a860610cac779bd211451106 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Mon, 8 Aug 2022 19:54:25 +0200 Subject: [PATCH 200/489] operating system compatability matrix update (#1400) Signed-off-by: Moath Qasim --- docs/operating-system.md | 111 +++------------------------------------ 1 file changed, 8 insertions(+), 103 deletions(-) diff --git a/docs/operating-system.md b/docs/operating-system.md index 998195a7a..beb70e6e6 100644 --- a/docs/operating-system.md +++ b/docs/operating-system.md @@ -9,11 +9,14 @@ | AWS | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | | Azure | ✓ | ✓ | ✓ | ✓ | x | x | ✓ | | Digitalocean | ✓ | ✓ | x | x | x | x | ✓ | +| Equinix Metal | ✓ | ✓ | ✓ | x | x | x | ✓ | | Google Cloud Platform | ✓ | x | x | x | x | x | x | | Hetzner | ✓ | ✓ | x | x | x | x | ✓ | -| Equinix Metal | ✓ | ✓ | x | x | x | x | x | +| KubeVirt | ✓ | ✓ | ✓ | ✓ | x | x | ✓ | +| Nutanix | ✓ | ✓ | x | x | x | x | x | | Openstack | ✓ | ✓ | ✓ | ✓ | x | x | ✓ | | VMware Cloud Director | ✓ | x | x | x | x | x | x | +| VSphere | ✓ | ✓ | ✓ | ✓ | x | x | ✓ | ## Configuring a operating system @@ -22,6 +25,7 @@ Allowed values: - `amzn2` - `centos` +- `flatcar` - `rhel` - `rockylinux` - `sles` @@ -38,106 +42,7 @@ Machine controller may work with other OS versions that are not listed in the ta |---|---| | AmazonLinux2 | 2.x | | CentOS | 7.4.x, 7.6.x, 7.7.x | -| RHEL | 8.0, 8.1 | +| RHEL | 8.x | | Rocky Linux | 8.5 | -| SLES | SLES 15 SP1 | -| Ubuntu | 18.04 LTS | - -### Ubuntu - -```yaml -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: machine1 - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerConfig: - value: - ... - operatingSystem: "ubuntu" - operatingSystemSpec: - # do a apt-get dist-upgrade on start and reboot if required - distUpgradeOnBoot: true -``` - -### Container Linux - -```yaml -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: machine1 - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerConfig: - value: - ... - operatingSystem: "flatcar" - operatingSystemSpec: - # disable auto update - disableAutoUpdate: true -``` - -### CentOS - -```yaml -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: machine1 - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerConfig: - value: - ... - operatingSystem: "centos" -``` +| SLES | SLES 15 SP3 | +| Ubuntu | 20.04 LTS | \ No newline at end of file From c53d6a43933e3aaaf6b725122926bf53ad290946 Mon Sep 17 00:00:00 2001 From: Marcin Franczyk Date: Wed, 10 Aug 2022 07:18:24 +0200 Subject: [PATCH 201/489] Align KubeVirt cloud config with upstream (#1401) KubeVirt kubeconfig doesn't have to be exposed to machines Signed-off-by: Marcin Franczyk --- pkg/cloudprovider/provider/kubevirt/provider.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 63c8ba9c5..2523c62f1 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -458,8 +458,9 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri if err != nil { return "", "", fmt.Errorf("failed to parse config: %w", err) } + cc := kubevirttypes.CloudConfig{ - Kubeconfig: c.Kubeconfig, + Namespace: c.Namespace, } ccs, err := cc.String() From 82cb5012d6c99dc9c9f1294dc956ddb80775cc59 Mon Sep 17 00:00:00 2001 From: Pratik Deoghare Date: Wed, 10 Aug 2022 11:45:23 +0530 Subject: [PATCH 202/489] enable DHCPv6 in RockyLinux (#1397) --- pkg/userdata/rockylinux/provider.go | 6 ++++++ pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml | 6 ++++++ .../rockylinux/testdata/kubelet-v1.23-aws-external.yaml | 6 ++++++ pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml | 6 ++++++ pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml | 6 ++++++ .../rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml | 6 ++++++ .../rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml | 6 ++++++ pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml | 6 ++++++ pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml | 6 ++++++ 9 files changed, 54 insertions(+) diff --git a/pkg/userdata/rockylinux/provider.go b/pkg/userdata/rockylinux/provider.go index 930984309..eebe3734e 100644 --- a/pkg/userdata/rockylinux/provider.go +++ b/pkg/userdata/rockylinux/provider.go @@ -233,6 +233,12 @@ write_files: {{ end }} {{ .ContainerRuntimeScript | indent 4 }} {{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml index 2e8b1aba4..2b58e6cc7 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml @@ -168,6 +168,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml index 1621348e7..fb1eafba0 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml @@ -168,6 +168,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml index 587dc6d17..e3082bbfc 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml @@ -168,6 +168,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml index bf9f1047e..721b11943 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml @@ -175,6 +175,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 5524b8cde..14cb1eabc 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -181,6 +181,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml index 931fdde4f..9273d0b8e 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -181,6 +181,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml index e701f5963..75d43c26f 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml @@ -173,6 +173,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml index 2b29d711a..16f21929b 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml @@ -167,6 +167,12 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + echo NETWORKING_IPV6=yes >> /etc/sysconfig/network + echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh From e1cae9aea6900fc82106ed31bbb51e1e3dc203d3 Mon Sep 17 00:00:00 2001 From: Pratik Deoghare Date: Wed, 10 Aug 2022 13:30:24 +0530 Subject: [PATCH 203/489] set IPv6AcceptRA=true for Flatcar (#1393) * set IPv6AcceptRA=true for Flatcar Fixes https://github.com/flatcar-linux/Flatcar/issues/384 * inline router advertisement setting; add comments --- pkg/userdata/flatcar/provider.go | 16 ++++++++++++++++ .../flatcar/testdata/cloud-init_v1.22.7.yaml | 9 +++++++++ .../flatcar/testdata/cloud-init_v1.23.5.yaml | 9 +++++++++ .../flatcar/testdata/cloud-init_v1.24.0.yaml | 9 +++++++++ pkg/userdata/flatcar/testdata/containerd.yaml | 9 +++++++++ .../flatcar/testdata/ignition_v1.22.7.json | 2 +- .../flatcar/testdata/ignition_v1.23.5.json | 2 +- .../flatcar/testdata/ignition_v1.24.0.json | 2 +- 8 files changed, 55 insertions(+), 3 deletions(-) diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index 8e89415bb..2e04d9d68 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -374,6 +374,13 @@ storage: inline: | {{ .NodeIPScript | indent 10 }} + - path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" + filesystem: root + mode: 0755 + contents: + inline: | + [Network] + IPv6AcceptRA=true - path: /etc/kubernetes/bootstrap-kubelet.conf filesystem: root mode: 0400 @@ -698,6 +705,15 @@ write_files: content: | {{ .NodeIPScript | indent 4 }} +- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" + permissions: "0755" + content: | + # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar + # so we enable IPv6 Router Advertisement here. + # See for details https://github.com/flatcar-linux/Flatcar/issues/384 + [Network] + IPv6AcceptRA=true + - path: /etc/kubernetes/bootstrap-kubelet.conf permissions: "0400" content: | diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml index e899ddd45..f990df896 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml @@ -301,6 +301,15 @@ write_files: fi +- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" + permissions: "0755" + content: | + # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar + # so we enable IPv6 Router Advertisement here. + # See for details https://github.com/flatcar-linux/Flatcar/issues/384 + [Network] + IPv6AcceptRA=true + - path: /etc/kubernetes/bootstrap-kubelet.conf permissions: "0400" content: | diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml index a222e8af1..5d1763617 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml @@ -299,6 +299,15 @@ write_files: fi +- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" + permissions: "0755" + content: | + # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar + # so we enable IPv6 Router Advertisement here. + # See for details https://github.com/flatcar-linux/Flatcar/issues/384 + [Network] + IPv6AcceptRA=true + - path: /etc/kubernetes/bootstrap-kubelet.conf permissions: "0400" content: | diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml index b1593ad03..82a947e25 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml @@ -298,6 +298,15 @@ write_files: fi +- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" + permissions: "0755" + content: | + # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar + # so we enable IPv6 Router Advertisement here. + # See for details https://github.com/flatcar-linux/Flatcar/issues/384 + [Network] + IPv6AcceptRA=true + - path: /etc/kubernetes/bootstrap-kubelet.conf permissions: "0400" content: | diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index 577086ee0..7599cc0bb 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -281,6 +281,15 @@ write_files: fi +- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" + permissions: "0755" + content: | + # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar + # so we enable IPv6 Router Advertisement here. + # See for details https://github.com/flatcar-linux/Flatcar/issues/384 + [Network] + IPv6AcceptRA=true + - path: /etc/kubernetes/bootstrap-kubelet.conf permissions: "0400" content: | diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json b/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json index 34834bf0b..fc865ed51 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.7%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.7%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json b/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json index 806865ad8..06911b23b 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json index 62a14b9df..cebdc9554 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file From 41d490b67867f352d7cbbf52a3a587e17f681333 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Mon, 15 Aug 2022 14:17:36 +0200 Subject: [PATCH 204/489] improve kubeconfig/gcp handling of credentials in e2e tests (#1404) * make it clear where and how to base64 encode the kubevirt kubeconfig * same goodness for GCP * fix weirdness * bring back weirdness * things * ffs vscode --- docs/cloud-provider.md | 7 ++++--- examples/gce-machinedeployment.yaml | 9 +++++++-- examples/kubevirt-machinedeployment.yaml | 6 +++--- pkg/cloudprovider/provider/gce/config.go | 16 +++++++++++----- pkg/cloudprovider/provider/gce/types/types.go | 1 + test/e2e/provisioning/all_e2e_test.go | 17 +++++++++++++++-- .../testdata/machinedeployment-gce.yaml | 5 +++-- .../testdata/machinedeployment-kubevirt.yaml | 2 +- 8 files changed, 45 insertions(+), 18 deletions(-) diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index c18284f43..75a065640 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -152,7 +152,8 @@ tags: ### machine.spec.providerConfig.cloudProviderSpec ```yaml -serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>" +# The service account needs to be base64-encoded. +serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>" # See https://cloud.google.com/compute/docs/regions-zones/ zone: "europe-west3-a" # See https://cloud.google.com/compute/docs/machine-types @@ -302,8 +303,8 @@ tags: ### machine.spec.providerConfig.cloudProviderSpec ```yaml -# kubeconfig to access KubeVirt cluster -kubeconfig: '<< KUBECONFIG >>' +# base64-encoded kubeconfig to access KubeVirt cluster +kubeconfig: '<< KUBECONFIG_BASE64 >>' # KubeVirt namespace namespace: kube-system # kubernetes storage class diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index 37c8eecb2..c7ab56ffa 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -6,8 +6,13 @@ metadata: name: machine-controller-gce namespace: kube-system type: Opaque -stringData: - serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>" +data: + # The base64 encoding here is only to satisfy Kubernetes' + # Secret storage and to prevent multiline string replacement + # issues if we used stringData here (because the GCP SA is + # a multiline JSON string). + serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>" + --- apiVersion: "cluster.k8s.io/v1alpha1" kind: MachineDeployment diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index a40b95703..c7d959a25 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -28,9 +28,9 @@ spec: cloudProviderSpec: auth: kubeconfig: - # Can also be set via the env var 'KUBEVIRT_KUBECONFIG' on the machine-controller - # If specified directly, this value should be a base64 encoded kubeconfig in either yaml or json format. - value: "<< KUBECONFIG >>" + # Can also be set via the env var 'KUBEVIRT_KUBECONFIG' on the machine-controller. + # If instead specified directly, this value should be a base64 encoded kubeconfig. + value: "<< KUBECONFIG_BASE64 >>" virtualMachine: template: cpus: "1" diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index cacea48e6..934adca06 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -226,17 +226,23 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide // postprocessServiceAccount processes the service account and creates a JWT configuration // out of it. func (cfg *config) postprocessServiceAccount() error { - sa, err := base64.StdEncoding.DecodeString(cfg.serviceAccount) - if err != nil { - return fmt.Errorf("failed to decode base64 service account: %w", err) + sa := cfg.serviceAccount + + // safely decode the service account, in case we did not read the value + // from a "known-safe" location (like the MachineDeployment), but from + // an environment variable. + decoded, err := base64.StdEncoding.DecodeString(cfg.serviceAccount) + if err == nil { + sa = string(decoded) } + sam := map[string]string{} - err = json.Unmarshal(sa, &sam) + err = json.Unmarshal([]byte(sa), &sam) if err != nil { return fmt.Errorf("failed unmarshalling service account: %w", err) } cfg.projectID = sam["project_id"] - cfg.jwtConfig, err = google.JWTConfigFromJSON(sa, compute.ComputeScope) + cfg.jwtConfig, err = google.JWTConfigFromJSON([]byte(sa), compute.ComputeScope) if err != nil { return fmt.Errorf("failed preparing JWT: %w", err) } diff --git a/pkg/cloudprovider/provider/gce/types/types.go b/pkg/cloudprovider/provider/gce/types/types.go index c1b059d8d..0ac0f3335 100644 --- a/pkg/cloudprovider/provider/gce/types/types.go +++ b/pkg/cloudprovider/provider/gce/types/types.go @@ -30,6 +30,7 @@ import ( // CloudProviderSpec contains the specification of the cloud provider taken // from the provider configuration. type CloudProviderSpec struct { + // ServiceAccount must be base64-encoded. ServiceAccount providerconfigtypes.ConfigVarString `json:"serviceAccount,omitempty"` Zone providerconfigtypes.ConfigVarString `json:"zone"` MachineType providerconfigtypes.ConfigVarString `json:"machineType"` diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index c19c979c6..eb13c126d 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -20,6 +20,7 @@ package provisioning import ( "context" + "encoding/base64" "flag" "fmt" "os" @@ -293,12 +294,23 @@ func TestKubevirtProvisioningE2E(t *testing.T) { selector := OsSelector("ubuntu", "centos", "flatcar", "rockylinux") params := []string{ - fmt.Sprintf("<< KUBECONFIG >>=%s", kubevirtKubeconfig), + fmt.Sprintf("<< KUBECONFIG_BASE64 >>=%s", safeBase64Encoding(kubevirtKubeconfig)), } runScenarios(t, selector, params, kubevirtManifest, fmt.Sprintf("kubevirt-%s", *testRunIdentifier)) } +// safeBase64Encoding takes a value and encodes it with base64 +// if it is not already encoded. +func safeBase64Encoding(value string) string { + // If there was no error, the original value was already encoded. + if _, err := base64.StdEncoding.DecodeString(value); err == nil { + return value + } + + return base64.StdEncoding.EncodeToString([]byte(value)) +} + func TestOpenstackProvisioningE2E(t *testing.T) { t.Parallel() @@ -708,8 +720,9 @@ func TestGCEProvisioningE2E(t *testing.T) { // Act. GCE does not support CentOS. selector := OsSelector("ubuntu") params := []string{ - fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT >>=%s", googleServiceAccount), + fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>=%s", safeBase64Encoding(googleServiceAccount)), } + runScenarios(t, selector, params, GCEManifest, fmt.Sprintf("gce-%s", *testRunIdentifier)) } diff --git a/test/e2e/provisioning/testdata/machinedeployment-gce.yaml b/test/e2e/provisioning/testdata/machinedeployment-gce.yaml index a2d9eb4d3..5fb0a6c82 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-gce.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-gce.yaml @@ -24,8 +24,9 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "gce" cloudProviderSpec: - # If empty, can be set via GOOGLE_SERVICE_ACCOUNT env var - serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>" + # If empty, can be set via GOOGLE_SERVICE_ACCOUNT env var. The environment variable + # should be plaintext. The value in the cloudProviderSpec however must be base64-encoded. + serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>" # See https://cloud.google.com/compute/docs/regions-zones/ zone: "europe-west3-a" # See https://cloud.google.com/compute/docs/machine-types diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index 29a42da25..5c66b5622 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -28,7 +28,7 @@ spec: cloudProviderSpec: auth: kubeconfig: - value: '<< KUBECONFIG >>' + value: '<< KUBECONFIG_BASE64 >>' virtualMachine: template: cpus: "1" From d24cab926eb21dd7545aafa19ae421f9896b2cba Mon Sep 17 00:00:00 2001 From: Mara Sophie Grosch Date: Mon, 15 Aug 2022 16:16:41 +0200 Subject: [PATCH 205/489] Anexia: extend disk configuration (#1402) * Anexia: cleanup Config/RawConfig attribute duplication types.Config and types.RawConfig contained the same members, with RawConfig using ConfigVars and Config storing the resolved data. This commit does some cleanup to ease adding new config values in the future. Signed-off-by: Mara Sophie Grosch * Anexia: extend disk configuration Adds a new config value "Disks", with each entry configuring the size and performance type of a single disk. The config format can be used for configuring multiple disks in the future, which is on our roadmap, but right now only a single disk is supported - with the added feature of defining the performance type. Signed-off-by: Mara Sophie Grosch * Anexia: update MachineDeployment example Signed-off-by: Mara Sophie Grosch * CI: disable OperationSystemManager for Anexia Signed-off-by: Mara Sophie Grosch Signed-off-by: Mara Sophie Grosch --- .prow/provider-anexia.yaml | 4 + examples/anexia-machinedeployment.yaml | 10 +- pkg/cloudprovider/provider/anexia/provider.go | 139 ++++++++++++++---- .../provider/anexia/provider_test.go | 98 ++++++++---- .../{utils/utils.go => reconcile_context.go} | 21 +-- .../provider/anexia/types/types.go | 28 ++-- 6 files changed, 218 insertions(+), 82 deletions(-) rename pkg/cloudprovider/provider/anexia/{utils/utils.go => reconcile_context.go} (70%) diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 96a7ab09e..807b1adfe 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -32,6 +32,10 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestAnexiaProvisioningE2E" + env: + # OperatingSystemManager does not yet support Anexia + - name: OPERATING_SYSTEM_MANAGER + value: "false" securityContext: privileged: true resources: diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index 156cfd21b..f5210a441 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -36,7 +36,15 @@ spec: locationID: "<< ANEXIA_LOCATION_ID >>" cpus: 2 memory: 2048 - diskSize: 60 + + # only a single disk is currently supported, but support for multiple disks is planned already + disks: + - size: 60 + performanceType: ENT6 + + # You may have this old disk config attribute in your config - please migrate to the disks attribute. + # For now it is still recognized though. + #diskSize: 60 # Flatcar is the only supported operating system operatingSystem: "flatcar" operatingSystemSpec: diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 7d451e84b..da8718cbf 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -37,7 +37,6 @@ import ( cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/utils" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -54,10 +53,37 @@ const ( ProvisionedType = "Provisioned" ) +var ( + // ErrConfigDiskSizeAndDisks is returned when the config has both DiskSize and Disks set, which is unsupported. + ErrConfigDiskSizeAndDisks = errors.New("both the deprecated DiskSize and new Disks attribute are set") + + // ErrMultipleDisksNotYetImplemented is returned when multiple disks are configured. + ErrMultipleDisksNotYetImplemented = errors.New("multiple disks configured, but this feature is not yet implemented") +) + type provider struct { configVarResolver *providerconfig.ConfigVarResolver } +// resolvedDisk contains the resolved values from types.RawDisk. +type resolvedDisk struct { + anxtypes.RawDisk + + PerformanceType string +} + +// resolvedConfig contains the resolved values from types.RawConfig. +type resolvedConfig struct { + anxtypes.RawConfig + + Token string + VlanID string + LocationID string + TemplateID string + + Disks []resolvedDisk +} + func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance instance.Instance, retErr error) { status := getProviderStatus(machine) klog.V(3).Infof(fmt.Sprintf("'%s' has status %#v", machine.Name, status)) @@ -70,10 +96,10 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, fmt.Errorf("unable to get provider config: %w", err) } - ctx = utils.CreateReconcileContext(ctx, utils.ReconcileContext{ + ctx = createReconcileContext(ctx, reconcileContext{ Status: &status, UserData: userdata, - Config: config, + Config: *config, ProviderData: data, Machine: machine, }) @@ -108,7 +134,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, } func waitForVM(ctx context.Context, client anxclient.Client) error { - reconcileContext := utils.GetReconcileContext(ctx) + reconcileContext := getReconcileContext(ctx) api := vsphere.NewAPI(client) var identifier string err := wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) { @@ -135,7 +161,7 @@ func waitForVM(ctx context.Context, client anxclient.Client) error { } func provisionVM(ctx context.Context, client anxclient.Client) error { - reconcileContext := utils.GetReconcileContext(ctx) + reconcileContext := getReconcileContext(ctx) vmAPI := vsphere.NewAPI(client) ctx, cancel := context.WithTimeout(ctx, anxtypes.CreateRequestTimeout) @@ -164,12 +190,15 @@ func provisionVM(ctx context.Context, client anxclient.Client) error { reconcileContext.Machine.Name, config.CPUs, config.Memory, - config.DiskSize, + config.Disks[0].Size, networkInterfaces, ) + vm.DiskType = config.Disks[0].PerformanceType + vm.Script = base64.StdEncoding.EncodeToString([]byte(reconcileContext.UserData)) + // We generate a fresh SSH key but will never actually use it - we just want a valid public key to disable password authentication for our fresh VM. sshKey, err := ssh.NewKey() if err != nil { return newError(common.CreateMachineError, "failed to generate ssh key: %v", err) @@ -220,7 +249,7 @@ func provisionVM(ctx context.Context, client anxclient.Client) error { } func getIPAddress(ctx context.Context, client anxclient.Client) (string, error) { - reconcileContext := utils.GetReconcileContext(ctx) + reconcileContext := getReconcileContext(ctx) status := reconcileContext.Status // only use IP if it is still unbound @@ -228,6 +257,7 @@ func getIPAddress(ctx context.Context, client anxclient.Client) (string, error) klog.Info("reusing already provisioned ip", "IP", status.ReservedIP) return status.ReservedIP, nil } + klog.Info(fmt.Sprintf("Creating a new IP for machine %q", reconcileContext.Machine.Name)) addrAPI := anxaddr.NewAPI(client) config := reconcileContext.Config @@ -251,7 +281,7 @@ func getIPAddress(ctx context.Context, client anxclient.Client) (string, error) } func isAlreadyProvisioning(ctx context.Context) bool { - status := utils.GetReconcileContext(ctx).Status + status := getReconcileContext(ctx).Status condition := meta.FindStatusCondition(status.Conditions, ProvisionedType) lastChange := condition.LastTransitionTime.Time const reasonInProvisioning = "InProvisioning" @@ -278,50 +308,85 @@ func ensureConditions(status *anxtypes.ProviderStatus) { } } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*anxtypes.Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerSpec.value is nil") +func (p *provider) resolveConfig(config anxtypes.RawConfig) (*resolvedConfig, error) { + var err error + ret := resolvedConfig{ + RawConfig: config, } - pconfig, err := providerconfigtypes.GetConfig(provSpec) + + ret.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(config.Token, anxtypes.AnxTokenEnv) if err != nil { - return nil, nil, err + return nil, fmt.Errorf("failed to get 'token': %w", err) } - if pconfig.OperatingSystemSpec.Raw == nil { - return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") + ret.LocationID, err = p.configVarResolver.GetConfigVarStringValue(config.LocationID) + if err != nil { + return nil, fmt.Errorf("failed to get 'locationID': %w", err) } - rawConfig, err := anxtypes.GetConfig(*pconfig) + ret.TemplateID, err = p.configVarResolver.GetConfigVarStringValue(config.TemplateID) if err != nil { - return nil, nil, err + return nil, fmt.Errorf("failed to get 'templateID': %w", err) } - c := anxtypes.Config{} - c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, anxtypes.AnxTokenEnv) + ret.VlanID, err = p.configVarResolver.GetConfigVarStringValue(config.VlanID) if err != nil { - return nil, nil, fmt.Errorf("failed to get 'token': %w", err) + return nil, fmt.Errorf("failed to get 'vlanID': %w", err) + } + + if config.DiskSize != 0 { + if len(config.Disks) != 0 { + return nil, ErrConfigDiskSizeAndDisks + } + + klog.Warningf("Configuration uses the deprecated DiskSize attribute, please migrate to the Disks array instead.") + + config.Disks = []anxtypes.RawDisk{ + { + Size: config.DiskSize, + }, + } + config.DiskSize = 0 } - c.CPUs = rawConfig.CPUs - c.Memory = rawConfig.Memory - c.DiskSize = rawConfig.DiskSize + ret.Disks = make([]resolvedDisk, len(config.Disks)) + + for idx, disk := range config.Disks { + ret.Disks[idx].RawDisk = disk - c.LocationID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.LocationID) + ret.Disks[idx].PerformanceType, err = p.configVarResolver.GetConfigVarStringValue(disk.PerformanceType) + if err != nil { + return nil, fmt.Errorf("failed to get 'performanceType' of disk %v: %w", idx, err) + } + } + + return &ret, nil +} + +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*resolvedConfig, *providerconfigtypes.Config, error) { + if provSpec.Value == nil { + return nil, nil, fmt.Errorf("machine.spec.providerSpec.value is nil") + } + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { - return nil, nil, fmt.Errorf("failed to get 'locationID': %w", err) + return nil, nil, err + } + + if pconfig.OperatingSystemSpec.Raw == nil { + return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } - c.TemplateID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.TemplateID) + rawConfig, err := anxtypes.GetConfig(*pconfig) if err != nil { - return nil, nil, fmt.Errorf("failed to get 'templateID': %w", err) + return nil, nil, fmt.Errorf("error parsing provider config: %w", err) } - c.VlanID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VlanID) + resolvedConfig, err := p.resolveConfig(*rawConfig) if err != nil { - return nil, nil, fmt.Errorf("failed to get 'vlanID': %w", err) + return nil, nil, fmt.Errorf("error resolving config: %w", err) } - return &c, pconfig, nil + return resolvedConfig, pconfig, nil } // New returns an Anexia provider. @@ -349,8 +414,18 @@ func (p *provider) Validate(_ context.Context, machinespec clusterv1alpha1.Machi return errors.New("cpu count is missing") } - if config.DiskSize == 0 { - return errors.New("disk size is missing") + if len(config.Disks) == 0 { + return errors.New("no disks configured") + } + + if len(config.Disks) > 1 { + return ErrMultipleDisksNotYetImplemented + } + + for _, disk := range config.Disks { + if disk.Size == 0 { + return errors.New("disk size is missing") + } } if config.Memory == 0 { diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index 8592f5764..b26d29f30 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -33,7 +33,6 @@ import ( "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/utils" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "k8s.io/apimachinery/pkg/api/meta" @@ -58,13 +57,13 @@ func TestAnexiaProvider(t *testing.T) { testhelper.Mux.HandleFunc("/api/vsphere/v1/search/by_name.json", createSearchHandler(t, waitUntilVMIsFound)) providerStatus := anxtypes.ProviderStatus{} - ctx := utils.CreateReconcileContext(context.Background(), utils.ReconcileContext{ + ctx := createReconcileContext(context.Background(), reconcileContext{ Machine: &v1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{Name: "TestMachine"}, }, Status: &providerStatus, UserData: "", - Config: &anxtypes.Config{}, + Config: resolvedConfig{}, ProviderData: &cloudprovidertypes.ProviderData{ Update: func(m *clusterv1alpha1.Machine, mod ...cloudprovidertypes.MachineModifier) error { @@ -150,19 +149,27 @@ func TestAnexiaProvider(t *testing.T) { }) providerStatus := anxtypes.ProviderStatus{} - ctx := utils.CreateReconcileContext(context.Background(), utils.ReconcileContext{ + ctx := createReconcileContext(context.Background(), reconcileContext{ Machine: &v1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{Name: "TestMachine"}, }, Status: &providerStatus, UserData: "", - Config: &anxtypes.Config{ + Config: resolvedConfig{ VlanID: "VLAN-ID", LocationID: "LOCATION-ID", TemplateID: "TEMPLATE-ID", - CPUs: 5, - Memory: 5, - DiskSize: 5, + Disks: []resolvedDisk{ + { + RawDisk: anxtypes.RawDisk{ + Size: 5, + }, + }, + }, + RawConfig: anxtypes.RawConfig{ + CPUs: 5, + Memory: 5, + }, }, ProviderData: &cloudprovidertypes.ProviderData{ Update: func(m *clusterv1alpha1.Machine, mods ...cloudprovidertypes.MachineModifier) error { @@ -186,10 +193,10 @@ func TestAnexiaProvider(t *testing.T) { }, }, } - ctx := utils.CreateReconcileContext(context.Background(), utils.ReconcileContext{ + ctx := createReconcileContext(context.Background(), reconcileContext{ Status: &providerStatus, UserData: "", - Config: nil, + Config: resolvedConfig{}, ProviderData: nil, }) @@ -214,7 +221,7 @@ func TestAnexiaProvider(t *testing.T) { ReservedIP: "", IPState: "", } - ctx := utils.CreateReconcileContext(context.Background(), utils.ReconcileContext{Status: providerStatus}) + ctx := createReconcileContext(context.Background(), reconcileContext{Status: providerStatus}) t.Run("with unbound reserved IP", func(t *testing.T) { expectedIP := "8.8.8.8" @@ -230,42 +237,79 @@ func TestAnexiaProvider(t *testing.T) { func TestValidate(t *testing.T) { t.Parallel() + // this generates a full config and allows hooking into it to e.g. remove a value + hookableConfig := func(hook func(*anxtypes.RawConfig)) anxtypes.RawConfig { + config := anxtypes.RawConfig{ + CPUs: 1, + + Memory: 2, + + Disks: []anxtypes.RawDisk{ + {Size: 5, PerformanceType: newConfigVarString("ENT6")}, + }, + + Token: newConfigVarString("test-token"), + VlanID: newConfigVarString("test-vlan"), + LocationID: newConfigVarString("test-location"), + TemplateID: newConfigVarString("test-template"), + } + + if hook != nil { + hook(&config) + } + + return config + } + var configCases []ConfigTestCase configCases = append(configCases, ConfigTestCase{ - Config: anxtypes.RawConfig{}, + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Token.Value = "" }), Error: errors.New("token is missing"), }, ConfigTestCase{ - Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN")}, + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.CPUs = 0 }), Error: errors.New("cpu count is missing"), }, ConfigTestCase{ - Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1}, + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Disks = []anxtypes.RawDisk{} }), + Error: errors.New("no disks configured"), + }, + ConfigTestCase{ + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.DiskSize = 10 }), + Error: ErrConfigDiskSizeAndDisks, + }, + ConfigTestCase{ + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Disks = append(c.Disks, anxtypes.RawDisk{Size: 10}) }), + Error: ErrMultipleDisksNotYetImplemented, + }, + ConfigTestCase{ + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Disks[0].Size = 0 }), Error: errors.New("disk size is missing"), }, ConfigTestCase{ - Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1, DiskSize: 5}, + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Memory = 0 }), Error: errors.New("memory size is missing"), }, ConfigTestCase{ - Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1, DiskSize: 5, Memory: 5}, + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.LocationID.Value = "" }), Error: errors.New("location id is missing"), }, ConfigTestCase{ - Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1, DiskSize: 5, Memory: 5, - LocationID: newConfigVarString("TLID")}, - Error: errors.New("template id is missing"), + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.TemplateID.Value = "" }), + Error: errors.New("template id is missing"), }, ConfigTestCase{ - Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1, DiskSize: 5, Memory: 5, - LocationID: newConfigVarString("LID"), TemplateID: newConfigVarString("TID")}, - Error: errors.New("vlan id is missing"), + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.VlanID.Value = "" }), + Error: errors.New("vlan id is missing"), }, ConfigTestCase{ - Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1, DiskSize: 5, Memory: 5, - LocationID: newConfigVarString("LID"), TemplateID: newConfigVarString("TID"), VlanID: newConfigVarString("VLAN")}, - Error: nil, + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.DiskSize = 10; c.Disks = []anxtypes.RawDisk{} }), + Error: nil, + }, + ConfigTestCase{ + Config: hookableConfig(nil), + Error: nil, }, ) @@ -273,7 +317,9 @@ func TestValidate(t *testing.T) { for _, testCase := range getSpecsForValidationTest(t, configCases) { err := provider.Validate(context.Background(), testCase.Spec) if testCase.ExpectedError != nil { - testhelper.AssertEquals(t, testCase.ExpectedError.Error(), err.Error()) + if !errors.Is(err, testCase.ExpectedError) { + testhelper.AssertEquals(t, testCase.ExpectedError.Error(), err.Error()) + } } else { testhelper.AssertEquals(t, testCase.ExpectedError, err) } diff --git a/pkg/cloudprovider/provider/anexia/utils/utils.go b/pkg/cloudprovider/provider/anexia/reconcile_context.go similarity index 70% rename from pkg/cloudprovider/provider/anexia/utils/utils.go rename to pkg/cloudprovider/provider/anexia/reconcile_context.go index 51a2a549b..dea3577c8 100644 --- a/pkg/cloudprovider/provider/anexia/utils/utils.go +++ b/pkg/cloudprovider/provider/anexia/reconcile_context.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package utils +package anexia import ( "context" @@ -26,24 +26,25 @@ import ( type contextKey byte -const MachineReconcileContextKey contextKey = 0 +const machineReconcileContextKey contextKey = 0 -type ReconcileContext struct { +type reconcileContext struct { Machine *v1alpha1.Machine Status *anxtypes.ProviderStatus UserData string - Config *anxtypes.Config + Config resolvedConfig ProviderData *cloudprovidertypes.ProviderData } -func CreateReconcileContext(ctx context.Context, cc ReconcileContext) context.Context { - return context.WithValue(ctx, MachineReconcileContextKey, cc) +func createReconcileContext(ctx context.Context, cc reconcileContext) context.Context { + return context.WithValue(ctx, machineReconcileContextKey, cc) } -func GetReconcileContext(ctx context.Context) ReconcileContext { - rawContext := ctx.Value(MachineReconcileContextKey) - if recContext, ok := rawContext.(ReconcileContext); ok { +func getReconcileContext(ctx context.Context) reconcileContext { + rawContext := ctx.Value(machineReconcileContextKey) + if recContext, ok := rawContext.(reconcileContext); ok { return recContext } - return ReconcileContext{} + + return reconcileContext{} } diff --git a/pkg/cloudprovider/provider/anexia/types/types.go b/pkg/cloudprovider/provider/anexia/types/types.go index dd0faca84..25547e9d4 100644 --- a/pkg/cloudprovider/provider/anexia/types/types.go +++ b/pkg/cloudprovider/provider/anexia/types/types.go @@ -46,14 +46,26 @@ var StatusUpdateFailed = cloudprovidererrors.TerminalError{ Message: "Unable to update the machine status", } +// RawDisk specifies a single disk, with some values maybe being fetched from secrets. +type RawDisk struct { + Size int `json:"size"` + PerformanceType providerconfigtypes.ConfigVarString `json:"performanceType"` +} + +// RawConfig contains all the configuration values for VMs to create, with some values maybe being fetched from secrets. type RawConfig struct { Token providerconfigtypes.ConfigVarString `json:"token,omitempty"` VlanID providerconfigtypes.ConfigVarString `json:"vlanID"` LocationID providerconfigtypes.ConfigVarString `json:"locationID"` TemplateID providerconfigtypes.ConfigVarString `json:"templateID"` - CPUs int `json:"cpus"` - Memory int `json:"memory"` - DiskSize int `json:"diskSize"` + + CPUs int `json:"cpus"` + Memory int `json:"memory"` + + // Deprecated, use Disks instead. + DiskSize int `json:"diskSize"` + + Disks []RawDisk `json:"disks"` } type ProviderStatus struct { @@ -65,16 +77,6 @@ type ProviderStatus struct { Conditions []v1.Condition `json:"conditions,omitempty"` } -type Config struct { - Token string - VlanID string - LocationID string - TemplateID string - CPUs int - Memory int - DiskSize int -} - func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { rawConfig := &RawConfig{} From 9cb3a6db4fcd1324881ca33f65c759adfa20d582 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 23 Aug 2022 17:44:48 +0500 Subject: [PATCH 206/489] Fix E2E tests for vSphere and OpenStack (#1412) Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- .prow/provider-vsphere.yaml | 2 +- examples/operating-system-manager.yaml | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- test/e2e/provisioning/all_e2e_test.go | 2 +- test/e2e/provisioning/helper.go | 10 +++++----- .../machinedeployment-vsphere-datastore-cluster.yaml | 4 ++-- .../machinedeployment-vsphere-resource-pool.yaml | 4 ++-- .../testdata/machinedeployment-vsphere-static-ip.yaml | 6 +++--- .../testdata/machinedeployment-vsphere.yaml | 6 +++--- 10 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 104d807c8..ece985542 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -20,7 +20,7 @@ presubmits: labels: preset-hetzner: "true" preset-e2e-ssh: "true" - preset-vsphere-legacy: "true" + preset-vsphere: "true" preset-rhel: "true" preset-goproxy: "true" preset-kind-volume-mounts: "true" diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index 150871828..9e513819b 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -1076,7 +1076,7 @@ spec: spec: serviceAccountName: operating-system-manager-webhook containers: - - image: quay.io/kubermatic/operating-system-manager:v0.6.0 + - image: quay.io/kubermatic/operating-system-manager:v1.0.0 imagePullPolicy: IfNotPresent name: webhook command: @@ -1377,7 +1377,7 @@ spec: spec: serviceAccountName: operating-system-manager containers: - - image: quay.io/kubermatic/operating-system-manager:v0.6.0 + - image: quay.io/kubermatic/operating-system-manager:v1.0.0 imagePullPolicy: IfNotPresent name: operating-system-manager command: diff --git a/go.mod b/go.mod index 6ece1c6ae..bc2ae6ae2 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 // Please ensure that you update the image tags in `examples/operating-system-manager.yaml` as well. - k8c.io/operating-system-manager v0.6.0 + k8c.io/operating-system-manager v1.0.0 k8s.io/api v0.24.2 k8s.io/apiextensions-apiserver v0.24.2 k8s.io/apimachinery v0.24.2 diff --git a/go.sum b/go.sum index b138f343f..a3615c58d 100644 --- a/go.sum +++ b/go.sum @@ -1521,8 +1521,8 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8c.io/operating-system-manager v0.6.0 h1:c+WJOV+BlW9NgSi7/QCNKCTXVwcW89s3PlWQDqQBRhA= -k8c.io/operating-system-manager v0.6.0/go.mod h1:8Q1xpjJomTG9X6lfx/y3+yGHCackHtqxuYEk0TIPMfA= +k8c.io/operating-system-manager v1.0.0 h1:E1dCaLHypgaaLNgm50jcT3uwk3vok3xWYOnFcspXJ38= +k8c.io/operating-system-manager v1.0.0/go.mod h1:8Q1xpjJomTG9X6lfx/y3+yGHCackHtqxuYEk0TIPMfA= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.24.2 h1:g518dPU/L7VRLxWfcadQn2OnsiGWVOadTLpdnqgY2OI= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index eb13c126d..205d82d32 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -871,7 +871,7 @@ func getVSphereTestParams(t *testing.T) []string { func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() - selector := Not(OsSelector("sles", "amzn2")) + selector := Not(OsSelector("sles", "amzn2", "rockylinux")) params := getVSphereTestParams(t) runScenarios(t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index cefad450f..f88254342 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -57,11 +57,11 @@ var ( } vSphereOSImageTemplates = map[string]string{ - string(providerconfigtypes.OperatingSystemCentOS): "machine-controller-e2e-centos", - string(providerconfigtypes.OperatingSystemFlatcar): "machine-controller-e2e-flatcar", - string(providerconfigtypes.OperatingSystemRHEL): "machine-controller-e2e-rhel", - string(providerconfigtypes.OperatingSystemRockyLinux): "machine-controller-e2e-rockylinux", - string(providerconfigtypes.OperatingSystemUbuntu): "machine-controller-e2e-ubuntu", + string(providerconfigtypes.OperatingSystemCentOS): "centos-7", + string(providerconfigtypes.OperatingSystemFlatcar): "flatcar-3033.2.2", + string(providerconfigtypes.OperatingSystemRHEL): "rhel-8.6", + string(providerconfigtypes.OperatingSystemRockyLinux): "rockylinux-8", + string(providerconfigtypes.OperatingSystemUbuntu): "ubuntu-20.04", } ) diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml index 3242f90e0..743faff47 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml @@ -27,8 +27,8 @@ spec: templateVMName: '<< OS_Image_Template >>' username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' - datacenter: 'dc-1' - folder: '/dc-1/vm/e2e-tests' + datacenter: 'Hamburg' + folder: '/Hamburg/vm/Kubermatic-dev' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically datastoreCluster: 'dsc-1' diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml index a54021366..13d8af421 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml @@ -27,8 +27,8 @@ spec: templateVMName: '<< OS_Image_Template >>' username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' - datacenter: 'dc-1' - folder: '/dc-1/vm/e2e-tests' + datacenter: 'Hamburg' + folder: '/Hamburg/vm/Kubermatic-dev' password: << VSPHERE_PASSWORD >> datastoreCluster: 'dsc-1' resourcePool: 'e2e-resource-pool' diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml index eac0bdcfa..7ebc21556 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml @@ -27,11 +27,11 @@ spec: templateVMName: '<< OS_Image_Template >>' username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' - datacenter: 'dc-1' - folder: '/dc-1/vm/e2e-tests' + datacenter: 'Hamburg' + folder: '/Hamburg/vm/Kubermatic-dev' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically - datastore: HS-FreeNAS + datastore: alpha1 allowInsecure: true cpus: 2 MemoryMB: 2048 diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml index 921d00669..e14f7f4a3 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml @@ -27,11 +27,11 @@ spec: templateVMName: '<< OS_Image_Template >>' username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' - datacenter: 'dc-1' - folder: '/dc-1/vm/e2e-tests' + datacenter: 'Hamburg' + folder: '/Hamburg/vm/Kubermatic-dev' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically - datastore: HS-FreeNAS + datastore: alpha1 cpus: 2 MemoryMB: 4096 diskSizeGB: << DISK_SIZE >> From e31742d1b5ab74d42b8a0e9041c1d0dbc082d545 Mon Sep 17 00:00:00 2001 From: Mara Sophie Grosch Date: Tue, 23 Aug 2022 18:09:24 +0200 Subject: [PATCH 207/489] Anexia: return reserved IP when instance IPs are requested (#1410) When creating a machine at, we always first reserve an IP address for it. Later when retrieving the addresses of that machine, we rely on the vminfo API, which has some delay as it retrieves the IPs from the running VM. This commit adds the reserved address to the list of addresses returned from an instance, which should reduce provisioning time a bit and make it more stable. Also fixes a long-standing TODO comment: marking internal IPs as internal. Signed-off-by: Mara Sophie Grosch Signed-off-by: Mara Sophie Grosch --- pkg/cloudprovider/provider/anexia/instance.go | 32 +++-- .../provider/anexia/instance_test.go | 127 ++++++++++++++++++ pkg/cloudprovider/provider/anexia/provider.go | 16 ++- 3 files changed, 161 insertions(+), 14 deletions(-) create mode 100644 pkg/cloudprovider/provider/anexia/instance_test.go diff --git a/pkg/cloudprovider/provider/anexia/instance.go b/pkg/cloudprovider/provider/anexia/instance.go index cc9a9e607..fa53467a1 100644 --- a/pkg/cloudprovider/provider/anexia/instance.go +++ b/pkg/cloudprovider/provider/anexia/instance.go @@ -17,6 +17,8 @@ limitations under the License. package anexia import ( + "net" + "go.anx.io/go-anxcloud/pkg/vsphere/info" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" @@ -26,7 +28,8 @@ import ( ) type anexiaInstance struct { - info *info.Info + info *info.Info + reservedAddresses []string } func (ai *anexiaInstance) Name() string { @@ -52,19 +55,30 @@ func (ai *anexiaInstance) ProviderID() string { func (ai *anexiaInstance) Addresses() map[string]v1.NodeAddressType { addresses := map[string]v1.NodeAddressType{} - if ai.info == nil { - return addresses + if ai.reservedAddresses != nil { + for _, reservedIP := range ai.reservedAddresses { + addresses[reservedIP] = v1.NodeExternalIP + } } - for _, network := range ai.info.Network { - for _, ip := range network.IPv4 { - addresses[ip] = v1.NodeExternalIP + if ai.info != nil { + for _, network := range ai.info.Network { + for _, ip := range network.IPv4 { + addresses[ip] = v1.NodeExternalIP + } + for _, ip := range network.IPv6 { + addresses[ip] = v1.NodeExternalIP + } } - for _, ip := range network.IPv6 { + } + + for ip := range addresses { + parsed := net.ParseIP(ip) + if parsed.IsPrivate() { + addresses[ip] = v1.NodeInternalIP + } else { addresses[ip] = v1.NodeExternalIP } - - // TODO mark RFC1918 and RFC4193 addresses as internal } return addresses diff --git a/pkg/cloudprovider/provider/anexia/instance_test.go b/pkg/cloudprovider/provider/anexia/instance_test.go new file mode 100644 index 000000000..0d7641a5f --- /dev/null +++ b/pkg/cloudprovider/provider/anexia/instance_test.go @@ -0,0 +1,127 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package anexia + +import ( + "testing" + + "github.com/gophercloud/gophercloud/testhelper" + + "go.anx.io/go-anxcloud/pkg/vsphere/info" + v1 "k8s.io/api/core/v1" +) + +func TestAnexiaInstance(t *testing.T) { + addressCheck := func(t *testing.T, testcase string, instance *anexiaInstance, expected map[string]v1.NodeAddressType) { + t.Run(testcase, func(t *testing.T) { + addresses := instance.Addresses() + + testhelper.AssertDeepEquals(t, expected, addresses) + }) + } + + t.Run("empty instance", func(t *testing.T) { + instance := anexiaInstance{} + addressCheck(t, "no addresses", &instance, map[string]v1.NodeAddressType{}) + }) + + t.Run("instance with only reservedAddresses set", func(t *testing.T) { + instance := anexiaInstance{ + reservedAddresses: []string{"10.0.0.2", "fda0:23::2", "8.8.8.8", "2001:db8::2"}, + } + + addressCheck(t, "expected addresses", &instance, map[string]v1.NodeAddressType{ + "10.0.0.2": v1.NodeInternalIP, + "fda0:23::2": v1.NodeInternalIP, + "8.8.8.8": v1.NodeExternalIP, + "2001:db8::2": v1.NodeExternalIP, + }) + }) + + t.Run("instance with only info set", func(t *testing.T) { + instance := anexiaInstance{ + info: &info.Info{ + Network: []info.Network{ + { + IPv4: []string{"10.0.0.2"}, + IPv6: []string{"fda0:23::2"}, + }, + { + IPv4: []string{"8.8.8.8"}, + IPv6: []string{"2001:db8::2"}, + }, + }, + }, + } + + addressCheck(t, "expected addresses", &instance, map[string]v1.NodeAddressType{ + "10.0.0.2": v1.NodeInternalIP, + "fda0:23::2": v1.NodeInternalIP, + "8.8.8.8": v1.NodeExternalIP, + "2001:db8::2": v1.NodeExternalIP, + }) + }) + + t.Run("instance with both reservedAddresses and info set, full overlapping set", func(t *testing.T) { + instance := anexiaInstance{ + reservedAddresses: []string{"10.0.0.2", "fda0:23::2", "8.8.8.8", "2001:db8::2"}, + info: &info.Info{ + Network: []info.Network{ + { + IPv4: []string{"10.0.0.2"}, + IPv6: []string{"fda0:23::2"}, + }, + { + IPv4: []string{"8.8.8.8"}, + IPv6: []string{"2001:db8::2"}, + }, + }, + }, + } + + addressCheck(t, "expected addresses", &instance, map[string]v1.NodeAddressType{ + "10.0.0.2": v1.NodeInternalIP, + "fda0:23::2": v1.NodeInternalIP, + "8.8.8.8": v1.NodeExternalIP, + "2001:db8::2": v1.NodeExternalIP, + }) + }) + + t.Run("instance with both reservedAddresses and info set, some overlap, each adding some", func(t *testing.T) { + instance := anexiaInstance{ + reservedAddresses: []string{"10.0.0.2", "8.8.8.8", "2001:db8::2"}, + info: &info.Info{ + Network: []info.Network{ + { + IPv4: []string{"10.0.0.2"}, + IPv6: []string{"fda0:23::2"}, + }, + { + IPv6: []string{"2001:db8::2"}, + }, + }, + }, + } + + addressCheck(t, "expected addresses", &instance, map[string]v1.NodeAddressType{ + "10.0.0.2": v1.NodeInternalIP, + "fda0:23::2": v1.NodeInternalIP, + "8.8.8.8": v1.NodeExternalIP, + "2001:db8::2": v1.NodeExternalIP, + }) + }) +} diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index da8718cbf..841723724 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -254,7 +254,7 @@ func getIPAddress(ctx context.Context, client anxclient.Client) (string, error) // only use IP if it is still unbound if status.ReservedIP != "" && status.IPState == anxtypes.IPStateUnbound { - klog.Info("reusing already provisioned ip", "IP", status.ReservedIP) + klog.Infof("reusing already provisioned ip %q", status.ReservedIP) return status.ReservedIP, nil } @@ -450,7 +450,7 @@ func (p *provider) Validate(_ context.Context, machinespec clusterv1alpha1.Machi func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, "failed to parse MachineSpec: %v", err) + return nil, newError(common.InvalidConfigurationMachineError, "failed to retrieve config: %v", err) } cli, err := getClient(config.Token) @@ -463,10 +463,17 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ if err != nil { return nil, newError(common.InvalidConfigurationMachineError, "failed to get machine status: %v", err) } + if status.InstanceID == "" { return nil, cloudprovidererrors.ErrInstanceNotFound } + instance := anexiaInstance{} + + if status.IPState == anxtypes.IPStateBound && status.ReservedIP != "" { + instance.reservedAddresses = []string{status.ReservedIP} + } + timeoutCtx, cancel := context.WithTimeout(ctx, anxtypes.GetRequestTimeout) defer cancel() @@ -474,10 +481,9 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ if err != nil { return nil, fmt.Errorf("failed get machine info: %w", err) } + instance.info = &info - return &anexiaInstance{ - info: &info, - }, nil + return &instance, nil } func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string, error) { From 38beb157cac62e08aa725f01beaae900db561782 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Wed, 24 Aug 2022 17:10:04 +0200 Subject: [PATCH 208/489] enable flatcar, rockylinux and rhel for azure (#1416) * enable flatcar, rockylinux and rhel for azure Signed-off-by: Moath Qasim * update vsphere os images Signed-off-by: Moath Qasim Signed-off-by: Moath Qasim --- test/e2e/provisioning/all_e2e_test.go | 2 +- test/e2e/provisioning/helper.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 205d82d32..252440656 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -629,7 +629,7 @@ func TestAzureProvisioningE2E(t *testing.T) { t.Fatal("unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") } - selector := Not(OsSelector("sles", "amzn2", "rhel", "rockylinux", "flatcar")) + selector := Not(OsSelector("sles", "amzn2")) // act params := []string{ fmt.Sprintf("<< AZURE_TENANT_ID >>=%s", azureTenantID), diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index f88254342..39812773a 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -57,11 +57,11 @@ var ( } vSphereOSImageTemplates = map[string]string{ - string(providerconfigtypes.OperatingSystemCentOS): "centos-7", - string(providerconfigtypes.OperatingSystemFlatcar): "flatcar-3033.2.2", - string(providerconfigtypes.OperatingSystemRHEL): "rhel-8.6", - string(providerconfigtypes.OperatingSystemRockyLinux): "rockylinux-8", - string(providerconfigtypes.OperatingSystemUbuntu): "ubuntu-20.04", + string(providerconfigtypes.OperatingSystemCentOS): "kkp-centos-7", + string(providerconfigtypes.OperatingSystemFlatcar): "kkp-flatcar-3033.2.2", + string(providerconfigtypes.OperatingSystemRHEL): "kkp-rhel-8.6", + string(providerconfigtypes.OperatingSystemRockyLinux): "kkp-rockylinux-8", + string(providerconfigtypes.OperatingSystemUbuntu): "kkp-ubuntu-20.04", } ) From 76ef02f416413a155cddb4803092944c29dc329d Mon Sep 17 00:00:00 2001 From: Mara Sophie Grosch Date: Wed, 24 Aug 2022 21:12:02 +0200 Subject: [PATCH 209/489] Makefile: fix test-unit-docker (#1406) GOFLAGS need to be passed with quotes to ensure they are not split over multiple arguments to `make`. Signed-off-by: Mara Sophie Grosch Signed-off-by: Mara Sophie Grosch --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bfb27d53b..feba53315 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,7 @@ test-unit-docker: -e GOCACHE=/cache \ -w /go/src/github.com/kubermatic/machine-controller \ golang:$(GO_VERSION) \ - make test-unit GOFLAGS=$(GOFLAGS) + make test-unit "GOFLAGS=$(GOFLAGS)" .PHONY: test-unit test-unit: From 84ceefcd2d334b8c04546536caad2bad8326a819 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Fri, 2 Sep 2022 14:05:24 +0200 Subject: [PATCH 210/489] Migrate AWS cloud provider to AWS Go SDK v2 (#1423) * Check in partially migrated code base Signed-off-by: Marvin Beckers * Finish migration to AWS SDK v2 Signed-off-by: Marvin Beckers * Fix linter issues Signed-off-by: Marvin Beckers * Bump memory to 7Gi to prevent random OOM kills Signed-off-by: Marvin Beckers * Move AWS SDK requires into shared block Signed-off-by: Marvin Beckers Signed-off-by: Marvin Beckers --- .prow/e2e-features.yaml | 16 +- .prow/provider-alibaba.yaml | 4 +- .prow/provider-anexia.yaml | 4 +- .prow/provider-aws.yaml | 40 +- .prow/provider-azure.yaml | 12 +- .prow/provider-digitalocean.yaml | 4 +- .prow/provider-equinix-metal.yaml | 4 +- .prow/provider-gcp.yaml | 4 +- .prow/provider-hetzner.yaml | 4 +- .prow/provider-kubevirt.yaml | 4 +- .prow/provider-linode.yaml | 4 +- .prow/provider-nutanix.yaml | 4 +- .prow/provider-openstack.yaml | 8 +- .prow/provider-scaleway.yaml | 4 +- .prow/provider-vmware-cloud-director.yaml | 4 +- .prow/provider-vsphere.yaml | 12 +- .prow/verify.yaml | 12 +- go.mod | 14 +- go.sum | 28 +- pkg/cloudprovider/provider/aws/provider.go | 348 +++++++++--------- pkg/cloudprovider/provider/aws/types/types.go | 4 +- 21 files changed, 279 insertions(+), 259 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index aa4c12907..4b7c43255 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -43,10 +43,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-custom-ca always_run: true @@ -72,10 +72,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-ubuntu-upgrade always_run: true @@ -100,10 +100,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-deployment-upgrade always_run: true @@ -127,7 +127,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index e0cfc2ed2..e15cbae29 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -38,7 +38,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 807b1adfe..86e5aeb5e 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -40,7 +40,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 1cc78bcc3..16861a4d3 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -37,10 +37,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-aws-legacy-userdata always_run: true @@ -69,10 +69,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-aws-arm always_run: true @@ -97,10 +97,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-aws-ebs-encryption-enabled always_run: true @@ -125,10 +125,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-aws-flatcar-containerd always_run: true @@ -153,10 +153,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-aws-spot-instance always_run: true @@ -182,10 +182,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-aws-sles always_run: false @@ -210,10 +210,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-aws-flatcar-coreos-cloud-init always_run: false @@ -238,10 +238,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-aws-centos8 always_run: false @@ -266,10 +266,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-aws-assume-role always_run: false @@ -294,7 +294,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 6bee8b9f2..9ffc50a52 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -37,10 +37,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-azure-custom-image-reference always_run: true @@ -66,10 +66,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-azure-redhat-satellite optional: true @@ -96,7 +96,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 45f53e484..293b3ee3e 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -36,7 +36,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 4193213d6..8c091eeb0 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -37,7 +37,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index d16d63ae7..ea87dbc94 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -37,7 +37,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 493507eec..88b05f5d0 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -35,7 +35,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 29184045a..606aead91 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -38,7 +38,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 2e3cf370f..f4403378c 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -37,7 +37,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index a1e28f26b..66b0ebd76 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -39,7 +39,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index a9205eeb4..a7034f4b4 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -37,10 +37,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-openstack-project-auth always_run: true @@ -66,7 +66,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 1a2f06e0c..5b6628d31 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -36,7 +36,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 32052adcd..ce78da462 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -38,7 +38,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index ece985542..75dcb5e59 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -37,10 +37,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-vsphere-datastore-cluster always_run: false @@ -66,10 +66,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-vsphere-resource-pool always_run: false @@ -95,7 +95,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 43de08569..edcdea58e 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -29,10 +29,10 @@ presubmits: - all resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-dependencies always_run: true @@ -71,9 +71,9 @@ presubmits: resources: requests: cpu: 800m - memory: 6Gi + memory: 7Gi limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-yamllint always_run: true @@ -177,6 +177,6 @@ presubmits: resources: requests: cpu: 3 - memory: 6Gi + memory: 7Gi limits: - memory: 6Gi + memory: 7Gi diff --git a/go.mod b/go.mod index bc2ae6ae2..4c59e5d3a 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,12 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 github.com/Masterminds/sprig/v3 v3.2.2 github.com/aliyun/alibaba-cloud-sdk-go v1.61.1645 - github.com/aws/aws-sdk-go v1.44.37 + github.com/aws/aws-sdk-go-v2 v1.16.12 + github.com/aws/aws-sdk-go-v2/config v1.17.3 + github.com/aws/aws-sdk-go-v2/credentials v1.12.16 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.54.2 + github.com/aws/aws-sdk-go-v2/service/sts v1.16.15 + github.com/aws/smithy-go v1.13.0 github.com/coreos/container-linux-config-transpiler v0.9.0 github.com/davecgh/go-spew v1.1.1 github.com/digitalocean/godo v1.81.0 @@ -74,6 +79,13 @@ require ( github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.13 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.19 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/coreos/go-semver v0.3.0 // indirect diff --git a/go.sum b/go.sum index a3615c58d..e9c854746 100644 --- a/go.sum +++ b/go.sum @@ -139,9 +139,33 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.44.37 h1:KvDxCX6dfJeEDC77U5GPGSP0ErecmNnhDHFxw+NIvlI= -github.com/aws/aws-sdk-go v1.44.37/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aws/aws-sdk-go-v2 v1.16.12 h1:wbMYa2PlFysFx2GLIQojr6FJV5+OWCM/BwyHXARxETA= +github.com/aws/aws-sdk-go-v2 v1.16.12/go.mod h1:C+Ym0ag2LIghJbXhfXZ0YEEp49rBWowxKzJLUoob0ts= +github.com/aws/aws-sdk-go-v2/config v1.17.3 h1:s1As/fiVMmM3CObC4GcSaSbkhm88S6a5qn8St3wgal0= +github.com/aws/aws-sdk-go-v2/config v1.17.3/go.mod h1:tRGUOfk9Rrf6UCJm5qDlL9AizSsgvteuKX4qajAV3pU= +github.com/aws/aws-sdk-go-v2/credentials v1.12.16 h1:HXczS88Pg36j8dq0KSjtHBPFs8gdRyBSS1hueeG/rxA= +github.com/aws/aws-sdk-go-v2/credentials v1.12.16/go.mod h1:eLJ+j1lwQdHJ0c56tRoDWcgss1e/laVmvW2AaOicuAw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.13 h1:+uferi8SUDZtMloCDt24Zenyy/i71C/ua5mjUCpbpN0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.13/go.mod h1:y0eXmsNBFIVjUE8ZBjES8myOHlMsXDz7qGT93+MVdjk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19 h1:gC5mudiFrWGhzcdoWj1iCGUfrzCpQG0MQIQf0CXFFQQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19/go.mod h1:llxE6bwUZhuCas0K7qGiu5OgMis3N7kdWtFSxoHmJ7E= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13 h1:qezY57na06d6kSE7uuB0N7XEflu914AXx/hg2L8Ykcw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13/go.mod h1:lB12mkZqCSo5PsdBFLNqc2M/OOYgNAy8UtaktyuWvE8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.20 h1:GvszACAU8GSV3+Tant5GutW6smY8WavrP8ZuRS9Ku4Q= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.20/go.mod h1:bfTcsThj5a9P5pIGRy0QudJ8k4+issxXX+O6Djnd5Cs= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.54.2 h1:rlqJWpugIyaw7UROyETCgao75pyiq2pkETFSg8oq+fU= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.54.2/go.mod h1:d1gxyomADOqOm0m9lGsr1m61ubU7lUyWhxkEeJBgPF4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.13 h1:ObfthqDyhe7rMAOa7pqft6974VHIk8BAJB7kYdoIfTA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.13/go.mod h1:V390DK4MQxLpDdXxFqizyz8KUxuWImkW/xzgXMz0yyk= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.19 h1:WdCwfJmu23XiIDeZwclSyAorQe916M3LeHd53xqBjfA= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.19/go.mod h1:ytmEi5+qwcSNcV2pVA8PIb1DnKT/0Bu/K4nfJHwoM6c= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.1 h1:p48IfndYbRk3iDsoQAmVXdCKEM5+7Y50JAPikjwk8gI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.1/go.mod h1:NY+G+8PW0ISyJ7/6t5mgOe6qpJiwZa9Jix05WPscJjg= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.15 h1:ApuR2BK9vf5/XXsImHBBsYJ6aUhmUhBHnZMPyhJo1jQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.16.15/go.mod h1:Y+BUV19q3OmQVqNUlbZ40zVi3NM6Biuxwkx/qdSD/CY= +github.com/aws/smithy-go v1.13.0 h1:YfyEmSJLo7fAv8FbuDK4R8F9aAmi9DZ88Zb/KJJmUl0= +github.com/aws/smithy-go v1.13.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 8b96c6b4d..5f56270ea 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -26,12 +26,15 @@ import ( "sync" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/sts" + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + awscredentials "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/aws/smithy-go" + gocache "github.com/patrickmn/go-cache" "github.com/prometheus/client_golang/prometheus" @@ -49,7 +52,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog" "k8s.io/utils/pointer" @@ -89,14 +91,14 @@ const ( ) var ( - volumeTypes = sets.NewString( - ec2.VolumeTypeStandard, - ec2.VolumeTypeIo1, - ec2.VolumeTypeGp2, - ec2.VolumeTypeGp3, - ec2.VolumeTypeSc1, - ec2.VolumeTypeSt1, - ) + volumeTypes = map[ec2types.VolumeType]interface{}{ + ec2types.VolumeTypeStandard: nil, + ec2types.VolumeTypeIo1: nil, + ec2types.VolumeTypeGp2: nil, + ec2types.VolumeTypeGp3: nil, + ec2types.VolumeTypeSc1: nil, + ec2types.VolumeTypeSt1: nil, + } amiFilters = map[providerconfigtypes.OperatingSystem]map[awstypes.CPUArchitecture]amiFilter{ // Source: https://wiki.centos.org/Cloud/AWS @@ -205,11 +207,11 @@ type Config struct { SubnetID string SecurityGroupIDs []string InstanceProfile string - InstanceType string + InstanceType ec2types.InstanceType AMI string - DiskSize int64 - DiskType string - DiskIops *int64 + DiskSize int32 + DiskType ec2types.VolumeType + DiskIops *int32 EBSVolumeEncrypted bool Tags map[string]string AssignPublicIP *bool @@ -229,7 +231,7 @@ type amiFilter struct { productCode string } -func getDefaultAMIID(client *ec2.EC2, os providerconfigtypes.OperatingSystem, region string, cpuArchitecture awstypes.CPUArchitecture) (string, error) { +func getDefaultAMIID(ctx context.Context, client *ec2.Client, os providerconfigtypes.OperatingSystem, region string, cpuArchitecture awstypes.CPUArchitecture) (string, error) { cacheLock.Lock() defer cacheLock.Unlock() @@ -251,35 +253,35 @@ func getDefaultAMIID(client *ec2.EC2, os providerconfigtypes.OperatingSystem, re } describeImagesInput := &ec2.DescribeImagesInput{ - Owners: aws.StringSlice([]string{filter.owner}), - Filters: []*ec2.Filter{ + Owners: []string{filter.owner}, + Filters: []ec2types.Filter{ { Name: aws.String("description"), - Values: aws.StringSlice([]string{filter.description}), + Values: []string{filter.description}, }, { Name: aws.String("virtualization-type"), - Values: aws.StringSlice([]string{"hvm"}), + Values: []string{"hvm"}, }, { Name: aws.String("root-device-type"), - Values: aws.StringSlice([]string{"ebs"}), + Values: []string{"ebs"}, }, { Name: aws.String("architecture"), - Values: aws.StringSlice([]string{string(cpuArchitecture)}), + Values: []string{string(cpuArchitecture)}, }, }, } if filter.productCode != "" { - describeImagesInput.Filters = append(describeImagesInput.Filters, &ec2.Filter{ + describeImagesInput.Filters = append(describeImagesInput.Filters, ec2types.Filter{ Name: aws.String("product-code"), - Values: aws.StringSlice([]string{filter.productCode}), + Values: []string{filter.productCode}, }) } - imagesOut, err := client.DescribeImages(describeImagesInput) + imagesOut, err := client.DescribeImages(ctx, describeImagesInput) if err != nil { return "", err } @@ -308,10 +310,10 @@ func getDefaultAMIID(client *ec2.EC2, os providerconfigtypes.OperatingSystem, re return *image.ImageId, nil } -func getCPUArchitecture(client *ec2.EC2, instanceType string) (awstypes.CPUArchitecture, error) { +func getCPUArchitecture(ctx context.Context, client *ec2.Client, instanceType ec2types.InstanceType) (awstypes.CPUArchitecture, error) { // read the instance type to know which cpu architecture is needed in the AMI - instanceTypes, err := client.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{ - InstanceTypes: []*string{aws.String(instanceType)}, + instanceTypes, err := client.DescribeInstanceTypes(ctx, &ec2.DescribeInstanceTypesInput{ + InstanceTypes: []ec2types.InstanceType{instanceType}, }) if err != nil { @@ -327,7 +329,7 @@ func getCPUArchitecture(client *ec2.EC2, instanceType string) (awstypes.CPUArchi for _, v := range instanceTypes.InstanceTypes[0].ProcessorInfo.SupportedArchitectures { // machine-controller currently supports x86_64 and ARM64, so only CPU architectures // that are supported will be returned if found in the AWS API response - if arch := awstypes.CPUArchitecture(*v); arch == awstypes.CPUArchitectureX86_64 || arch == awstypes.CPUArchitectureARM64 { + if arch := awstypes.CPUArchitecture(v); arch == awstypes.CPUArchitectureX86_64 || arch == awstypes.CPUArchitectureARM64 { return arch, nil } } @@ -414,20 +416,26 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if err != nil { return nil, nil, nil, err } - c.InstanceType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.InstanceType) + + instanceTypeStr, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.InstanceType) if err != nil { return nil, nil, nil, err } + + c.InstanceType = ec2types.InstanceType(instanceTypeStr) + c.AMI, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.AMI) if err != nil { return nil, nil, nil, err } c.DiskSize = rawConfig.DiskSize - c.DiskType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.DiskType) + diskTypeStr, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.DiskType) if err != nil { return nil, nil, nil, err } - if c.DiskType == ec2.VolumeTypeIo1 { + c.DiskType = ec2types.VolumeType(diskTypeStr) + + if c.DiskType == ec2types.VolumeTypeIo1 { if rawConfig.DiskIops == nil { return nil, nil, nil, errors.New("Missing required field `diskIops`") } @@ -438,7 +446,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } c.DiskIops = rawConfig.DiskIops - } else if c.DiskType == ec2.VolumeTypeGp3 && rawConfig.DiskIops != nil { + } else if c.DiskType == ec2types.VolumeTypeGp3 && rawConfig.DiskIops != nil { // gp3 disks start with 3000 IOPS by default, we _can_ pass better IOPS, but it is not a required field iops := *rawConfig.DiskIops @@ -489,58 +497,38 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return &c, pconfig, rawConfig, err } -func getSession(id, secret, token, region, assumeRoleARN, assumeRoleExternalID string) (*session.Session, error) { - config := aws.NewConfig() - config = config.WithRegion(region) - config = config.WithCredentials(credentials.NewStaticCredentials(id, secret, token)) - config = config.WithMaxRetries(maxRetries) - awsSession, err := session.NewSession(config) +func getAwsConfig(ctx context.Context, id, secret, token, region, assumeRoleARN, assumeRoleExternalID string) (aws.Config, error) { + cfg, err := awsconfig.LoadDefaultConfig(ctx, + awsconfig.WithRegion(region), + awsconfig.WithCredentialsProvider(awscredentials.NewStaticCredentialsProvider(id, secret, token)), + awsconfig.WithRetryMaxAttempts(maxRetries), + ) + if err != nil { - return nil, fmt.Errorf("failed to create AWS session: %w", err) + return aws.Config{}, err } - // Assume IAM role of e.g. external AWS account if configured if assumeRoleARN != "" { - awsSession, err = getAssumeRoleSession(awsSession, assumeRoleARN, assumeRoleExternalID, region) - if err != nil { - return nil, fmt.Errorf("failed to create temporary AWS session for assumed role: %w", err) - } - } - - return awsSession, err -} + stsSvc := sts.NewFromConfig(cfg) + creds := stscreds.NewAssumeRoleProvider(stsSvc, assumeRoleARN, + func(o *stscreds.AssumeRoleOptions) { + o.ExternalID = pointer.String(assumeRoleExternalID) + }, + ) -func getAssumeRoleSession(awsSession *session.Session, assumeRoleARN, assumeRoleExternalID, region string) (*session.Session, error) { - assumeRoleOutput, err := getAssumeRoleCredentials(awsSession, assumeRoleARN, assumeRoleExternalID) - if err != nil { - return nil, awsErrorToTerminalError(err, "unable to initialize aws external id session") + cfg.Credentials = creds } - assumedRoleConfig := aws.NewConfig() - assumedRoleConfig = assumedRoleConfig.WithRegion(region) - assumedRoleConfig = assumedRoleConfig.WithCredentials(credentials.NewStaticCredentials(*assumeRoleOutput.Credentials.AccessKeyId, - *assumeRoleOutput.Credentials.SecretAccessKey, - *assumeRoleOutput.Credentials.SessionToken)) - assumedRoleConfig = assumedRoleConfig.WithMaxRetries(maxRetries) - return session.NewSession(assumedRoleConfig) -} - -func getAssumeRoleCredentials(session *session.Session, assumeRoleARN, assumeRoleExternalID string) (*sts.AssumeRoleOutput, error) { - stsSession := sts.New(session) - sessionName := "kubermatic-machine-controller" - return stsSession.AssumeRole(&sts.AssumeRoleInput{ - ExternalId: &assumeRoleExternalID, - RoleArn: &assumeRoleARN, - RoleSessionName: &sessionName, - }) + return cfg, nil } -func getEC2client(id, secret, region, assumeRoleArn, assumeRoleExternalID string) (*ec2.EC2, error) { - sess, err := getSession(id, secret, "", region, assumeRoleArn, assumeRoleExternalID) +func getEC2client(ctx context.Context, id, secret, region, assumeRoleArn, assumeRoleExternalID string) (*ec2.Client, error) { + cfg, err := getAwsConfig(ctx, id, secret, "", region, assumeRoleArn, assumeRoleExternalID) if err != nil { - return nil, awsErrorToTerminalError(err, "failed to get aws session") + return nil, awsErrorToTerminalError(err, "failed to get aws configuration") } - return ec2.New(sess), nil + + return ec2.NewFromConfig(cfg), nil } func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { @@ -549,7 +537,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, err } if rawConfig.DiskType.Value == "" { - rawConfig.DiskType.Value = ec2.VolumeTypeStandard + rawConfig.DiskType.Value = string(ec2types.VolumeTypeStandard) } if rawConfig.AssignPublicIP == nil { rawConfig.AssignPublicIP = aws.Bool(true) @@ -564,7 +552,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, err } -func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { config, pc, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -574,7 +562,7 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) return fmt.Errorf("unsupported os %s", pc.OperatingSystem) } - if !volumeTypes.Has(config.DiskType) { + if _, ok := volumeTypes[config.DiskType]; !ok { return fmt.Errorf("invalid volume type %s specified. Supported: %s", config.DiskType, volumeTypes) } @@ -588,20 +576,20 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) return fmt.Errorf("diskSize must be specified and > 0") } - ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) + ec2Client, err := getEC2client(ctx, config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { return fmt.Errorf("failed to create ec2 client: %w", err) } if config.AMI != "" { - _, err := ec2Client.DescribeImages(&ec2.DescribeImagesInput{ - ImageIds: aws.StringSlice([]string{config.AMI}), + _, err := ec2Client.DescribeImages(ctx, &ec2.DescribeImagesInput{ + ImageIds: []string{config.AMI}, }) if err != nil { return fmt.Errorf("failed to validate ami: %w", err) } } - vpc, err := getVpc(ec2Client, config.VpcID) + vpc, err := getVpc(ctx, ec2Client, config.VpcID) if err != nil { return fmt.Errorf("invalid vpc %q specified: %w", config.VpcID, err) } @@ -611,18 +599,18 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) // noop case util.IPv6, util.DualStack: if len(vpc.Ipv6CidrBlockAssociationSet) == 0 { - return fmt.Errorf("vpc %q does not have IPv6 CIDR block", aws.StringValue(vpc.VpcId)) + return fmt.Errorf("vpc %s does not have IPv6 CIDR block", pointer.StringDeref(vpc.VpcId, "")) } default: return fmt.Errorf(util.ErrUnknownNetworkFamily, f) } - _, err = ec2Client.DescribeAvailabilityZones(&ec2.DescribeAvailabilityZonesInput{ZoneNames: aws.StringSlice([]string{config.AvailabilityZone})}) + _, err = ec2Client.DescribeAvailabilityZones(ctx, &ec2.DescribeAvailabilityZonesInput{ZoneNames: []string{config.AvailabilityZone}}) if err != nil { return fmt.Errorf("invalid zone %q specified: %w", config.AvailabilityZone, err) } - _, err = ec2Client.DescribeRegions(&ec2.DescribeRegionsInput{RegionNames: aws.StringSlice([]string{config.Region})}) + _, err = ec2Client.DescribeRegions(ctx, &ec2.DescribeRegionsInput{RegionNames: []string{config.Region}}) if err != nil { return fmt.Errorf("invalid region %q specified: %w", config.Region, err) } @@ -630,8 +618,8 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) if len(config.SecurityGroupIDs) == 0 { return errors.New("no security groups were specified") } - _, err = ec2Client.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ - GroupIds: aws.StringSlice(config.SecurityGroupIDs), + _, err = ec2Client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{ + GroupIds: config.SecurityGroupIDs, }) if err != nil { return fmt.Errorf("failed to validate security group id's: %w", err) @@ -650,10 +638,10 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) return nil } -func getVpc(client *ec2.EC2, id string) (*ec2.Vpc, error) { - vpcOut, err := client.DescribeVpcs(&ec2.DescribeVpcsInput{ - Filters: []*ec2.Filter{ - {Name: aws.String("vpc-id"), Values: []*string{aws.String(id)}}, +func getVpc(ctx context.Context, client *ec2.Client, id string) (*ec2types.Vpc, error) { + vpcOut, err := client.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{ + Filters: []ec2types.Filter{ + {Name: aws.String("vpc-id"), Values: []string{id}}, }, }) @@ -665,10 +653,10 @@ func getVpc(client *ec2.EC2, id string) (*ec2.Vpc, error) { return nil, fmt.Errorf("unable to find specified vpc with id %q", id) } - return vpcOut.Vpcs[0], nil + return &vpcOut.Vpcs[0], nil } -func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -677,7 +665,7 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d } } - ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) + ec2Client, err := getEC2client(ctx, config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { return nil, err } @@ -690,7 +678,7 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d amiID := config.AMI if amiID == "" { // read the instance type to know which cpu architecture is needed in the AMI - cpuArchitecture, err := getCPUArchitecture(ec2Client, config.InstanceType) + cpuArchitecture, err := getCPUArchitecture(ctx, ec2Client, config.InstanceType) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -699,7 +687,7 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d } } - if amiID, err = getDefaultAMIID(ec2Client, pc.OperatingSystem, config.Region, cpuArchitecture); err != nil { + if amiID, err = getDefaultAMIID(ctx, ec2Client, pc.OperatingSystem, config.Region, cpuArchitecture); err != nil { return nil, cloudprovidererrors.TerminalError{ Reason: common.InvalidConfigurationMachineError, Message: fmt.Sprintf("Failed to get AMI-ID for operating system %s in region %s: %v", pc.OperatingSystem, config.Region, err), @@ -715,7 +703,7 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d } } - tags := []*ec2.Tag{ + tags := []ec2types.Tag{ { Key: aws.String(nameTag), Value: aws.String(machine.Spec.Name), @@ -727,16 +715,16 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d } for k, v := range config.Tags { - tags = append(tags, &ec2.Tag{ + tags = append(tags, ec2types.Tag{ Key: aws.String(k), Value: aws.String(v), }) } - var instanceMarketOptions *ec2.InstanceMarketOptionsRequest + var instanceMarketOptions *ec2types.InstanceMarketOptionsRequest if config.IsSpotInstance != nil && *config.IsSpotInstance { - spotOpts := &ec2.SpotMarketOptions{ - SpotInstanceType: pointer.StringPtr(ec2.SpotInstanceTypeOneTime), + spotOpts := &ec2types.SpotMarketOptions{ + SpotInstanceType: ec2types.SpotInstanceTypeOneTime, } if config.SpotMaxPrice != nil && *config.SpotMaxPrice != "" { @@ -744,16 +732,16 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d } if config.SpotPersistentRequest != nil && *config.SpotPersistentRequest { - spotOpts.SpotInstanceType = pointer.StringPtr(ec2.SpotInstanceTypePersistent) - spotOpts.InstanceInterruptionBehavior = pointer.StringPtr(ec2.InstanceInterruptionBehaviorStop) + spotOpts.SpotInstanceType = ec2types.SpotInstanceTypePersistent + spotOpts.InstanceInterruptionBehavior = ec2types.InstanceInterruptionBehaviorStop if config.SpotInterruptionBehavior != nil && *config.SpotInterruptionBehavior != "" { - spotOpts.InstanceInterruptionBehavior = config.SpotInterruptionBehavior + spotOpts.InstanceInterruptionBehavior = ec2types.InstanceInterruptionBehavior(*config.SpotInterruptionBehavior) } } - instanceMarketOptions = &ec2.InstanceMarketOptionsRequest{ - MarketType: aws.String(ec2.MarketTypeSpot), + instanceMarketOptions = &ec2types.InstanceMarketOptionsRequest{ + MarketType: ec2types.MarketTypeSpot, SpotOptions: spotOpts, } } @@ -765,63 +753,63 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d instanceRequest := &ec2.RunInstancesInput{ ImageId: aws.String(amiID), InstanceMarketOptions: instanceMarketOptions, - BlockDeviceMappings: []*ec2.BlockDeviceMapping{ + BlockDeviceMappings: []ec2types.BlockDeviceMapping{ { DeviceName: aws.String(rootDevicePath), - Ebs: &ec2.EbsBlockDevice{ - VolumeSize: aws.Int64(config.DiskSize), + Ebs: &ec2types.EbsBlockDevice{ + VolumeSize: aws.Int32(config.DiskSize), DeleteOnTermination: aws.Bool(true), - VolumeType: aws.String(config.DiskType), + VolumeType: config.DiskType, Iops: config.DiskIops, Encrypted: pointer.BoolPtr(config.EBSVolumeEncrypted), }, }, }, - MaxCount: aws.Int64(1), - MinCount: aws.Int64(1), - InstanceType: aws.String(config.InstanceType), + MaxCount: aws.Int32(1), + MinCount: aws.Int32(1), + InstanceType: config.InstanceType, UserData: aws.String(base64.StdEncoding.EncodeToString([]byte(userdata))), - Placement: &ec2.Placement{ + Placement: &ec2types.Placement{ AvailabilityZone: aws.String(config.AvailabilityZone), }, - NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{ + NetworkInterfaces: []ec2types.InstanceNetworkInterfaceSpecification{ { - DeviceIndex: aws.Int64(0), // eth0 + DeviceIndex: aws.Int32(0), // eth0 AssociatePublicIpAddress: aws.Bool(assignPublicIP), DeleteOnTermination: aws.Bool(true), SubnetId: aws.String(config.SubnetID), - Groups: aws.StringSlice(config.SecurityGroupIDs), + Groups: config.SecurityGroupIDs, }, }, - IamInstanceProfile: &ec2.IamInstanceProfileSpecification{ + IamInstanceProfile: &ec2types.IamInstanceProfileSpecification{ Name: aws.String(config.InstanceProfile), }, - TagSpecifications: []*ec2.TagSpecification{ + TagSpecifications: []ec2types.TagSpecification{ { - ResourceType: aws.String(ec2.ResourceTypeInstance), + ResourceType: ec2types.ResourceTypeInstance, Tags: tags, }, }, } if pc.Network.GetIPFamily() == util.IPv6 || pc.Network.GetIPFamily() == util.DualStack { - instanceRequest.NetworkInterfaces[0].Ipv6AddressCount = aws.Int64(1) + instanceRequest.NetworkInterfaces[0].Ipv6AddressCount = aws.Int32(1) } - runOut, err := ec2Client.RunInstances(instanceRequest) + runOut, err := ec2Client.RunInstances(ctx, instanceRequest) if err != nil { return nil, awsErrorToTerminalError(err, "failed create instance at aws") } - if err = p.waitForInstance(machine); err != nil { + if err = p.waitForInstance(ctx, machine); err != nil { return nil, awsErrorToTerminalError(err, "failed provision instance at aws") } - return &awsInstance{instance: runOut.Instances[0]}, nil + return &awsInstance{instance: &runOut.Instances[0]}, nil } -func (p *provider) Cleanup(_ context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { - ec2instance, err := p.get(machine) +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { + ec2instance, err := p.get(ctx, machine) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return true, nil @@ -839,45 +827,45 @@ func (p *provider) Cleanup(_ context.Context, machine *clusterv1alpha1.Machine, } } - ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) + ec2Client, err := getEC2client(ctx, config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { return false, err } if config.IsSpotInstance != nil && *config.IsSpotInstance && config.SpotPersistentRequest != nil && *config.SpotPersistentRequest { - cOut, err := ec2Client.CancelSpotInstanceRequests(&ec2.CancelSpotInstanceRequestsInput{ - SpotInstanceRequestIds: aws.StringSlice([]string{*ec2instance.instance.SpotInstanceRequestId}), + cOut, err := ec2Client.CancelSpotInstanceRequests(ctx, &ec2.CancelSpotInstanceRequestsInput{ + SpotInstanceRequestIds: []string{*ec2instance.instance.SpotInstanceRequestId}, }) if err != nil { return false, awsErrorToTerminalError(err, "failed to cancel spot instance request") } - if *cOut.CancelledSpotInstanceRequests[0].State == ec2.CancelSpotInstanceRequestStateCancelled { + if cOut.CancelledSpotInstanceRequests[0].State == ec2types.CancelSpotInstanceRequestStateCancelled { klog.V(3).Infof("successfully canceled spot instance request %s at aws", *ec2instance.instance.SpotInstanceRequestId) } } - tOut, err := ec2Client.TerminateInstances(&ec2.TerminateInstancesInput{ - InstanceIds: aws.StringSlice([]string{ec2instance.ID()}), + tOut, err := ec2Client.TerminateInstances(ctx, &ec2.TerminateInstancesInput{ + InstanceIds: []string{ec2instance.ID()}, }) if err != nil { return false, awsErrorToTerminalError(err, "failed to terminate instance") } - if *tOut.TerminatingInstances[0].PreviousState.Name != *tOut.TerminatingInstances[0].CurrentState.Name { + if tOut.TerminatingInstances[0].PreviousState.Name != tOut.TerminatingInstances[0].CurrentState.Name { klog.V(3).Infof("successfully triggered termination of instance %s at aws", ec2instance.ID()) } return false, nil } -func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - return p.get(machine) +func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { + return p.get(ctx, machine) } -func (p *provider) get(machine *clusterv1alpha1.Machine) (*awsInstance, error) { +func (p *provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (*awsInstance, error) { config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -886,16 +874,16 @@ func (p *provider) get(machine *clusterv1alpha1.Machine) (*awsInstance, error) { } } - ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) + ec2Client, err := getEC2client(ctx, config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { return nil, err } - inOut, err := ec2Client.DescribeInstances(&ec2.DescribeInstancesInput{ - Filters: []*ec2.Filter{ + inOut, err := ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ + Filters: []ec2types.Filter{ { Name: aws.String("tag:" + machineUIDTag), - Values: aws.StringSlice([]string{string(machine.UID)}), + Values: []string{string(machine.UID)}, }, }, }) @@ -907,16 +895,12 @@ func (p *provider) get(machine *clusterv1alpha1.Machine) (*awsInstance, error) { // Thus we need to find the instance which is not in the terminated state for _, reservation := range inOut.Reservations { for _, i := range reservation.Instances { - if i.State == nil || i.State.Name == nil { - continue - } - - if *i.State.Name == ec2.InstanceStateNameTerminated { + if i.State == nil || i.State.Name == ec2types.InstanceStateNameTerminated { continue } return &awsInstance{ - instance: i, + instance: &i, }, nil } } @@ -951,7 +935,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err == nil { - labels["size"] = c.InstanceType + labels["size"] = string(c.InstanceType) labels["region"] = c.Region labels["az"] = c.AvailabilityZone labels["ami"] = c.AMI @@ -960,8 +944,8 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } -func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { - machineInstance, err := p.get(machine) +func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { + machineInstance, err := p.get(ctx, machine) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return nil @@ -977,14 +961,14 @@ func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machin } } - ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) + ec2Client, err := getEC2client(ctx, config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { return fmt.Errorf("failed to get EC2 client: %w", err) } - _, err = ec2Client.CreateTags(&ec2.CreateTagsInput{ - Resources: aws.StringSlice([]string{machineInstance.ID()}), - Tags: []*ec2.Tag{{Key: aws.String(machineUIDTag), Value: aws.String(string(newUID))}}}) + _, err = ec2Client.CreateTags(ctx, &ec2.CreateTagsInput{ + Resources: []string{machineInstance.ID()}, + Tags: []ec2types.Tag{{Key: aws.String(machineUIDTag), Value: aws.String(string(newUID))}}}) if err != nil { return fmt.Errorf("failed to update instance with new machineUIDTag: %w", err) } @@ -993,7 +977,7 @@ func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machin } type awsInstance struct { - instance *ec2.Instance + instance *ec2types.Instance } func (d *awsInstance) Name() string { @@ -1001,7 +985,7 @@ func (d *awsInstance) Name() string { } func (d *awsInstance) ID() string { - return aws.StringValue(d.instance.InstanceId) + return pointer.StringDeref(d.instance.InstanceId, "") } func (d *awsInstance) ProviderID() string { @@ -1017,15 +1001,15 @@ func (d *awsInstance) ProviderID() string { func (d *awsInstance) Addresses() map[string]v1.NodeAddressType { addresses := map[string]v1.NodeAddressType{ - aws.StringValue(d.instance.PublicIpAddress): v1.NodeExternalIP, - aws.StringValue(d.instance.PublicDnsName): v1.NodeExternalDNS, - aws.StringValue(d.instance.PrivateIpAddress): v1.NodeInternalIP, - aws.StringValue(d.instance.PrivateDnsName): v1.NodeInternalDNS, + pointer.StringDeref(d.instance.PublicIpAddress, ""): v1.NodeExternalIP, + pointer.StringDeref(d.instance.PublicDnsName, ""): v1.NodeExternalDNS, + pointer.StringDeref(d.instance.PrivateIpAddress, ""): v1.NodeInternalIP, + pointer.StringDeref(d.instance.PrivateDnsName, ""): v1.NodeInternalDNS, } for _, netInterface := range d.instance.NetworkInterfaces { for _, addr := range netInterface.Ipv6Addresses { - ipAddr := aws.StringValue(addr.Ipv6Address) + ipAddr := pointer.StringDeref(addr.Ipv6Address, "") // link-local addresses not very useful in machine status // filter them out @@ -1041,21 +1025,21 @@ func (d *awsInstance) Addresses() map[string]v1.NodeAddressType { } func (d *awsInstance) Status() instance.Status { - switch *d.instance.State.Name { - case ec2.InstanceStateNameRunning: + switch d.instance.State.Name { + case ec2types.InstanceStateNameRunning: return instance.StatusRunning - case ec2.InstanceStateNamePending: + case ec2types.InstanceStateNamePending: return instance.StatusCreating - case ec2.InstanceStateNameTerminated: + case ec2types.InstanceStateNameTerminated: return instance.StatusDeleted - case ec2.InstanceStateNameShuttingDown: + case ec2types.InstanceStateNameShuttingDown: return instance.StatusDeleting default: return instance.StatusUnknown } } -func getTagValue(name string, tags []*ec2.Tag) string { +func getTagValue(name string, tags []ec2types.Tag) string { for _, t := range tags { if *t.Key == name { return *t.Value @@ -1075,11 +1059,11 @@ func awsErrorToTerminalError(err error, msg string) error { } if err != nil { - var aerr awserr.Error + var aerr smithy.APIError if !errors.As(err, &aerr) { return prepareAndReturnError() } - switch aerr.Code() { + switch aerr.ErrorCode() { case "InstanceLimitExceeded": return cloudprovidererrors.TerminalError{ Reason: common.InsufficientResourcesMachineError, @@ -1130,6 +1114,8 @@ func setProviderSpec(rawConfig awstypes.RawConfig, provSpec clusterv1alpha1.Prov } func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { + ctx := context.Background() + metricInstancesForMachines.Reset() if len(machines.Items) < 1 { @@ -1163,14 +1149,14 @@ func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) e } } - allReservations := []*ec2.Reservation{} + allReservations := []ec2types.Reservation{} for _, cred := range machineEc2Credentials { - ec2Client, err := getEC2client(cred.accessKeyID, cred.secretAccessKey, cred.region, cred.assumeRoleARN, cred.assumeRoleExternalID) + ec2Client, err := getEC2client(ctx, cred.accessKeyID, cred.secretAccessKey, cred.region, cred.assumeRoleARN, cred.assumeRoleExternalID) if err != nil { machineErrors = append(machineErrors, fmt.Errorf("failed to get EC2 client: %w", err)) continue } - inOut, err := ec2Client.DescribeInstances(&ec2.DescribeInstancesInput{}) + inOut, err := ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{}) if err != nil { machineErrors = append(machineErrors, fmt.Errorf("failed to get EC2 instances: %w", err)) continue @@ -1190,13 +1176,11 @@ func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) e return nil } -func getInstanceCountForMachine(machine clusterv1alpha1.Machine, reservations []*ec2.Reservation) float64 { +func getInstanceCountForMachine(machine clusterv1alpha1.Machine, reservations []ec2types.Reservation) float64 { var count float64 for _, reservation := range reservations { for _, i := range reservation.Instances { - if i.State == nil || - i.State.Name == nil || - *i.State.Name == ec2.InstanceStateNameTerminated { + if i.State == nil || i.State.Name == ec2types.InstanceStateNameTerminated { continue } @@ -1215,8 +1199,8 @@ func getInstanceCountForMachine(machine clusterv1alpha1.Machine, reservations [] return count } -func filterSupportedRHELImages(images []*ec2.Image) ([]*ec2.Image, error) { - var filteredImages []*ec2.Image +func filterSupportedRHELImages(images []ec2types.Image) ([]ec2types.Image, error) { + var filteredImages []ec2types.Image for _, image := range images { if strings.HasPrefix(*image.Name, "RHEL-8") { filteredImages = append(filteredImages, image) @@ -1237,9 +1221,9 @@ func filterSupportedRHELImages(images []*ec2.Image) ([]*ec2.Image, error) { // That could result in two or more instances created for one Machine object. // This happens more often in some AWS regions because some regions have // slower instance creation (e.g. us-east-1 and us-west-2). -func (p *provider) waitForInstance(machine *clusterv1alpha1.Machine) error { +func (p *provider) waitForInstance(ctx context.Context, machine *clusterv1alpha1.Machine) error { return wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) { - _, err := p.get(machine) + _, err := p.get(ctx, machine) if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { // Retry if instance is not found return false, nil diff --git a/pkg/cloudprovider/provider/aws/types/types.go b/pkg/cloudprovider/provider/aws/types/types.go index ac03734da..243eb209c 100644 --- a/pkg/cloudprovider/provider/aws/types/types.go +++ b/pkg/cloudprovider/provider/aws/types/types.go @@ -36,9 +36,9 @@ type RawConfig struct { InstanceProfile providerconfigtypes.ConfigVarString `json:"instanceProfile,omitempty"` InstanceType providerconfigtypes.ConfigVarString `json:"instanceType,omitempty"` AMI providerconfigtypes.ConfigVarString `json:"ami,omitempty"` - DiskSize int64 `json:"diskSize"` + DiskSize int32 `json:"diskSize"` DiskType providerconfigtypes.ConfigVarString `json:"diskType,omitempty"` - DiskIops *int64 `json:"diskIops,omitempty"` + DiskIops *int32 `json:"diskIops,omitempty"` EBSVolumeEncrypted providerconfigtypes.ConfigVarBool `json:"ebsVolumeEncrypted"` Tags map[string]string `json:"tags,omitempty"` AssignPublicIP *bool `json:"assignPublicIP,omitempty"` From a7e95a2f320e9c4606323cd059b2d3c6c1e20c74 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 2 Sep 2022 18:29:24 +0500 Subject: [PATCH 211/489] Update to Go 1.19 (#1422) * Update to Go 1.19 Signed-off-by: Waleed Malik * Update license validation CI job Signed-off-by: Waleed Malik * refactored code Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- .prow/e2e-features.yaml | 8 +-- .prow/postsubmits.yaml | 4 +- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 20 +++---- .prow/provider-azure.yaml | 6 +-- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 +- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 6 +-- .prow/verify.yaml | 17 +++--- Dockerfile | 2 +- Makefile | 2 +- go.mod | 29 +++++------ go.sum | 52 +++++++++++-------- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 7 ++- pkg/admission/admission.go | 4 +- .../v1alpha1/conversions/conversions_test.go | 10 ++-- .../providerconfig_to_providerspec_test.go | 14 ++--- pkg/apis/cluster/v1alpha1/machine_types.go | 8 +-- .../cluster/v1alpha1/machineclass_types.go | 2 +- .../v1alpha1/machinedeployment_types.go | 10 ++-- pkg/apis/cluster/v1alpha1/machineset_types.go | 8 +-- .../plugins/tinkerbell/metadata/client.go | 9 +++- .../provider/openstack/types/cloudconfig.go | 6 +-- pkg/cloudprovider/provider/vsphere/helper.go | 9 ++-- .../util/cloud_init_settings_test.go | 4 +- pkg/cloudprovider/util/http.go | 4 +- pkg/clusterinfo/configmap.go | 4 +- pkg/controller/machine/machine_controller.go | 1 + pkg/controller/machinedeployment/sync.go | 8 +-- pkg/controller/util/machine_deployment.go | 13 ++--- pkg/machines/v1alpha1/types.go | 5 +- pkg/rhsm/satellite_subscription_manager.go | 10 ++-- .../satellite_subscription_manager_test.go | 3 +- pkg/rhsm/subscription_manager.go | 6 +-- pkg/test/helper.go | 6 +-- test/e2e/provisioning/verify.go | 4 +- 46 files changed, 171 insertions(+), 158 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 4b7c43255..537e8dd3d 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -91,7 +91,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -118,7 +118,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index baba3fd92..b2ab4d12b 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index e15cbae29..8745901bb 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 86e5aeb5e..78da9e24f 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 16861a4d3..99038333a 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -57,7 +57,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -88,7 +88,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -116,7 +116,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -144,7 +144,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -173,7 +173,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -201,7 +201,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -229,7 +229,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -257,7 +257,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -285,7 +285,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 9ffc50a52..075df153b 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -57,7 +57,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -87,7 +87,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 293b3ee3e..d58f2a01f 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 8c091eeb0..20a2ea8ee 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index ea87dbc94..ccd1180f8 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 88b05f5d0..d712ce4ae 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 606aead91..c54c43661 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index f4403378c..0289bb263 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 66b0ebd76..4775ec78f 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index a7034f4b4..74ab54f68 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -57,7 +57,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 5b6628d31..c56328077 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index ce78da462..361c1f1a6 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 75dcb5e59..682c8678e 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -57,7 +57,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -86,7 +86,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index edcdea58e..a6953b3ed 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.3 + - image: golang:1.19.0 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.3 + - image: golang:1.19.0 command: - make args: @@ -63,7 +63,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golangci/golangci-lint:v1.46.1 + - image: golangci/golangci-lint:v1.49.0 command: - make args: @@ -149,16 +149,13 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/wwhrd:0.4.0-1 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 command: - ./hack/verify-licenses.sh resources: requests: - memory: 32Mi - cpu: 50m - limits: - memory: 256Mi - cpu: 250m + memory: 2Gi + cpu: 2 - name: pull-machine-controller-test always_run: true @@ -168,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.3 + - image: golang:1.19.0 command: - make args: diff --git a/Dockerfile b/Dockerfile index 1a835fe9c..3e04a892c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.18.3 +ARG GO_VERSION=1.19.0 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index feba53315..21e8bcf74 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.18.3 +GO_VERSION ?= 1.19.0 GOOS ?= $(shell go env GOOS) diff --git a/go.mod b/go.mod index 4c59e5d3a..6518d124e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/kubermatic/machine-controller -go 1.18 +go 1.19 require ( cloud.google.com/go/logging v1.4.2 @@ -50,13 +50,13 @@ require ( gopkg.in/yaml.v3 v3.0.1 // Please ensure that you update the image tags in `examples/operating-system-manager.yaml` as well. k8c.io/operating-system-manager v1.0.0 - k8s.io/api v0.24.2 + k8s.io/api v0.25.0 k8s.io/apiextensions-apiserver v0.24.2 - k8s.io/apimachinery v0.24.2 + k8s.io/apimachinery v0.25.0 k8s.io/client-go v12.0.0+incompatible k8s.io/klog v1.0.0 k8s.io/kubelet v0.24.2 - k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 + k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed kubevirt.io/api v0.54.0 kubevirt.io/containerized-data-importer-api v1.50.0 sigs.k8s.io/controller-runtime v0.12.1 @@ -67,8 +67,8 @@ require ( cloud.google.com/go v0.100.2 // indirect cloud.google.com/go/compute v1.5.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.24 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect + github.com/Azure/go-autorest/autorest v0.11.27 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect @@ -145,9 +145,9 @@ require ( go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.21.0 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 // indirect + golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect + golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect @@ -160,14 +160,11 @@ require ( gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/component-base v0.24.2 // indirect - k8s.io/klog/v2 v2.60.1 // indirect - k8s.io/kube-openapi v0.0.0-20220614142933-1062c7ade5f8 // indirect + k8s.io/klog/v2 v2.70.1 // indirect + k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect - sigs.k8s.io/json v0.0.0-20220525155127-227cbc7cc124 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect + sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) -replace ( - k8s.io/client-go => k8s.io/client-go v0.24.2 - k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.24.2 -) +replace k8s.io/client-go => k8s.io/client-go v0.25.0 diff --git a/go.sum b/go.sum index e9c854746..0c6d23eaf 100644 --- a/go.sum +++ b/go.sum @@ -61,20 +61,21 @@ github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest v0.11.24 h1:1fIGgHKqVm54KIPT+q8Zmd1QlVsmHqeUGso5qm2BqqE= github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ= +github.com/Azure/go-autorest/autorest v0.11.27 h1:F3R3q42aWytozkV8ihzcgMO4OA4cuqr3bNlsEuF6//A= +github.com/Azure/go-autorest/autorest v0.11.27/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.20 h1:gJ3E98kMpFB1MFqQCvA1yFab8vthOeD4VlFRQULxahg= +github.com/Azure/go-autorest/autorest/adal v0.9.20/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 h1:P6bYXFoao05z5uhOQzbC3Qd8JqF3jUoocoTeIxkp2cA= github.com/Azure/go-autorest/autorest/azure/auth v0.5.11/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 h1:0W/yGmFdTIT77fvdlGZ0LMISoLHFJ7Tx4U0yeB+uFs4= github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= @@ -293,7 +294,6 @@ github.com/flatcar-linux/ignition v0.36.1 h1:yNvS9sQvm9HJ8VgxXskx88DsF73qdF35ALJ github.com/flatcar-linux/ignition v0.36.1/go.mod h1:0jS5n4AopgOdwgi7QDo5MFgkMx/fQUDYjuxlGJC1Txg= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= @@ -536,7 +536,7 @@ github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= @@ -604,7 +604,6 @@ github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czP github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -981,6 +980,7 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1079,8 +1079,8 @@ golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 h1:Yqz/iviulwKwAREEeUd3nbBFn0XuyJqkoft2IlrvOhc= -golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1205,8 +1205,9 @@ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c h1:aFV+BgZ4svzjfabn8ERpuB4JI4N6/rdy1iusx77G3oU= -golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= @@ -1548,16 +1549,18 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 k8c.io/operating-system-manager v1.0.0 h1:E1dCaLHypgaaLNgm50jcT3uwk3vok3xWYOnFcspXJ38= k8c.io/operating-system-manager v1.0.0/go.mod h1:8Q1xpjJomTG9X6lfx/y3+yGHCackHtqxuYEk0TIPMfA= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= -k8s.io/api v0.24.2 h1:g518dPU/L7VRLxWfcadQn2OnsiGWVOadTLpdnqgY2OI= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= +k8s.io/api v0.25.0 h1:H+Q4ma2U/ww0iGB78ijZx6DRByPz6/733jIuFpX70e0= +k8s.io/api v0.25.0/go.mod h1:ttceV1GyV1i1rnmvzT3BST08N6nGt+dudGrquzVQWPk= k8s.io/apiextensions-apiserver v0.24.2 h1:/4NEQHKlEz1MlaK/wHT5KMKC9UKYz6NZz6JE6ov4G6k= k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZfJiGOW6mm2hLQ= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.24.2 h1:5QlH9SL2C8KMcrNJPor+LbXVTaZRReml7svPEh4OKDM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= +k8s.io/apimachinery v0.25.0 h1:MlP0r6+3XbkUG2itd6vp3oxbtdQLQI94fD5gCS+gnoU= +k8s.io/apimachinery v0.25.0/go.mod h1:qMx9eAk0sZQGsXGu86fab8tZdffHbwUfsvzqKn4mfB0= k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI= -k8s.io/client-go v0.24.2 h1:CoXFSf8if+bLEbinDqN9ePIDGzcLtqhfd6jpfnwGOFA= -k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= +k8s.io/client-go v0.25.0 h1:CVWIaCETLMBNiTUta3d5nzRbXvY5Hy9Dpl+VvREpu5E= +k8s.io/client-go v0.25.0/go.mod h1:lxykvypVfKilxhTklov0wz1FoaUZ8X4EwbhS6rpRfN8= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/code-generator v0.24.2/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= k8s.io/component-base v0.24.2 h1:kwpQdoSfbcH+8MPN4tALtajLDfSfYxBDYlXobNWI6OU= @@ -1570,19 +1573,21 @@ k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= +k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= -k8s.io/kube-openapi v0.0.0-20220614142933-1062c7ade5f8 h1:IyQ1DifCBk589JD4Cm2CT2poIdO3lfPzz3WwVh1Ugf8= -k8s.io/kube-openapi v0.0.0-20220614142933-1062c7ade5f8/go.mod h1:guXtiQW/y/AWAfPSOaI/1eY0TGBAmL5OygiIyUOKDRc= +k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA= +k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= k8s.io/kubelet v0.24.2 h1:VAvULig8RiylCtyxudgHV7nhKsLnNIrdVBCRD4bXQ3Y= k8s.io/kubelet v0.24.2/go.mod h1:Xm9DkWQjwOs+uGOUIIGIPMvvmenvj0lDVOErvIKOOt0= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= +k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= kubevirt.io/api v0.54.0 h1:rVHaKrsxpYf5Cu6rhASOxNTChS76Nvtn5tArtG2M2Ds= kubevirt.io/api v0.54.0/go.mod h1:mK8ilpVLcZraqgo7hv2OSNQ5vdsA3G9Pxn8LY2/1+IY= kubevirt.io/containerized-data-importer-api v1.50.0 h1:O01F8L5K8qRLnkYICIfmAu0dU0P48jdO42uFPElht38= @@ -1597,11 +1602,12 @@ sigs.k8s.io/controller-runtime v0.12.1 h1:4BJY01xe9zKQti8oRjj/NeHKRXthf1YkYJAgLO sigs.k8s.io/controller-runtime v0.12.1/go.mod h1:BKhxlA4l7FPK4AQcsuL4X6vZeWnKDXez/vp1Y8dxTU0= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= -sigs.k8s.io/json v0.0.0-20220525155127-227cbc7cc124 h1:2sgAQQcY0dEW2SsQwTXhQV4vO6+rSslYx8K3XmM5hqQ= -sigs.k8s.io/json v0.0.0-20220525155127-227cbc7cc124/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index f91ab4a2f..077ab9420 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.18.3 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.19.0 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 2f6c3e087..cf0783724 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -17,9 +17,12 @@ set -euo pipefail cd $(dirname $0)/.. +source hack/lib.sh + +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 containerize ./hack/verify-licenses.sh go mod vendor -echo "Checking licenses..." +echodate "Checking licenses..." wwhrd check -q -echo "Check successful." +echodate "Check successful." diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go index 87259ac0b..a31ec388c 100644 --- a/pkg/admission/admission.go +++ b/pkg/admission/admission.go @@ -21,7 +21,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "reflect" "time" @@ -181,7 +181,7 @@ func readReview(r *http.Request) (*admissionv1.AdmissionReview, error) { if r.Body == nil { return nil, fmt.Errorf("request has no body") } - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { return nil, fmt.Errorf("error reading data from request body: %w", err) } diff --git a/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go b/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go index e16dabe96..fe87ba1b9 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go +++ b/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go @@ -20,7 +20,7 @@ import ( "bytes" "flag" "fmt" - "io/ioutil" + "os" "testing" "github.com/ghodss/yaml" @@ -34,13 +34,13 @@ import ( var update = flag.Bool("update", false, "update .testdata files") func getMachinesV1Alpha1TestMachines() (machines []machinesv1alpha1.Machine, err error) { - files, err := ioutil.ReadDir("testdata/machinesv1alpha1machine") + files, err := os.ReadDir("testdata/machinesv1alpha1machine") if err != nil { return nil, err } for _, file := range files { newMachine := &machinesv1alpha1.Machine{} - fileContent, err := ioutil.ReadFile(fmt.Sprintf("testdata/machinesv1alpha1machine/%s", file.Name())) + fileContent, err := os.ReadFile(fmt.Sprintf("testdata/machinesv1alpha1machine/%s", file.Name())) if err != nil { return nil, err } @@ -71,11 +71,11 @@ func TestMigratingMachine(t *testing.T) { t.Errorf("Failed to marshal machine: %v", err) } if *update { - if err = ioutil.WriteFile(fixtureFilePath, outMachineRaw, 0644); err != nil { + if err = os.WriteFile(fixtureFilePath, outMachineRaw, 0644); err != nil { t.Fatalf("Failed to write updated test fixture: %v", err) } } - expected, err := ioutil.ReadFile(fixtureFilePath) + expected, err := os.ReadFile(fixtureFilePath) if err != nil { t.Fatalf("Failed to read fixture: %v", err) } diff --git a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go b/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go index c6d58cb78..7f290e02a 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go +++ b/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go @@ -19,7 +19,7 @@ package conversions import ( "encoding/json" "fmt" - "io/ioutil" + "os" "testing" "github.com/ghodss/yaml" @@ -28,13 +28,13 @@ import ( ) func Test_Convert_MachineDeployment_ProviderConfig_To_ProviderSpec(t *testing.T) { - fixtures, err := ioutil.ReadDir("testdata/clusterv1alpha1machineDeploymentWithProviderConfig") + fixtures, err := os.ReadDir("testdata/clusterv1alpha1machineDeploymentWithProviderConfig") if err != nil { t.Fatalf("failed to list fixtures: %v", err) } for _, fixture := range fixtures { - fixtureYamlByte, err := ioutil.ReadFile(fmt.Sprintf("testdata/clusterv1alpha1machineDeploymentWithProviderConfig/%s", fixture.Name())) + fixtureYamlByte, err := os.ReadFile(fmt.Sprintf("testdata/clusterv1alpha1machineDeploymentWithProviderConfig/%s", fixture.Name())) if err != nil { t.Errorf("failed to read fixture file %s: %v", fixture.Name(), err) continue @@ -67,13 +67,13 @@ func Test_Convert_MachineDeployment_ProviderConfig_To_ProviderSpec(t *testing.T) } func Test_Convert_MachineSet_ProviderConfig_To_ProviderSpec(t *testing.T) { - fixtures, err := ioutil.ReadDir("testdata/clusterv1alpha1machineSetWithProviderConfig") + fixtures, err := os.ReadDir("testdata/clusterv1alpha1machineSetWithProviderConfig") if err != nil { t.Fatalf("failed to list fixtures: %v", err) } for _, fixture := range fixtures { - fixtureYamlByte, err := ioutil.ReadFile(fmt.Sprintf("testdata/clusterv1alpha1machineSetWithProviderConfig/%s", fixture.Name())) + fixtureYamlByte, err := os.ReadFile(fmt.Sprintf("testdata/clusterv1alpha1machineSetWithProviderConfig/%s", fixture.Name())) if err != nil { t.Errorf("failed to read fixture file %s: %v", fixture.Name(), err) continue @@ -107,13 +107,13 @@ func Test_Convert_MachineSet_ProviderConfig_To_ProviderSpec(t *testing.T) { } func Test_Convert_Machine_ProviderConfig_To_ProviderSpec(t *testing.T) { - fixtures, err := ioutil.ReadDir("testdata/clusterv1alpha1machineWithProviderConfig") + fixtures, err := os.ReadDir("testdata/clusterv1alpha1machineWithProviderConfig") if err != nil { t.Fatalf("failed to list fixtures: %v", err) } for _, fixture := range fixtures { - fixtureYamlByte, err := ioutil.ReadFile(fmt.Sprintf("testdata/clusterv1alpha1machineWithProviderConfig/%s", fixture.Name())) + fixtureYamlByte, err := os.ReadFile(fmt.Sprintf("testdata/clusterv1alpha1machineWithProviderConfig/%s", fixture.Name())) if err != nil { t.Errorf("failed to read fixture file %s: %v", fixture.Name(), err) continue diff --git a/pkg/apis/cluster/v1alpha1/machine_types.go b/pkg/apis/cluster/v1alpha1/machine_types.go index c6c9636f1..1308910d0 100644 --- a/pkg/apis/cluster/v1alpha1/machine_types.go +++ b/pkg/apis/cluster/v1alpha1/machine_types.go @@ -35,7 +35,7 @@ const ( // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -/// [Machine] +// / [Machine] // Machine is the Schema for the machines API // +k8s:openapi-gen=true // +kubebuilder:resource:shortName=ma @@ -53,7 +53,7 @@ type Machine struct { /// [Machine] -/// [MachineSpec] +// / [MachineSpec] // MachineSpec defines the desired state of Machine. type MachineSpec struct { // ObjectMeta will autopopulate the Node created. Use this to @@ -108,7 +108,7 @@ type MachineSpec struct { /// [MachineSpec] -/// [MachineStatus] +// / [MachineStatus] // MachineStatus defines the observed state of Machine. type MachineStatus struct { // NodeRef will point to the corresponding Node if it exists. @@ -224,7 +224,7 @@ type LastOperation struct { /// [MachineStatus] -/// [MachineVersionInfo] +// / [MachineVersionInfo] // Holds information regarding kubelet and controlplane versions for machine. type MachineVersionInfo struct { // Kubelet is the semantic version of kubelet to run diff --git a/pkg/apis/cluster/v1alpha1/machineclass_types.go b/pkg/apis/cluster/v1alpha1/machineclass_types.go index a63b62e15..b73553fbd 100644 --- a/pkg/apis/cluster/v1alpha1/machineclass_types.go +++ b/pkg/apis/cluster/v1alpha1/machineclass_types.go @@ -25,7 +25,7 @@ import ( // +genclient:noStatus // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -/// [MachineClass] +// / [MachineClass] // MachineClass can be used to templatize and re-use provider configuration // across multiple Machines / MachineSets / MachineDeployments. // +k8s:openapi-gen=true diff --git a/pkg/apis/cluster/v1alpha1/machinedeployment_types.go b/pkg/apis/cluster/v1alpha1/machinedeployment_types.go index 1455d7129..68aa5410d 100644 --- a/pkg/apis/cluster/v1alpha1/machinedeployment_types.go +++ b/pkg/apis/cluster/v1alpha1/machinedeployment_types.go @@ -23,7 +23,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) -/// [MachineDeploymentSpec] +// / [MachineDeploymentSpec] // MachineDeploymentSpec defines the desired state of MachineDeployment. type MachineDeploymentSpec struct { // Number of desired machines. Defaults to 1. @@ -70,7 +70,7 @@ type MachineDeploymentSpec struct { /// [MachineDeploymentSpec] -/// [MachineDeploymentStrategy] +// / [MachineDeploymentStrategy] // MachineDeploymentStrategy describes how to replace existing machines // with new ones. type MachineDeploymentStrategy struct { @@ -88,7 +88,7 @@ type MachineDeploymentStrategy struct { /// [MachineDeploymentStrategy] -/// [MachineRollingUpdateDeployment] +// / [MachineRollingUpdateDeployment] // Spec to control the desired behavior of rolling update. type MachineRollingUpdateDeployment struct { // The maximum number of machines that can be unavailable during the update. @@ -125,7 +125,7 @@ type MachineRollingUpdateDeployment struct { /// [MachineRollingUpdateDeployment] -/// [MachineDeploymentStatus] +// / [MachineDeploymentStatus] // MachineDeploymentStatus defines the observed state of MachineDeployment. type MachineDeploymentStatus struct { // The generation observed by the deployment controller. @@ -165,7 +165,7 @@ type MachineDeploymentStatus struct { // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -/// [MachineDeployment] +// / [MachineDeployment] // MachineDeployment is the Schema for the machinedeployments API // +k8s:openapi-gen=true // +kubebuilder:resource:shortName=md diff --git a/pkg/apis/cluster/v1alpha1/machineset_types.go b/pkg/apis/cluster/v1alpha1/machineset_types.go index 3d0e8adfb..fcd0bc1be 100644 --- a/pkg/apis/cluster/v1alpha1/machineset_types.go +++ b/pkg/apis/cluster/v1alpha1/machineset_types.go @@ -30,7 +30,7 @@ import ( // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -/// [MachineSet] +// / [MachineSet] // MachineSet ensures that a specified number of machines replicas are running at any given time. // +k8s:openapi-gen=true // +kubebuilder:resource:shortName=ms @@ -46,7 +46,7 @@ type MachineSet struct { /// [MachineSet] -/// [MachineSetSpec] +// / [MachineSetSpec] // MachineSetSpec defines the desired state of MachineSet. type MachineSetSpec struct { // Replicas is the number of desired replicas. @@ -103,7 +103,7 @@ const ( /// [MachineSetSpec] // doxygen marker -/// [MachineTemplateSpec] // doxygen marker +// / [MachineTemplateSpec] // doxygen marker // MachineTemplateSpec describes the data needed to create a Machine from a template. type MachineTemplateSpec struct { // Standard object's metadata. @@ -119,7 +119,7 @@ type MachineTemplateSpec struct { /// [MachineTemplateSpec] -/// [MachineSetStatus] +// / [MachineSetStatus] // MachineSetStatus defines the observed state of MachineSet. type MachineSetStatus struct { // Replicas is the most recently observed number of replicas. diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata/client.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata/client.go index 0ef4d0433..29ce3d9c8 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata/client.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata/client.go @@ -17,10 +17,11 @@ limitations under the License. package metadata import ( + "context" "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "time" ) @@ -80,6 +81,8 @@ func NewMetadataClient(cfg *Config) (Client, error) { func (d *defaultClient) GetMachineMetadata() (*MachineMetadata, error) { req, err := http.NewRequest(http.MethodGet, d.metadataEndpoint, nil) + // TODO: Fix this + req = req.WithContext(context.TODO()) if err != nil { return nil, fmt.Errorf("failed to create a get metadata request: %w", err) } @@ -92,10 +95,12 @@ func (d *defaultClient) GetMachineMetadata() (*MachineMetadata, error) { return nil, fmt.Errorf("failed to execute get metadata request: %w", err) } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { return nil, fmt.Errorf("failed to execute get metadata request with status code: %v", res.StatusCode) } - data, err := ioutil.ReadAll(res.Body) + data, err := io.ReadAll(res.Body) if err != nil { return nil, fmt.Errorf("failed to read response body: %w", err) } diff --git a/pkg/cloudprovider/provider/openstack/types/cloudconfig.go b/pkg/cloudprovider/provider/openstack/types/cloudconfig.go index 83899c292..8015fb2fa 100644 --- a/pkg/cloudprovider/provider/openstack/types/cloudconfig.go +++ b/pkg/cloudprovider/provider/openstack/types/cloudconfig.go @@ -27,9 +27,9 @@ import ( "github.com/kubermatic/machine-controller/pkg/ini" ) -// use-octavia is enabled by default in CCM since v1.17.0, and disabled by -// default with the in-tree cloud provider. -// https://v1-18.docs.kubernetes.io/docs/concepts/cluster-administration/cloud-providers/#load-balancer +// use-octavia is enabled by default in CCM since v1.17.0, and disabled by +// default with the in-tree cloud provider. +// https://v1-18.docs.kubernetes.io/docs/concepts/cluster-administration/cloud-providers/#load-balancer const ( cloudConfigTpl = `[Global] auth-url = {{ .Global.AuthURL | iniEscape }} diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go index 86ec839d7..a11ace772 100644 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ b/pkg/cloudprovider/provider/vsphere/helper.go @@ -22,7 +22,6 @@ import ( "encoding/base64" "errors" "fmt" - "io/ioutil" "math" "os" "os/exec" @@ -320,7 +319,7 @@ func uploadAndAttachISO(ctx context.Context, session *Session, vmRef *object.Vir func generateLocalUserdataISO(userdata, name string) (string, error) { // We must create a directory, because the iso-generation commands // take a directory as input - userdataDir, err := ioutil.TempDir(localTempDir, name) + userdataDir, err := os.MkdirTemp(localTempDir, name) if err != nil { return "", fmt.Errorf("failed to create local temp directory for userdata at %s: %w", userdataDir, err) } @@ -350,11 +349,11 @@ func generateLocalUserdataISO(userdata, name string) (string, error) { return "", fmt.Errorf("failed to render metadata: %w", err) } - if err := ioutil.WriteFile(userdataFilePath, []byte(userdata), 0644); err != nil { + if err := os.WriteFile(userdataFilePath, []byte(userdata), 0644); err != nil { return "", fmt.Errorf("failed to locally write userdata file to %s: %w", userdataFilePath, err) } - if err := ioutil.WriteFile(metadataFilePath, metadata.Bytes(), 0644); err != nil { + if err := os.WriteFile(metadataFilePath, metadata.Bytes(), 0644); err != nil { return "", fmt.Errorf("failed to locally write metadata file to %s: %w", userdataFilePath, err) } @@ -431,7 +430,7 @@ func validateDiskResizing(disks []*types.VirtualDisk, requestedSize int64) error return nil } -//getDatastoreFromVM gets the datastore where the VM files are located. +// getDatastoreFromVM gets the datastore where the VM files are located. func getDatastoreFromVM(ctx context.Context, session *Session, vmRef *object.VirtualMachine) (*object.Datastore, error) { var props mo.VirtualMachine // Obtain VM properties diff --git a/pkg/cloudprovider/util/cloud_init_settings_test.go b/pkg/cloudprovider/util/cloud_init_settings_test.go index 8a72079f2..991fdfb39 100644 --- a/pkg/cloudprovider/util/cloud_init_settings_test.go +++ b/pkg/cloudprovider/util/cloud_init_settings_test.go @@ -18,7 +18,7 @@ package util import ( "context" - "io/ioutil" + "os" "testing" corev1 "k8s.io/api/core/v1" @@ -60,7 +60,7 @@ func TestCloudInitGeneration(t *testing.T) { WithObjects(test.secret). Build() - userdata, err := ioutil.ReadFile(test.userdata) + userdata, err := os.ReadFile(test.userdata) if err != nil { t.Fatalf("failed to read userdata testing file: %v", err) } diff --git a/pkg/cloudprovider/util/http.go b/pkg/cloudprovider/util/http.go index 06ce4da53..253a48009 100644 --- a/pkg/cloudprovider/util/http.go +++ b/pkg/cloudprovider/util/http.go @@ -22,9 +22,9 @@ import ( "crypto/x509" "errors" "fmt" - "io/ioutil" "net/http" "net/http/httputil" + "os" "time" "github.com/google/uuid" @@ -45,7 +45,7 @@ var ( // global CABundle with a new one. The file must contain at least one // valid certificate. func SetCABundleFile(filename string) error { - content, err := ioutil.ReadFile(filename) + content, err := os.ReadFile(filename) if err != nil { return fmt.Errorf("failed to read file: %w", err) } diff --git a/pkg/clusterinfo/configmap.go b/pkg/clusterinfo/configmap.go index 02dfc19b4..399de944f 100644 --- a/pkg/clusterinfo/configmap.go +++ b/pkg/clusterinfo/configmap.go @@ -20,8 +20,8 @@ import ( "context" "errors" "fmt" - "io/ioutil" "net" + "os" "strconv" corev1 "k8s.io/api/core/v1" @@ -134,7 +134,7 @@ func getCAData(config *rest.Config) ([]byte, error) { return config.TLSClientConfig.CAData, nil } - return ioutil.ReadFile(config.TLSClientConfig.CAFile) + return os.ReadFile(config.TLSClientConfig.CAFile) } func (p *KubeconfigProvider) GetBearerToken() string { diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 9bd7472a4..eaab0e49c 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -686,6 +686,7 @@ func (r *Reconciler) deleteCloudProviderInstance(ctx context.Context, prov cloud if rhelConfig.RHELUseSatelliteServer { if kuberneteshelper.HasFinalizer(machine, rhsm.RedhatSubscriptionFinalizer) { err = r.satelliteSubscriptionManager.DeleteSatelliteHost( + ctx, machineName, rhelConfig.RHELSubscriptionManagerUser, rhelConfig.RHELSubscriptionManagerPassword, diff --git a/pkg/controller/machinedeployment/sync.go b/pkg/controller/machinedeployment/sync.go index 8e700c4aa..f0865ef51 100644 --- a/pkg/controller/machinedeployment/sync.go +++ b/pkg/controller/machinedeployment/sync.go @@ -65,10 +65,10 @@ func (r *ReconcileMachineDeployment) sync(ctx context.Context, d *clusterv1alpha // msList should come from getMachineSetsForDeployment(d). // machineMap should come from getMachineMapForDeployment(d, msList). // -// 1. Get all old MSes this deployment targets, and calculate the max revision number among them (maxOldV). -// 2. Get new MS this deployment targets (whose machine template matches deployment's), and update new MS's revision number to (maxOldV + 1), -// only if its revision number is smaller than (maxOldV + 1). If this step failed, we'll update it in the next deployment sync loop. -// 3. Copy new MS's revision number to deployment (update deployment's revision). If this step failed, we'll update it in the next deployment sync loop. +// 1. Get all old MSes this deployment targets, and calculate the max revision number among them (maxOldV). +// 2. Get new MS this deployment targets (whose machine template matches deployment's), and update new MS's revision number to (maxOldV + 1), +// only if its revision number is smaller than (maxOldV + 1). If this step failed, we'll update it in the next deployment sync loop. +// 3. Copy new MS's revision number to deployment (update deployment's revision). If this step failed, we'll update it in the next deployment sync loop. // // Note that currently the deployment controller is using caches to avoid querying the server for reads. // This may lead to stale reads of machine sets, thus incorrect deployment status. diff --git a/pkg/controller/util/machine_deployment.go b/pkg/controller/util/machine_deployment.go index e24fdfbbf..034510ca7 100644 --- a/pkg/controller/util/machine_deployment.go +++ b/pkg/controller/util/machine_deployment.go @@ -166,7 +166,8 @@ var annotationsToSkip = map[string]bool{ // skipCopyAnnotation returns true if we should skip copying the annotation with the given annotation key // TODO: How to decide which annotations should / should not be copied? -// See https://github.com/kubernetes/kubernetes/pull/20035#issuecomment-179558615 +// +// See https://github.com/kubernetes/kubernetes/pull/20035#issuecomment-179558615 func skipCopyAnnotation(key string) bool { return annotationsToSkip[key] } @@ -399,9 +400,9 @@ func getMachineSetFraction(ms v1alpha1.MachineSet, d v1alpha1.MachineDeployment) // EqualIgnoreHash returns true if two given machineTemplateSpec are equal, ignoring the diff in value of Labels[machine-template-hash] // We ignore machine-template-hash because: -// 1. The hash result would be different upon machineTemplateSpec API changes -// (e.g. the addition of a new field will cause the hash code to change) -// 2. The deployment template won't have hash labels. +// 1. The hash result would be different upon machineTemplateSpec API changes +// (e.g. the addition of a new field will cause the hash code to change) +// 2. The deployment template won't have hash labels. func EqualIgnoreHash(template1, template2 *v1alpha1.MachineTemplateSpec) bool { t1Copy := template1.DeepCopy() t2Copy := template2.DeepCopy() @@ -429,8 +430,8 @@ func FindNewMachineSet(deployment *v1alpha1.MachineDeployment, msList []*v1alpha // FindOldMachineSets returns the old machine sets targeted by the given Deployment, with the given slice of MSes. // Returns two list of machine sets -// - the first contains all old machine sets with all non-zero replicas -// - the second contains all old machine sets +// - the first contains all old machine sets with all non-zero replicas +// - the second contains all old machine sets func FindOldMachineSets(deployment *v1alpha1.MachineDeployment, msList []*v1alpha1.MachineSet) ([]*v1alpha1.MachineSet, []*v1alpha1.MachineSet) { var requiredMSs []*v1alpha1.MachineSet allMSs := make([]*v1alpha1.MachineSet, 0, len(msList)) diff --git a/pkg/machines/v1alpha1/types.go b/pkg/machines/v1alpha1/types.go index 3627374cb..3e3458e48 100644 --- a/pkg/machines/v1alpha1/types.go +++ b/pkg/machines/v1alpha1/types.go @@ -126,8 +126,9 @@ type MachineSpec struct { // with new additions accommodating common cluster patterns, like dedicated // etcd Machines. // -// +-----------------------+------------------------+ -// | Master present | Master absent | +// +-----------------------+------------------------+ +// | Master present | Master absent | +// // +---------------+-----------------------+------------------------| // | Node present: | Install control plane | Join the cluster as | // | | and be schedulable | just a node | diff --git a/pkg/rhsm/satellite_subscription_manager.go b/pkg/rhsm/satellite_subscription_manager.go index b32deac10..396800941 100644 --- a/pkg/rhsm/satellite_subscription_manager.go +++ b/pkg/rhsm/satellite_subscription_manager.go @@ -17,6 +17,7 @@ limitations under the License. package rhsm import ( + "context" "crypto/tls" "errors" "fmt" @@ -30,7 +31,7 @@ import ( // SatelliteSubscriptionManager manages the communications between machine-controller and redhat satellite server. type SatelliteSubscriptionManager interface { - DeleteSatelliteHost(machineName, username, password, serverURL string) error + DeleteSatelliteHost(ctx context.Context, machineName, username, password, serverURL string) error } // DefaultSatelliteSubscriptionManager default manager for redhat satellite server. @@ -57,7 +58,7 @@ func NewSatelliteSubscriptionManager() SatelliteSubscriptionManager { } } -func (s *DefaultSatelliteSubscriptionManager) DeleteSatelliteHost(machineName, username, password, serverURL string) error { +func (s *DefaultSatelliteSubscriptionManager) DeleteSatelliteHost(ctx context.Context, machineName, username, password, serverURL string) error { if machineName == "" || username == "" || password == "" || serverURL == "" { return errors.New("satellite server url, username or password cannot be empty") } @@ -68,7 +69,7 @@ func (s *DefaultSatelliteSubscriptionManager) DeleteSatelliteHost(machineName, u ) for retries < maxRetries { - if err := s.executeDeleteRequest(machineName, username, password, serverURL); err != nil { + if err := s.executeDeleteRequest(ctx, machineName, username, password, serverURL); err != nil { klog.Errorf("failed to execute satellite subscription deletion: %v", err) retries++ time.Sleep(500 * time.Second) @@ -82,7 +83,7 @@ func (s *DefaultSatelliteSubscriptionManager) DeleteSatelliteHost(machineName, u return errors.New("failed to delete system profile after max retires number has been reached") } -func (s *DefaultSatelliteSubscriptionManager) executeDeleteRequest(machineName, username, password, serverURL string) error { +func (s *DefaultSatelliteSubscriptionManager) executeDeleteRequest(ctx context.Context, machineName, username, password, serverURL string) error { var requestURL url.URL requestURL.Scheme = "http" if !s.useHTTP { @@ -92,6 +93,7 @@ func (s *DefaultSatelliteSubscriptionManager) executeDeleteRequest(machineName, requestURL.Path = path.Join("api", "v2", "hosts", machineName) deleteHostRequest, err := http.NewRequest(http.MethodDelete, requestURL.String(), nil) + deleteHostRequest = deleteHostRequest.WithContext(ctx) if err != nil { return fmt.Errorf("failed to create a delete host request: %w", err) } diff --git a/pkg/rhsm/satellite_subscription_manager_test.go b/pkg/rhsm/satellite_subscription_manager_test.go index f9b74dd10..38c3f33c1 100644 --- a/pkg/rhsm/satellite_subscription_manager_test.go +++ b/pkg/rhsm/satellite_subscription_manager_test.go @@ -17,6 +17,7 @@ limitations under the License. package rhsm import ( + "context" "fmt" "net/http" "net/http/httptest" @@ -55,7 +56,7 @@ func TestDefaultRedHatSatelliteManager_DeleteSatelliteHost(t *testing.T) { t.Fatalf("failed to parse testing server url: %v", err) } - err = manager.DeleteSatelliteHost("satellite-vm", satelliteUsername, satellitePassword, parsedURL.Host) + err = manager.DeleteSatelliteHost(context.TODO(), "satellite-vm", satelliteUsername, satellitePassword, parsedURL.Host) if err != nil { t.Fatalf("failed to execute redhat host deletion") } diff --git a/pkg/rhsm/subscription_manager.go b/pkg/rhsm/subscription_manager.go index 90f6b9181..9d73bdac0 100644 --- a/pkg/rhsm/subscription_manager.go +++ b/pkg/rhsm/subscription_manager.go @@ -21,7 +21,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "time" @@ -159,7 +159,7 @@ func (d *defaultRedHatSubscriptionManager) deleteSubscription(ctx context.Contex } defer res.Body.Close() - data, err := ioutil.ReadAll(res.Body) + data, err := io.ReadAll(res.Body) if err != nil { return fmt.Errorf("failed while reading response: %w", err) } @@ -187,7 +187,7 @@ func (d *defaultRedHatSubscriptionManager) executeFindSystemsRequest(ctx context } defer res.Body.Close() - data, err := ioutil.ReadAll(res.Body) + data, err := io.ReadAll(res.Body) if err != nil { return nil, fmt.Errorf("failed while reading response: %w", err) } diff --git a/pkg/test/helper.go b/pkg/test/helper.go index 2204e09d5..e50d9113b 100644 --- a/pkg/test/helper.go +++ b/pkg/test/helper.go @@ -17,7 +17,7 @@ limitations under the License. package test import ( - "io/ioutil" + "os" "path/filepath" "testing" @@ -30,11 +30,11 @@ func CompareOutput(t *testing.T, name, output string, update bool) { t.Fatalf("failed to get absolute path to testdata file: %v", err) } if update { - if err := ioutil.WriteFile(golden, []byte(output), 0644); err != nil { + if err := os.WriteFile(golden, []byte(output), 0644); err != nil { t.Fatalf("failed to write updated fixture: %v", err) } } - expected, err := ioutil.ReadFile(golden) + expected, err := os.ReadFile(golden) if err != nil { t.Fatalf("failed to read testdata file: %v", err) } diff --git a/test/e2e/provisioning/verify.go b/test/e2e/provisioning/verify.go index a474d37e4..c6b6d22da 100644 --- a/test/e2e/provisioning/verify.go +++ b/test/e2e/provisioning/verify.go @@ -19,7 +19,7 @@ package provisioning import ( "context" "fmt" - "io/ioutil" + "os" "strings" "time" @@ -318,7 +318,7 @@ func isNodeForMachine(node *corev1.Node, machine *clusterv1alpha1.Machine) bool } func readAndModifyManifest(pathToManifest string, keyValuePairs []string) (string, error) { - contentRaw, err := ioutil.ReadFile(pathToManifest) + contentRaw, err := os.ReadFile(pathToManifest) if err != nil { return "", err } From 9120418efb103643ad0ed7388c512973add8f93b Mon Sep 17 00:00:00 2001 From: Matthias Osthues Date: Fri, 2 Sep 2022 18:12:24 +0200 Subject: [PATCH 212/489] switch to kubermati-ci folder in e2e tests (#1426) --- .../testdata/machinedeployment-vsphere-datastore-cluster.yaml | 2 +- .../testdata/machinedeployment-vsphere-resource-pool.yaml | 2 +- .../testdata/machinedeployment-vsphere-static-ip.yaml | 2 +- test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml index 743faff47..c07b9c0f1 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml @@ -28,7 +28,7 @@ spec: username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' datacenter: 'Hamburg' - folder: '/Hamburg/vm/Kubermatic-dev' + folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically datastoreCluster: 'dsc-1' diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml index 13d8af421..4ead4d9e7 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml @@ -28,7 +28,7 @@ spec: username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' datacenter: 'Hamburg' - folder: '/Hamburg/vm/Kubermatic-dev' + folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> datastoreCluster: 'dsc-1' resourcePool: 'e2e-resource-pool' diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml index 7ebc21556..369207022 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml @@ -28,7 +28,7 @@ spec: username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' datacenter: 'Hamburg' - folder: '/Hamburg/vm/Kubermatic-dev' + folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically datastore: alpha1 diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml index e14f7f4a3..223ffc5b2 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml @@ -28,7 +28,7 @@ spec: username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' datacenter: 'Hamburg' - folder: '/Hamburg/vm/Kubermatic-dev' + folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically datastore: alpha1 From cb174d46f111ef6ce1c6688d25f2df910c96e79b Mon Sep 17 00:00:00 2001 From: Sachin Tiptur <56350000+sachintiptur@users.noreply.github.com> Date: Tue, 6 Sep 2022 14:23:38 +0200 Subject: [PATCH 213/489] Fix to mitigate race condition while configuring kubelet node ips (#1424) Signed-off-by: Sachin Tiptur Signed-off-by: Sachin Tiptur --- pkg/userdata/helper/helper.go | 5 +++++ pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml | 5 +++++ pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/pkg/userdata/helper/helper.go b/pkg/userdata/helper/helper.go index 63155f26c..86163d982 100644 --- a/pkg/userdata/helper/helper.go +++ b/pkg/userdata/helper/helper.go @@ -160,6 +160,11 @@ func SetupNodeIPEnvScript(ipFamily util.IPFamily) string { case util.DualStack: defaultIfcIP = `DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") +if [ -z "${DEFAULT_IFC_IP6}" ] +then + echodate "Failed to get IPv6 address for the default route interface" + exit 1 +fi DEFAULT_IFC_IP=$DEFAULT_IFC_IP,$DEFAULT_IFC_IP6` default: defaultIfcIP = defaultIfcIPv4 diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml index d76ac8ad9..97ef06d47 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml @@ -259,6 +259,11 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") + if [ -z "${DEFAULT_IFC_IP6}" ] + then + echodate "Failed to get IPv6 address for the default route interface" + exit 1 + fi DEFAULT_IFC_IP=$DEFAULT_IFC_IP,$DEFAULT_IFC_IP6 # get the full hostname diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml index 77904a297..671e2c6f5 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml @@ -259,6 +259,11 @@ write_files: # get the default interface IP address DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") + if [ -z "${DEFAULT_IFC_IP6}" ] + then + echodate "Failed to get IPv6 address for the default route interface" + exit 1 + fi DEFAULT_IFC_IP=$DEFAULT_IFC_IP,$DEFAULT_IFC_IP6 # get the full hostname From 3f309f16991238db83b73afdc2d971d8dde73366 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 19 Sep 2022 12:46:25 +0500 Subject: [PATCH 214/489] Bump Kubernetes versions used for E2E tests to 1.22.14, 1.23.11 and 1.24.5 respectively (#1431) * Bump kubernetes versions used for E2E tests to 1.22.14, 1.23.11 and 1.24.5 respectively Signed-off-by: Waleed Malik * Skip TLS verification for vSphere Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- test/e2e/provisioning/all_e2e_test.go | 2 +- test/e2e/provisioning/helper.go | 6 +++--- .../machinedeployment-vsphere-datastore-cluster.yaml | 1 + .../testdata/machinedeployment-vsphere-resource-pool.yaml | 1 + .../testdata/machinedeployment-vsphere-static-ip.yaml | 2 +- .../provisioning/testdata/machinedeployment-vsphere.yaml | 1 + 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 252440656..f1a9052cd 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -80,7 +80,7 @@ const ( nutanixManifest = "./testdata/machinedeployment-nutanix.yaml" ) -const defaultKubernetesVersion = "1.23.5" +const defaultKubernetesVersion = "1.23.11" var testRunIdentifier = flag.String("identifier", "local", "The unique identifier for this test run") diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 39812773a..6f992772c 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,9 +33,9 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.22.7"), - semver.MustParse("v1.23.5"), - semver.MustParse("v1.24.0"), + semver.MustParse("v1.22.14"), + semver.MustParse("v1.23.11"), + semver.MustParse("v1.24.5"), } operatingSystems = []providerconfigtypes.OperatingSystem{ diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml index c07b9c0f1..b42016a6c 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml @@ -35,6 +35,7 @@ spec: cpus: 2 MemoryMB: 2048 diskSizeGB: << DISK_SIZE >> + allowInsecure: true operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml index 4ead4d9e7..0e505c1fe 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml @@ -35,6 +35,7 @@ spec: cpus: 2 MemoryMB: 2048 diskSizeGB: << DISK_SIZE >> + allowInsecure: true operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml index 369207022..131e6f913 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml @@ -32,9 +32,9 @@ spec: password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically datastore: alpha1 - allowInsecure: true cpus: 2 MemoryMB: 2048 + allowInsecure: true operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml index 223ffc5b2..7ddc15a6b 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml @@ -35,6 +35,7 @@ spec: cpus: 2 MemoryMB: 4096 diskSizeGB: << DISK_SIZE >> + allowInsecure: true operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false From b0fca25755d4d8a9047299712381e1f5fe486eee Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Mon, 19 Sep 2022 12:34:28 +0200 Subject: [PATCH 215/489] Add leader election related flags to machine-controller (#1435) Signed-off-by: Marvin Beckers Signed-off-by: Marvin Beckers --- cmd/machine-controller/main.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 1f4f8f9c1..320e47052 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -69,6 +69,8 @@ var ( bootstrapTokenServiceAccountName string skipEvictionAfter time.Duration caBundleFile string + enableLeaderElection bool + leaderElectionNamespace string useOSM bool @@ -88,6 +90,7 @@ var ( const ( defaultLeaderElectionNamespace = "kube-system" + defaultLeaderElectionID = "machine-controller" ) // controllerRunOptions holds data that are required to create and run machine controller. @@ -152,6 +155,9 @@ func main() { flag.StringVar(&healthProbeAddress, "health-probe-address", "127.0.0.1:8085", "The address on which the liveness check on /healthz and readiness check on /readyz will be available") flag.StringVar(&metricsAddress, "metrics-address", "127.0.0.1:8080", "The address on which Prometheus metrics will be available under /metrics") flag.StringVar(&name, "name", "", "When set, the controller will only process machines with the label \"machine.k8s.io/controller\": name") + flag.BoolVar(&enableLeaderElection, "enable-leader-election", true, "Enable leader election for machine-controller. Enabling this will ensure there is only one active instance.") + flag.StringVar(&leaderElectionNamespace, "leader-election-namespace", "kube-system", "Namespace to use for leader election.") + flag.StringVar(&joinClusterTimeout, "join-cluster-timeout", "", "when set, machines that have an owner and do not join the cluster within the configured duration will be deleted, so the owner re-creates them") flag.StringVar(&bootstrapTokenServiceAccountName, "bootstrap-token-service-account-name", "", "When set use the service account token from this SA as bootstrap token instead of creating a temporary one. Passed in namespace/name format") flag.BoolVar(&profiling, "enable-profiling", false, "when set, enables the endpoints on the http server under /debug/pprof/") @@ -303,11 +309,16 @@ func main() { } func createManager(syncPeriod time.Duration, options controllerRunOptions) (manager.Manager, error) { + namespace := leaderElectionNamespace + if namespace == "" { + namespace = defaultLeaderElectionNamespace + } + mgr, err := manager.New(options.cfg, manager.Options{ SyncPeriod: &syncPeriod, - LeaderElection: true, - LeaderElectionID: "machine-controller", - LeaderElectionNamespace: defaultLeaderElectionNamespace, + LeaderElection: enableLeaderElection, + LeaderElectionID: defaultLeaderElectionID, + LeaderElectionNamespace: namespace, HealthProbeBindAddress: healthProbeAddress, MetricsBindAddress: metricsAddress, }) From 3bf84c14846061c62a349b0888b14489c85ee755 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 19 Sep 2022 16:43:35 +0500 Subject: [PATCH 216/489] Add kind/chore to PR template (#1432) Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- .github/PULL_REQUEST_TEMPLATE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index f035f3716..126374da7 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -19,6 +19,7 @@ Optionally add one or more of the following kinds if applicable: /kind failing-test /kind flake /kind regression +/kind chore --> **Special notes for your reviewer**: From 8b1feb485d77e314779287ddfc37638b7fd0fce4 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Mon, 19 Sep 2022 15:12:36 +0200 Subject: [PATCH 217/489] Remove references to proxy in Nutanix e2e testing (#1436) * Remove references to proxy in Nutanix e2e testing Signed-off-by: Marvin Beckers * Change run-e2e-test.sh script Signed-off-by: Marvin Beckers Signed-off-by: Marvin Beckers --- .prow/provider-nutanix.yaml | 5 +---- hack/ci/run-e2e-tests.sh | 5 ----- test/e2e/provisioning/all_e2e_test.go | 6 ------ .../provisioning/testdata/machinedeployment-nutanix.yaml | 1 - 4 files changed, 1 insertion(+), 16 deletions(-) diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 4775ec78f..c0b955224 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -14,10 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-nutanix - optional: true - always_run: false - # TODO uncomment this when Nutanix is in a working condition - #run_if_changed: "(pkg/cloudprovider/provider/nutanix/)" + run_if_changed: "(pkg/cloudprovider/provider/nutanix/)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: diff --git a/hack/ci/run-e2e-tests.sh b/hack/ci/run-e2e-tests.sh index 95b4687aa..cf848be61 100755 --- a/hack/ci/run-e2e-tests.sh +++ b/hack/ci/run-e2e-tests.sh @@ -68,11 +68,6 @@ beforeMCSetup=$(nowms) source hack/ci/setup-machine-controller-in-kind.sh pushElapsed kind_mc_setup_duration_milliseconds $beforeMCSetup -if [[ ! -z "${NUTANIX_E2E_PROXY_HOST:-}" ]]; then - vm_priv_addr=$(cat ./priv_addr) - export NUTANIX_E2E_PROXY_URL="http://${NUTANIX_E2E_PROXY_USERNAME}:${NUTANIX_E2E_PROXY_PASSWORD}@${vm_priv_addr}:${NUTANIX_E2E_PROXY_PORT}/" -fi - echo "Running e2e tests..." EXTRA_ARGS="" if [[ $# -gt 0 ]]; then diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index f1a9052cd..9cec18e1d 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -956,11 +956,6 @@ func getNutanixTestParams(t *testing.T) []string { "NUTANIX_E2E_ENDPOINT, NUTANIX_E2E_PROJECT_NAME or NUTANIX_E2E_SUBNET_NAME environment variables cannot be empty") } - // a proxy URL will be passed in our e2e test environment so - // a HTTP proxy can be used to access the Nutanix API in a different - // network segment. - proxyURL := os.Getenv("NUTANIX_E2E_PROXY_URL") - // set up parameters params := []string{fmt.Sprintf("<< NUTANIX_PASSWORD >>=%s", password), fmt.Sprintf("<< NUTANIX_USERNAME >>=%s", username), @@ -968,7 +963,6 @@ func getNutanixTestParams(t *testing.T) []string { fmt.Sprintf("<< NUTANIX_CLUSTER >>=%s", cluster), fmt.Sprintf("<< NUTANIX_PROJECT >>=%s", project), fmt.Sprintf("<< NUTANIX_SUBNET >>=%s", subnet), - fmt.Sprintf("<< NUTANIX_PROXY_URL >>=%s", proxyURL), } return params } diff --git a/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml b/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml index ddc753588..5a2bea06a 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml @@ -27,7 +27,6 @@ spec: username: '<< NUTANIX_USERNAME >>' password: '<< NUTANIX_PASSWORD >>' endpoint: '<< NUTANIX_ENDPOINT >>' - proxyURL: '<< NUTANIX_PROXY_URL >>' allowInsecure: true clusterName: '<< NUTANIX_CLUSTER >>' projectName: '<< NUTANIX_PROJECT >>' From 59d0e375ae9ef93cd562915de540bbcfaaf061bb Mon Sep 17 00:00:00 2001 From: Sachin Tiptur <56350000+sachintiptur@users.noreply.github.com> Date: Mon, 19 Sep 2022 15:37:35 +0200 Subject: [PATCH 218/489] Fix dualstack IPv6 addr assignment issue in rockylinux and RHEL (#1430) Signed-off-by: Sachin Tiptur Signed-off-by: Sachin Tiptur --- pkg/userdata/rhel/provider.go | 14 ++++++++++---- pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml | 14 ++++++++++---- .../rhel/testdata/kubelet-v1.22-nutanix.yaml | 14 ++++++++++---- .../rhel/testdata/kubelet-v1.23-aws-external.yaml | 14 ++++++++++---- pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml | 14 ++++++++++---- .../testdata/kubelet-v1.23-vsphere-mirrors.yaml | 14 ++++++++++---- .../rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml | 14 ++++++++++---- .../rhel/testdata/kubelet-v1.23-vsphere.yaml | 14 ++++++++++---- .../rhel/testdata/kubelet-v1.24-aws-external.yaml | 14 ++++++++++---- pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml | 14 ++++++++++---- .../rhel/testdata/pod-cidr-azure-rhel.yaml | 14 ++++++++++---- pkg/userdata/rockylinux/provider.go | 14 ++++++++++---- .../rockylinux/testdata/kubelet-v1.22-aws.yaml | 14 ++++++++++---- .../testdata/kubelet-v1.23-aws-external.yaml | 14 ++++++++++---- .../rockylinux/testdata/kubelet-v1.23-aws.yaml | 14 ++++++++++---- .../rockylinux/testdata/kubelet-v1.23-nutanix.yaml | 14 ++++++++++---- .../testdata/kubelet-v1.23-vsphere-mirrors.yaml | 14 ++++++++++---- .../testdata/kubelet-v1.23-vsphere-proxy.yaml | 14 ++++++++++---- .../rockylinux/testdata/kubelet-v1.23-vsphere.yaml | 14 ++++++++++---- .../rockylinux/testdata/kubelet-v1.24-aws.yaml | 14 ++++++++++---- 20 files changed, 200 insertions(+), 80 deletions(-) diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 46f3b3046..41016bc28 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -232,10 +232,16 @@ write_files: {{ .ContainerRuntimeScript | indent 4 }} {{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index dbdb97a14..46499581f 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -168,10 +168,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml index 82ee10299..940fff796 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml @@ -176,10 +176,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index 98a5f9b61..abf428dd2 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -168,10 +168,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index deeef0ad8..db5172808 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -168,10 +168,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 749268ce1..24a0d5571 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -182,10 +182,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index 1ec73c6a7..4cad3586f 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -182,10 +182,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index 49a601c3d..67eec4043 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -174,10 +174,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml index 88dfc8387..3c100fa6a 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml @@ -167,10 +167,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml index f1905c82b..49e9b76b1 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml @@ -167,10 +167,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index 0a661c056..61430455e 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -173,10 +173,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rockylinux/provider.go b/pkg/userdata/rockylinux/provider.go index eebe3734e..7efb05332 100644 --- a/pkg/userdata/rockylinux/provider.go +++ b/pkg/userdata/rockylinux/provider.go @@ -234,10 +234,16 @@ write_files: {{ .ContainerRuntimeScript | indent 4 }} {{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml index 2b58e6cc7..56d870933 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml @@ -169,10 +169,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml index fb1eafba0..0f11df4f8 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml @@ -169,10 +169,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml index e3082bbfc..377cb87bb 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml @@ -169,10 +169,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml index 721b11943..a201b9397 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml @@ -176,10 +176,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 14cb1eabc..beb60d9ab 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -182,10 +182,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml index 9273d0b8e..7ef8f1dea 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -182,10 +182,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml index 75d43c26f..ed4b93aa7 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml @@ -174,10 +174,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml index 16f21929b..3f6a48f7a 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml @@ -168,10 +168,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ From ee0d36714bedaa172b528b45ee5d02dd785460dc Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Tue, 20 Sep 2022 08:41:27 +0300 Subject: [PATCH 219/489] Upgrade CNI (#1437) Signed-off-by: Artiom Diomin Signed-off-by: Artiom Diomin --- pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-mirrors.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-proxy.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml | 2 +- pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml | 2 +- pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml | 2 +- pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml | 2 +- pkg/userdata/flatcar/testdata/containerd.yaml | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.22.7.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.23.5.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.24.0.json | 2 +- pkg/userdata/helper/download_binaries_script.go | 2 +- .../helper/testdata/safe_download_binaries_v1.22.5.golden | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml | 2 +- pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml | 2 +- pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml | 2 +- .../rockylinux/testdata/kubelet-v1.23-aws-external.yaml | 2 +- pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml | 2 +- pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml | 2 +- .../rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml | 2 +- .../rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml | 2 +- pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml | 2 +- pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml | 2 +- pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml | 2 +- .../sles/testdata/kubelet-version-without-v-prefix.yaml | 2 +- pkg/userdata/sles/testdata/multiple-dns-servers.yaml | 2 +- pkg/userdata/sles/testdata/multiple-ssh-keys.yaml | 2 +- .../sles/testdata/openstack-overwrite-cloud-config.yaml | 2 +- pkg/userdata/sles/testdata/openstack.yaml | 2 +- pkg/userdata/sles/testdata/version-1.22.7.yaml | 2 +- pkg/userdata/sles/testdata/version-1.23.5.yaml | 2 +- pkg/userdata/sles/testdata/version-1.24.0.yaml | 2 +- pkg/userdata/sles/testdata/vsphere-mirrors.yaml | 2 +- pkg/userdata/sles/testdata/vsphere-proxy.yaml | 2 +- pkg/userdata/sles/testdata/vsphere.yaml | 2 +- pkg/userdata/ubuntu/testdata/containerd.yaml | 2 +- pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml | 2 +- pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml | 2 +- pkg/userdata/ubuntu/testdata/docker.yaml | 2 +- .../ubuntu/testdata/kubelet-version-without-v-prefix.yaml | 2 +- pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml | 2 +- pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml | 2 +- pkg/userdata/ubuntu/testdata/nutanix.yaml | 2 +- pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml | 2 +- .../ubuntu/testdata/openstack-overwrite-cloud-config.yaml | 2 +- pkg/userdata/ubuntu/testdata/openstack.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.22.7.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.23.5.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.24.0.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere.yaml | 2 +- 71 files changed, 71 insertions(+), 71 deletions(-) diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml index 42864bd2e..3eaa0477a 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml @@ -118,7 +118,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml index 77275d458..a3590ffa4 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml @@ -118,7 +118,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml index 10cf2cebe..34502bff9 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml @@ -118,7 +118,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-mirrors.yaml index bb9ac745e..65c9163f2 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -131,7 +131,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-proxy.yaml index 9c36c7abd..b59b08e0f 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -131,7 +131,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere.yaml index c6eb43a78..3e814c32e 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere.yaml @@ -123,7 +123,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml index c8bf88bdb..c623222c1 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml @@ -121,7 +121,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index 92555b815..df0b612e8 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml index da8f203ec..4d9be1ebe 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index bcb8c3ade..28ffefc53 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml index ad28ad691..bbeb1993f 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml @@ -135,7 +135,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 11c411561..3b31ce3fe 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -141,7 +141,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml index 8cc2eb137..962ce1538 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -141,7 +141,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml index aee1a886d..aadaffc2c 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml @@ -133,7 +133,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml index 9830772bc..c2c9a339c 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml @@ -127,7 +127,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml index f990df896..0b598039c 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.22.7.yaml @@ -408,7 +408,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml index 5d1763617..1dc1284e8 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.23.5.yaml @@ -406,7 +406,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml index 82a947e25..e075a0484 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml @@ -405,7 +405,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index 7599cc0bb..6c9e0cfe5 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -388,7 +388,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json b/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json index fc865ed51..dfca82e3f 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.22.7.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.7%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.1.1%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.22.7%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \\\n --feature-gates=DynamicKubeletConfig=true \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json b/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json index 06911b23b..c78100c17 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.23.5.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.1.1%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.23.5%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20docker%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/docker/daemon.json","contents":{"source":"data:,%7B%22exec-opts%22%3A%5B%22native.cgroupdriver%3Dsystemd%22%5D%2C%22storage-driver%22%3A%22overlay2%22%2C%22log-driver%22%3A%22json-file%22%2C%22log-opts%22%3A%7B%22max-file%22%3A%225%22%2C%22max-size%22%3A%22100m%22%7D%7D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=docker.service\nRequires=docker.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=docker \\\n --container-runtime-endpoint=unix:///var/run/dockershim.sock \\\n --network-plugin=cni \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json index cebdc9554..9cfe46ecf 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v0.8.7%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.1.1%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/download_binaries_script.go b/pkg/userdata/helper/download_binaries_script.go index 6ff2df3a1..92c317be0 100644 --- a/pkg/userdata/helper/download_binaries_script.go +++ b/pkg/userdata/helper/download_binaries_script.go @@ -164,7 +164,7 @@ func SafeDownloadBinariesScript(kubeVersion string) (string, error) { } const ( - CNIVersion = "v0.8.7" + CNIVersion = "v1.1.1" CRIToolsVersion = "v1.22.0" ) diff --git a/pkg/userdata/helper/testdata/safe_download_binaries_v1.22.5.golden b/pkg/userdata/helper/testdata/safe_download_binaries_v1.22.5.golden index c3ef51f69..35d7137e9 100644 --- a/pkg/userdata/helper/testdata/safe_download_binaries_v1.22.5.golden +++ b/pkg/userdata/helper/testdata/safe_download_binaries_v1.22.5.golden @@ -18,7 +18,7 @@ aarch64) ;; esac fi -CNI_VERSION="${CNI_VERSION:-v0.8.7}" +CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index 46499581f..51669534d 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -122,7 +122,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml index 940fff796..676e5ecb4 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml @@ -130,7 +130,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index abf428dd2..5526273f7 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -122,7 +122,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index db5172808..65e5f05cf 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -122,7 +122,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 24a0d5571..35f202641 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -136,7 +136,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index 4cad3586f..44bfafcad 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -136,7 +136,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index 67eec4043..ac6f9f3b9 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml index 3c100fa6a..5b07fe6df 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml @@ -121,7 +121,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml index 49e9b76b1..ecd6b661c 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml @@ -121,7 +121,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index 61430455e..7d308d7fa 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -127,7 +127,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml index 56d870933..d431671f3 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml @@ -123,7 +123,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml index 0f11df4f8..f49299783 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml @@ -123,7 +123,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml index 377cb87bb..e2a3f3038 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml @@ -123,7 +123,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml index a201b9397..47f37e42b 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml @@ -130,7 +130,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml index beb60d9ab..ab2f47194 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -136,7 +136,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml index 7ef8f1dea..031233eaa 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -136,7 +136,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml index ed4b93aa7..f08819451 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml index 3f6a48f7a..9cb82e693 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml @@ -122,7 +122,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml index 9009e4bb3..9a67b7d41 100644 --- a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml @@ -82,7 +82,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml index 7e1c146ea..82e5c09ec 100644 --- a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml @@ -80,7 +80,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml index 3cdfba5f6..251106bbd 100644 --- a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml @@ -80,7 +80,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml index 6643d04e4..06cc3b7f4 100644 --- a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml @@ -82,7 +82,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml index 32a04e68b..4ef9dcb61 100644 --- a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml @@ -80,7 +80,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/sles/testdata/openstack.yaml b/pkg/userdata/sles/testdata/openstack.yaml index 1aa862a45..745f4434f 100644 --- a/pkg/userdata/sles/testdata/openstack.yaml +++ b/pkg/userdata/sles/testdata/openstack.yaml @@ -80,7 +80,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/sles/testdata/version-1.22.7.yaml b/pkg/userdata/sles/testdata/version-1.22.7.yaml index c2d9687d1..8b529f0cc 100644 --- a/pkg/userdata/sles/testdata/version-1.22.7.yaml +++ b/pkg/userdata/sles/testdata/version-1.22.7.yaml @@ -80,7 +80,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/sles/testdata/version-1.23.5.yaml b/pkg/userdata/sles/testdata/version-1.23.5.yaml index d42a82bc1..232f1854b 100644 --- a/pkg/userdata/sles/testdata/version-1.23.5.yaml +++ b/pkg/userdata/sles/testdata/version-1.23.5.yaml @@ -80,7 +80,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/sles/testdata/version-1.24.0.yaml b/pkg/userdata/sles/testdata/version-1.24.0.yaml index 179ecc5cb..579f094d2 100644 --- a/pkg/userdata/sles/testdata/version-1.24.0.yaml +++ b/pkg/userdata/sles/testdata/version-1.24.0.yaml @@ -80,7 +80,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml index 7d732ee9b..3b4368b66 100644 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml @@ -90,7 +90,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml index 9576ec66d..d94216edf 100644 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/sles/testdata/vsphere-proxy.yaml @@ -90,7 +90,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml index 76a9a138b..73b893b14 100644 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ b/pkg/userdata/sles/testdata/vsphere.yaml @@ -81,7 +81,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index 49a360520..bf311fa08 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -130,7 +130,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml index 97ef06d47..dc20e1dc2 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 72876104d..046728a2d 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -130,7 +130,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml index 0f2ee1524..242a6fd6d 100644 --- a/pkg/userdata/ubuntu/testdata/docker.yaml +++ b/pkg/userdata/ubuntu/testdata/docker.yaml @@ -130,7 +130,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index 77428740e..4b10b3a84 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index f0d5f55f0..2998b7aca 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index dc18f0a19..9e01e1b28 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -130,7 +130,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 58c16ae1f..726a65c6d 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -131,7 +131,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml index 671e2c6f5..8faed2f4a 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index 3866be991..15e2bcce3 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 6725d1b9b..88007a439 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml b/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml index 77428740e..4b10b3a84 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml index d05672a00..f6a1c61af 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml index c6f718374..9d45afa30 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index be7f75f56..cd58f8d84 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -138,7 +138,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 7dfa8d510..f99c2937b 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -138,7 +138,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index f73e59b71..90edde887 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -129,7 +129,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.1.1}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" From f405d185a750771118e20c9121c098abe04dc3fc Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 20 Sep 2022 15:48:25 +0500 Subject: [PATCH 220/489] Support for Ubuntu 22.04 (#1427) * Add support for Ubuntu 22.04 Signed-off-by: Waleed Malik * Use containerd v1.5 as default when docker is selected Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- docs/operating-system.md | 2 +- pkg/cloudprovider/provider/alibaba/provider.go | 2 +- pkg/cloudprovider/provider/aws/provider.go | 4 ++-- pkg/cloudprovider/provider/azure/provider.go | 4 ++-- .../provider/digitalocean/provider.go | 2 +- .../provider/equinixmetal/provider.go | 2 +- pkg/cloudprovider/provider/gce/config.go | 2 +- pkg/cloudprovider/provider/hetzner/provider.go | 2 +- pkg/containerruntime/docker.go | 4 +++- .../centos/testdata/kubelet-v1.22-aws.yaml | 2 +- .../testdata/kubelet-v1.23-aws-external.yaml | 2 +- .../centos/testdata/kubelet-v1.23-aws.yaml | 2 +- .../centos/testdata/kubelet-v1.23-nutanix.yaml | 2 +- .../testdata/kubelet-v1.23-vsphere-mirrors.yaml | 2 +- .../testdata/kubelet-v1.23-vsphere-proxy.yaml | 2 +- .../centos/testdata/kubelet-v1.23-vsphere.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml | 2 +- .../rhel/testdata/kubelet-v1.22-nutanix.yaml | 2 +- .../rhel/testdata/kubelet-v1.23-aws-external.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml | 2 +- .../testdata/kubelet-v1.23-vsphere-mirrors.yaml | 2 +- .../testdata/kubelet-v1.23-vsphere-proxy.yaml | 2 +- .../rhel/testdata/kubelet-v1.23-vsphere.yaml | 2 +- .../rhel/testdata/pod-cidr-azure-rhel.yaml | 2 +- .../rockylinux/testdata/kubelet-v1.22-aws.yaml | 2 +- .../testdata/kubelet-v1.23-aws-external.yaml | 2 +- .../rockylinux/testdata/kubelet-v1.23-aws.yaml | 2 +- .../testdata/kubelet-v1.23-nutanix.yaml | 2 +- .../testdata/kubelet-v1.23-vsphere-mirrors.yaml | 2 +- .../testdata/kubelet-v1.23-vsphere-proxy.yaml | 2 +- .../testdata/kubelet-v1.23-vsphere.yaml | 2 +- .../ubuntu/testdata/digitalocean-dualstack.yaml | 2 +- .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 2 +- pkg/userdata/ubuntu/testdata/docker.yaml | 2 +- .../kubelet-version-without-v-prefix.yaml | 2 +- .../ubuntu/testdata/multiple-dns-servers.yaml | 2 +- .../ubuntu/testdata/multiple-ssh-keys.yaml | 2 +- pkg/userdata/ubuntu/testdata/nutanix.yaml | 2 +- .../ubuntu/testdata/openstack-dualstack.yaml | 2 +- .../openstack-overwrite-cloud-config.yaml | 2 +- pkg/userdata/ubuntu/testdata/openstack.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.22.7.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.23.5.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere.yaml | 2 +- test/e2e/provisioning/helper.go | 15 +++++++++++++-- .../testdata/machinedeployment-kubevirt.yaml | 2 +- 48 files changed, 64 insertions(+), 51 deletions(-) diff --git a/docs/operating-system.md b/docs/operating-system.md index beb70e6e6..93f16a872 100644 --- a/docs/operating-system.md +++ b/docs/operating-system.md @@ -45,4 +45,4 @@ Machine controller may work with other OS versions that are not listed in the ta | RHEL | 8.x | | Rocky Linux | 8.5 | | SLES | SLES 15 SP3 | -| Ubuntu | 20.04 LTS | \ No newline at end of file +| Ubuntu | 20.04 LTS, 22.04 LTS | diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index 3b4f773f9..f70d42c45 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -44,7 +44,7 @@ import ( const ( machineUIDTag = "machine_uid" centosImageName = "CentOS 7.9 64 bit" - ubuntuImageName = "Ubuntu 20.04 64 bit" + ubuntuImageName = "Ubuntu 22.04 64 bit" finalizerInstance = "kubermatic.io/cleanup-alibaba-instance" ) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 5f56270ea..e72cdcae4 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -141,13 +141,13 @@ var ( providerconfigtypes.OperatingSystemUbuntu: { awstypes.CPUArchitectureX86_64: { // Be as precise as possible - otherwise we might get a nightly dev build - description: "Canonical, Ubuntu, 20.04 LTS, amd64 focal image build on ????-??-??", + description: "Canonical, Ubuntu, 22.04 LTS, amd64 jammy image build on ????-??-??", // The AWS marketplace ID from Canonical owner: "099720109477", }, awstypes.CPUArchitectureARM64: { // Be as precise as possible - otherwise we might get a nightly dev build - description: "Canonical, Ubuntu, 20.04 LTS, arm64 focal image build on ????-??-??", + description: "Canonical, Ubuntu, 22.04 LTS, arm64 jammy image build on ????-??-??", // The AWS marketplace ID from Canonical owner: "099720109477", }, diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index a68813d8e..b88f54c69 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -148,8 +148,8 @@ var imageReferences = map[providerconfigtypes.OperatingSystem]compute.ImageRefer }, providerconfigtypes.OperatingSystemUbuntu: { Publisher: to.StringPtr("Canonical"), - Offer: to.StringPtr("0001-com-ubuntu-server-focal"), - Sku: to.StringPtr("20_04-lts"), + Offer: to.StringPtr("0001-com-ubuntu-server-jammy"), + Sku: to.StringPtr("22_04-lts"), Version: to.StringPtr("latest"), }, providerconfigtypes.OperatingSystemRHEL: { diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index fbb121f6f..6f4a9a6c4 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -85,7 +85,7 @@ func (t *TokenSource) Token() (*oauth2.Token, error) { func getSlugForOS(os providerconfigtypes.OperatingSystem) (string, error) { switch os { case providerconfigtypes.OperatingSystemUbuntu: - return "ubuntu-20-04-x64", nil + return "ubuntu-22-04-x64", nil case providerconfigtypes.OperatingSystemCentOS: return "centos-7-x64", nil case providerconfigtypes.OperatingSystemRockyLinux: diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index 0fd67b6c8..2c290450e 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -471,7 +471,7 @@ func getDeviceByTag(client *packngo.Client, projectID, tag string) (*packngo.Dev func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { switch os { case providerconfigtypes.OperatingSystemUbuntu: - return "ubuntu_20_04", nil + return "ubuntu_22_04", nil case providerconfigtypes.OperatingSystemCentOS: return "centos_7", nil case providerconfigtypes.OperatingSystemFlatcar: diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index 934adca06..9901b7af7 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -48,7 +48,7 @@ var imageProjects = map[providerconfigtypes.OperatingSystem]string{ // imageFamilies maps the OS to the Google Cloud image projects. var imageFamilies = map[providerconfigtypes.OperatingSystem]string{ - providerconfigtypes.OperatingSystemUbuntu: "ubuntu-2004-lts", + providerconfigtypes.OperatingSystemUbuntu: "ubuntu-2204-lts", } // diskTypes are the disk types of the Google Cloud. Map is used for diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 23c4dddfe..9c385e809 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -70,7 +70,7 @@ type Config struct { func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { switch os { case providerconfigtypes.OperatingSystemUbuntu: - return "ubuntu-20.04", nil + return "ubuntu-22.04", nil case providerconfigtypes.OperatingSystemCentOS: return "centos-7", nil case providerconfigtypes.OperatingSystemRockyLinux: diff --git a/pkg/containerruntime/docker.go b/pkg/containerruntime/docker.go index e79920d7b..c8973e625 100644 --- a/pkg/containerruntime/docker.go +++ b/pkg/containerruntime/docker.go @@ -27,7 +27,8 @@ import ( ) const ( - DefaultDockerContainerdVersion = "1.4" + LegacyDockerContainerdVersion = "1.4" + DefaultDockerContainerdVersion = "1.5" DefaultDockerVersion = "20.10" LegacyDockerVersion = "19.03" ) @@ -89,6 +90,7 @@ func (eng *Docker) ScriptFor(os types.OperatingSystem) (string, error) { switch os { case types.OperatingSystemAmazonLinux2: + args.ContainerdVersion = LegacyDockerContainerdVersion err := dockerAmazonTemplate.Execute(&buf, args) return buf.String(), err case types.OperatingSystemCentOS, types.OperatingSystemRHEL, types.OperatingSystemRockyLinux: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index df0b612e8..d4d503441 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -99,7 +99,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml index 4d9be1ebe..ff6b3c5c5 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml @@ -99,7 +99,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index 28ffefc53..4a793afcd 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -99,7 +99,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml index bbeb1993f..2c6ecd081 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml @@ -106,7 +106,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 3b31ce3fe..2c64afd93 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -112,7 +112,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml index 962ce1538..fb5cc23d0 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -112,7 +112,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml index aadaffc2c..9b68d8f96 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml @@ -104,7 +104,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index 51669534d..5ee3c66bb 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -94,7 +94,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml index 676e5ecb4..ec9e1ee70 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml @@ -102,7 +102,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index 5526273f7..db2355f59 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -94,7 +94,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index 65e5f05cf..d5bab82bd 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -94,7 +94,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 35f202641..7e0ca2995 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -108,7 +108,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index 44bfafcad..82c5677dc 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -108,7 +108,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index ac6f9f3b9..5b24323eb 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -100,7 +100,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index 7d308d7fa..df98b4fe5 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -99,7 +99,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml index d431671f3..b4da33538 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml @@ -95,7 +95,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml index f49299783..04eeda059 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml @@ -95,7 +95,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml index e2a3f3038..bc2e54389 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml @@ -95,7 +95,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml index 47f37e42b..6377d7b31 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml @@ -102,7 +102,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml index ab2f47194..47ebef3ee 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -108,7 +108,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml index 031233eaa..ba853e3f9 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -108,7 +108,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml index f08819451..a2464cc16 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml @@ -100,7 +100,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.4* \ + containerd.io-1.5* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml index dc20e1dc2..7438e9e4b 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml @@ -99,7 +99,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.4* \ + containerd.io=1.5* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 046728a2d..b0a6d2cbe 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -101,7 +101,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.4* \ + containerd.io=1.5* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml index 242a6fd6d..d105086e8 100644 --- a/pkg/userdata/ubuntu/testdata/docker.yaml +++ b/pkg/userdata/ubuntu/testdata/docker.yaml @@ -101,7 +101,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.4* \ + containerd.io=1.5* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index 4b10b3a84..82318116f 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -99,7 +99,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.4* \ + containerd.io=1.5* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 2998b7aca..c100f10a0 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -99,7 +99,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.4* \ + containerd.io=1.5* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 9e01e1b28..913f2e38d 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -101,7 +101,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.4* \ + containerd.io=1.5* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 726a65c6d..6367bd5f2 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -102,7 +102,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.4* \ + containerd.io=1.5* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml index 8faed2f4a..a30b01173 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml @@ -99,7 +99,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.4* \ + containerd.io=1.5* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index 15e2bcce3..840756ce5 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -99,7 +99,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.4* \ + containerd.io=1.5* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 88007a439..8541fcfca 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -99,7 +99,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.4* \ + containerd.io=1.5* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml b/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml index 4b10b3a84..82318116f 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml @@ -99,7 +99,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.4* \ + containerd.io=1.5* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml index f6a1c61af..abe51931f 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml @@ -99,7 +99,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.4* \ + containerd.io=1.5* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index cd58f8d84..4372a2235 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -109,7 +109,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.4* \ + containerd.io=1.5* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index f99c2937b..035687f26 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -109,7 +109,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.4* \ + containerd.io=1.5* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 90edde887..c2f31b9f5 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -100,7 +100,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.4* \ + containerd.io=1.5* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 6f992772c..6c506ad6a 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -49,7 +49,7 @@ var ( } openStackImages = map[string]string{ - string(providerconfigtypes.OperatingSystemUbuntu): "machine-controller-e2e-ubuntu-20-04", + string(providerconfigtypes.OperatingSystemUbuntu): "kubermatic-ubuntu", string(providerconfigtypes.OperatingSystemCentOS): "machine-controller-e2e-centos", string(providerconfigtypes.OperatingSystemRHEL): "machine-controller-e2e-rhel-8-5", string(providerconfigtypes.OperatingSystemFlatcar): "machine-controller-e2e-flatcar-stable-2983", @@ -61,7 +61,15 @@ var ( string(providerconfigtypes.OperatingSystemFlatcar): "kkp-flatcar-3033.2.2", string(providerconfigtypes.OperatingSystemRHEL): "kkp-rhel-8.6", string(providerconfigtypes.OperatingSystemRockyLinux): "kkp-rockylinux-8", - string(providerconfigtypes.OperatingSystemUbuntu): "kkp-ubuntu-20.04", + string(providerconfigtypes.OperatingSystemUbuntu): "kkp-ubuntu-22.04", + } + + kubevirtImages = map[string]string{ + string(providerconfigtypes.OperatingSystemCentOS): "centos", + string(providerconfigtypes.OperatingSystemFlatcar): "flatcar", + string(providerconfigtypes.OperatingSystemRHEL): "rhel", + string(providerconfigtypes.OperatingSystemRockyLinux): "rockylinux", + string(providerconfigtypes.OperatingSystemUbuntu): "ubuntu-22.04", } ) @@ -237,6 +245,9 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar // only use by vSphere scenarios scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_Image_Template >>=%s", vSphereOSImageTemplates[testCase.osName])) + // only use by KubeVirt scenarios + scenarioParams = append(scenarioParams, fmt.Sprintf("<< KUBEVIRT_OS_IMAGE >>=%s", kubevirtImages[testCase.osName])) + // default kubeconfig to the hardcoded path at which `make e2e-cluster` creates its new kubeconfig gopath := os.Getenv("GOPATH") projectDir := filepath.Join(gopath, "src/github.com/kubermatic/machine-controller") diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index 5c66b5622..37c585d33 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -34,7 +34,7 @@ spec: cpus: "1" memory: "4096M" primaryDisk: - osImage: http://image-repo.kube-system.svc.cluster.local/images/<< OS_NAME >>.img + osImage: http://image-repo.kube-system.svc.cluster.local/images/<< KUBEVIRT_OS_IMAGE >>.img size: "25Gi" storageClassName: longhorn dnsPolicy: "None" From 1fc644f5d284f1e03610ce76276821afe5ebb1ac Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Wed, 21 Sep 2022 11:39:31 +0200 Subject: [PATCH 221/489] Add enableBootDiagnostics field to Azure cloud spec (#1438) * Add enableBootDiagnostics field to Azure cloud spec Signed-off-by: Marvin Beckers * Update complexity in .golangci.yml Signed-off-by: Marvin Beckers Signed-off-by: Marvin Beckers --- .golangci.yml | 2 +- pkg/cloudprovider/provider/azure/provider.go | 13 +++++++++++++ pkg/cloudprovider/provider/azure/types/types.go | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index ede7af007..08d4e6507 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -58,7 +58,7 @@ issues: - "cyclomatic complexity 31 of func `main` is high" - 'cyclomatic complexity 34 of func `\(\*provider\)\.getConfig` is high' - 'cyclomatic complexity 31 of func `\(\*provider\)\.Validate` is high' - - 'cyclomatic complexity 32 of func `\(\*provider\)\.Create` is high' + - 'cyclomatic complexity 33 of func `\(\*provider\)\.Create` is high' # SA1019: node.Spec.ConfigSource is deprecated: Previously used to specify the source of the node's configuration for the DynamicKubeletConfig feature. # This feature is removed from Kubelets as of 1.24 and will be fully removed in 1.26. +optional # We still support setting dynamic kubelet config feature in machine-controller. Hence, ignoring this error. diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index b88f54c69..f30e8ad5b 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -106,6 +106,7 @@ type config struct { AssignPublicIP bool PublicIPSKU *network.PublicIPAddressSkuName EnableAcceleratedNetworking *bool + EnableBootDiagnostics bool Tags map[string]string } @@ -380,6 +381,10 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p return nil, nil, fmt.Errorf("failed to get image id: %w", err) } + if rawCfg.EnableBootDiagnostics != nil { + c.EnableBootDiagnostics = *rawCfg.EnableBootDiagnostics + } + return &c, pconfig, nil } @@ -698,6 +703,14 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, vmSpec.VirtualMachineProperties.AvailabilitySet = &compute.SubResource{ID: to.StringPtr(asURI)} } + if config.EnableBootDiagnostics { + vmSpec.DiagnosticsProfile = &compute.DiagnosticsProfile{ + BootDiagnostics: &compute.BootDiagnostics{ + Enabled: pointer.Bool(config.EnableBootDiagnostics), + }, + } + } + klog.Infof("Creating machine %q", machine.Name) if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { if !kuberneteshelper.HasFinalizer(updatedMachine, finalizerDisks) { diff --git a/pkg/cloudprovider/provider/azure/types/types.go b/pkg/cloudprovider/provider/azure/types/types.go index a7b32de6a..7b472689e 100644 --- a/pkg/cloudprovider/provider/azure/types/types.go +++ b/pkg/cloudprovider/provider/azure/types/types.go @@ -43,6 +43,7 @@ type RawConfig struct { ImagePlan *ImagePlan `json:"imagePlan,omitempty"` ImageReference *ImageReference `json:"imageReference,omitempty"` EnableAcceleratedNetworking *bool `json:"enableAcceleratedNetworking"` + EnableBootDiagnostics *bool `json:"enableBootDiagnostics,omitempty"` ImageID providerconfigtypes.ConfigVarString `json:"imageID"` OSDiskSize int32 `json:"osDiskSize"` From 9f770ad8a9c10813f15f18f3e08a015cf53a5cee Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Thu, 22 Sep 2022 18:54:01 +0300 Subject: [PATCH 222/489] Upgrade containerd v1.6 (#1439) Signed-off-by: Artiom Diomin Signed-off-by: Artiom Diomin --- pkg/containerruntime/containerd.go | 2 +- pkg/containerruntime/docker.go | 2 +- pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml | 2 +- pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml | 2 +- pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml | 2 +- .../rockylinux/testdata/kubelet-v1.23-aws-external.yaml | 2 +- pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml | 2 +- pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml | 2 +- .../rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml | 2 +- .../rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml | 2 +- pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml | 2 +- pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml | 2 +- pkg/userdata/ubuntu/testdata/containerd.yaml | 2 +- pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml | 2 +- pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml | 2 +- pkg/userdata/ubuntu/testdata/docker.yaml | 2 +- .../ubuntu/testdata/kubelet-version-without-v-prefix.yaml | 2 +- pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml | 2 +- pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml | 2 +- pkg/userdata/ubuntu/testdata/nutanix.yaml | 2 +- pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml | 2 +- .../ubuntu/testdata/openstack-overwrite-cloud-config.yaml | 2 +- pkg/userdata/ubuntu/testdata/openstack.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.22.7.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.23.5.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.24.0.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere.yaml | 2 +- 45 files changed, 45 insertions(+), 45 deletions(-) diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index 413b0297f..248ff0bb4 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -28,7 +28,7 @@ import ( const ( LegacyContainerdVersion = "1.4" - DefaultContainerdVersion = "1.5" + DefaultContainerdVersion = "1.6" ) type Containerd struct { diff --git a/pkg/containerruntime/docker.go b/pkg/containerruntime/docker.go index c8973e625..173ce3f6c 100644 --- a/pkg/containerruntime/docker.go +++ b/pkg/containerruntime/docker.go @@ -28,7 +28,7 @@ import ( const ( LegacyDockerContainerdVersion = "1.4" - DefaultDockerContainerdVersion = "1.5" + DefaultDockerContainerdVersion = "1.6" DefaultDockerVersion = "20.10" LegacyDockerVersion = "19.03" ) diff --git a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml index d4d503441..df35961d0 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.22-aws.yaml @@ -99,7 +99,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml index ff6b3c5c5..c5189aac5 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml @@ -99,7 +99,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index 4a793afcd..f94370545 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -99,7 +99,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml index 2c6ecd081..19a57bbc9 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml @@ -106,7 +106,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 2c64afd93..dd1e83fc8 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -112,7 +112,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml index fb5cc23d0..6a7352d08 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -112,7 +112,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml index 9b68d8f96..25051264e 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml @@ -104,7 +104,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml index c2c9a339c..cf85e964d 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml @@ -100,7 +100,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - yum install -y containerd.io-1.5* yum-plugin-versionlock + yum install -y containerd.io-1.6* yum-plugin-versionlock yum versionlock add containerd.io systemctl daemon-reload diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml index 5ee3c66bb..2399d4188 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml @@ -94,7 +94,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml index ec9e1ee70..4ae2476f6 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml @@ -102,7 +102,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index db2355f59..a79090572 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -94,7 +94,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index d5bab82bd..159364690 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -94,7 +94,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 7e0ca2995..60d49d610 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -108,7 +108,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index 82c5677dc..dd7d36833 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -108,7 +108,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index 5b24323eb..1e4785fc6 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -100,7 +100,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml index 5b07fe6df..5cc284938 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml @@ -95,7 +95,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - yum install -y containerd.io-1.5* yum-plugin-versionlock + yum install -y containerd.io-1.6* yum-plugin-versionlock yum versionlock add containerd.io systemctl daemon-reload diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml index ecd6b661c..8e8a5918f 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml @@ -95,7 +95,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - yum install -y containerd.io-1.5* yum-plugin-versionlock + yum install -y containerd.io-1.6* yum-plugin-versionlock yum versionlock add containerd.io systemctl daemon-reload diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index df98b4fe5..7da831799 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -99,7 +99,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml index b4da33538..26715a389 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.22-aws.yaml @@ -95,7 +95,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml index 04eeda059..de9e8b81a 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml @@ -95,7 +95,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml index bc2e54389..871afd164 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml @@ -95,7 +95,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml index 6377d7b31..6280d8c5a 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml @@ -102,7 +102,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 47ebef3ee..c77004f2b 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -108,7 +108,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml index ba853e3f9..968eaf047 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -108,7 +108,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml index a2464cc16..2900f5c3c 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml @@ -100,7 +100,7 @@ write_files: yum install -y \ docker-ce-cli-20.10* \ - containerd.io-1.5* \ + containerd.io-1.6* \ docker-ce-20.10* \ yum-plugin-versionlock yum versionlock add docker-ce* containerd.io diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml index 9cb82e693..7af96633d 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml @@ -96,7 +96,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - yum install -y containerd.io-1.5* yum-plugin-versionlock + yum install -y containerd.io-1.6* yum-plugin-versionlock yum versionlock add containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index bf311fa08..fff144f5e 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -103,7 +103,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y --allow-downgrades containerd.io=1.5* + apt-get install -y --allow-downgrades containerd.io=1.6* apt-mark hold containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml index 7438e9e4b..83120b0c9 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml @@ -99,7 +99,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.5* \ + containerd.io=1.6* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index b0a6d2cbe..2857fb810 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -101,7 +101,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.5* \ + containerd.io=1.6* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml index d105086e8..d761c1a2e 100644 --- a/pkg/userdata/ubuntu/testdata/docker.yaml +++ b/pkg/userdata/ubuntu/testdata/docker.yaml @@ -101,7 +101,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.5* \ + containerd.io=1.6* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index 82318116f..fef7938b8 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -99,7 +99,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.5* \ + containerd.io=1.6* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index c100f10a0..5dd087238 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -99,7 +99,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.5* \ + containerd.io=1.6* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 913f2e38d..ed129f539 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -101,7 +101,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.5* \ + containerd.io=1.6* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 6367bd5f2..344806614 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -102,7 +102,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.5* \ + containerd.io=1.6* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml index a30b01173..be1c64a23 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml @@ -99,7 +99,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.5* \ + containerd.io=1.6* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index 840756ce5..5e7ec0126 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -99,7 +99,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.5* \ + containerd.io=1.6* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 8541fcfca..b087ed873 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -99,7 +99,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.5* \ + containerd.io=1.6* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml b/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml index 82318116f..fef7938b8 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml @@ -99,7 +99,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.5* \ + containerd.io=1.6* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml index abe51931f..6ad303d59 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml @@ -99,7 +99,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.5* \ + containerd.io=1.6* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml index 9d45afa30..d0773d79f 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml @@ -101,7 +101,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y --allow-downgrades containerd.io=1.5* + apt-get install -y --allow-downgrades containerd.io=1.6* apt-mark hold containerd.io systemctl daemon-reload diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index 4372a2235..31c647d6d 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -109,7 +109,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.5* \ + containerd.io=1.6* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 035687f26..16127b363 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -109,7 +109,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.5* \ + containerd.io=1.6* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index c2f31b9f5..8d838337d 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -100,7 +100,7 @@ write_files: EOF apt-get install --allow-downgrades -y \ - containerd.io=1.5* \ + containerd.io=1.6* \ docker-ce-cli=5:20.10* \ docker-ce=5:20.10* apt-mark hold docker-ce* containerd.io From 27a50539bcd610349872b87c61ea9cb4df22a8fe Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 23 Sep 2022 19:06:17 +0500 Subject: [PATCH 223/489] Update OSM image for e2e tests (#1440) * Update OSM image for e2e tests Signed-off-by: Waleed Malik * Add TODO Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- examples/operating-system-manager.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index 9e513819b..c025fa24b 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -1076,7 +1076,8 @@ spec: spec: serviceAccountName: operating-system-manager-webhook containers: - - image: quay.io/kubermatic/operating-system-manager:v1.0.0 + # TODO: Update this to a semver tag before release. + - image: quay.io/kubermatic/operating-system-manager:ea2250874bbcea46ad956cbba8972be7ffdf9ce2 imagePullPolicy: IfNotPresent name: webhook command: @@ -1377,7 +1378,8 @@ spec: spec: serviceAccountName: operating-system-manager containers: - - image: quay.io/kubermatic/operating-system-manager:v1.0.0 + # TODO: Update this to a semver tag before release. + - image: quay.io/kubermatic/operating-system-manager:ea2250874bbcea46ad956cbba8972be7ffdf9ce2 imagePullPolicy: IfNotPresent name: operating-system-manager command: From e1574d32343d5fb5f6f39eac0ec224f5e62b2ad8 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 27 Sep 2022 12:15:50 +0500 Subject: [PATCH 224/489] Upgrade to flannel v0.19.2 for E2E tests (#1444) Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- hack/ci/setup-kind-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/ci/setup-kind-cluster.sh b/hack/ci/setup-kind-cluster.sh index d03becb3a..063817c1d 100755 --- a/hack/ci/setup-kind-cluster.sh +++ b/hack/ci/setup-kind-cluster.sh @@ -146,7 +146,7 @@ if [ ! -f cni-plugin-deployed ]; then setup_cni_in_kind=$(cat hack/ci/setup-cni-in-kind.sh) docker exec $KIND_CLUSTER_NAME-control-plane bash -c "$setup_cni_in_kind &" ) - kubectl create -f https://raw.githubusercontent.com/flannel-io/flannel/v0.18.0/Documentation/kube-flannel.yml + kubectl create -f https://raw.githubusercontent.com/flannel-io/flannel/v0.19.2/Documentation/kube-flannel.yml touch cni-plugin-deployed fi From 0b32bbecb89b2459864192e1d3fb9289f0e04622 Mon Sep 17 00:00:00 2001 From: Marcin Franczyk Date: Fri, 30 Sep 2022 07:04:53 +0200 Subject: [PATCH 225/489] Add an option to enable nested virtualization and CPU platform for GCP (#1451) * Add an option to enable nested virtualization and CPU platform for GCP Signed-off-by: Marcin Franczyk * adjust minCPUPlatform name to linter rules Signed-off-by: Marcin Franczyk Signed-off-by: Marcin Franczyk --- examples/gce-machinedeployment.yaml | 2 ++ pkg/cloudprovider/provider/gce/config.go | 12 ++++++++++++ pkg/cloudprovider/provider/gce/provider.go | 10 ++++++++++ pkg/cloudprovider/provider/gce/provider_test.go | 2 ++ pkg/cloudprovider/provider/gce/types/types.go | 2 ++ 5 files changed, 28 insertions(+) diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index c7ab56ffa..84f247077 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -69,6 +69,8 @@ spec: assignPublicIPAddress: true customImage: "myCustomImage" disableMachineServiceAccount: false + enableNestedVirtualization: false + minCPUPlatform: "Intel Haswell" # Can be 'ubuntu' or 'rhel' operatingSystem: "ubuntu" operatingSystemSpec: diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index 9901b7af7..28777faab 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -112,6 +112,8 @@ type config struct { regional bool customImage string disableMachineServiceAccount bool + enableNestedVirtualization bool + minCPUPlatform string } // newConfig creates a Provider configuration out of the passed resolver and spec. @@ -220,6 +222,16 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide return nil, fmt.Errorf("failed to retrieve disable machine service account: %w", err) } + cfg.enableNestedVirtualization, _, err = resolver.GetConfigVarBoolValue(cpSpec.EnableNestedVirtualization) + if err != nil { + return nil, fmt.Errorf("failed to retrieve enable nested virtualization: %w", err) + } + + cfg.minCPUPlatform, err = resolver.GetConfigVarStringValue(cpSpec.MinCPUPlatform) + if err != nil { + return nil, fmt.Errorf("failed to retrieve min cpu platform: %w", err) + } + return cfg, nil } diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 58245db03..28e7cc3f4 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -280,6 +280,16 @@ func (p *Provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, inst.Scheduling.ProvisioningModel = *cfg.provisioningModel } + if cfg.enableNestedVirtualization { + inst.AdvancedMachineFeatures = &compute.AdvancedMachineFeatures{ + EnableNestedVirtualization: true, + } + } + + if cfg.minCPUPlatform != "" { + inst.MinCpuPlatform = cfg.minCPUPlatform + } + op, err := svc.Instances.Insert(cfg.projectID, cfg.zone, inst).Do() if err != nil { return nil, newError(common.InvalidConfigurationMachineError, errInsertInstance, err) diff --git a/pkg/cloudprovider/provider/gce/provider_test.go b/pkg/cloudprovider/provider/gce/provider_test.go index c792c6555..f3ffeddc9 100644 --- a/pkg/cloudprovider/provider/gce/provider_test.go +++ b/pkg/cloudprovider/provider/gce/provider_test.go @@ -55,6 +55,8 @@ func testProviderSpec() map[string]interface{} { }, "zone": "europe-west2-a", "disableMachineServiceAccount": false, + "enableNestedVirtualization": true, + "minCPUPlatform": "Intel Haswell", }, "operatingSystem": "ubuntu", "operatingSystemSpec": map[string]interface{}{ diff --git a/pkg/cloudprovider/provider/gce/types/types.go b/pkg/cloudprovider/provider/gce/types/types.go index 0ac0f3335..0bc95de02 100644 --- a/pkg/cloudprovider/provider/gce/types/types.go +++ b/pkg/cloudprovider/provider/gce/types/types.go @@ -48,6 +48,8 @@ type CloudProviderSpec struct { Regional providerconfigtypes.ConfigVarBool `json:"regional"` CustomImage providerconfigtypes.ConfigVarString `json:"customImage,omitempty"` DisableMachineServiceAccount providerconfigtypes.ConfigVarBool `json:"disableMachineServiceAccount,omitempty"` + EnableNestedVirtualization providerconfigtypes.ConfigVarBool `json:"enableNestedVirtualization,omitempty"` + MinCPUPlatform providerconfigtypes.ConfigVarString `json:"MinCPUPlatform,omitempty"` } // UpdateProviderSpec updates the given provider spec with changed From 5819bb5f27f7cb9f0ca83045de1eb0f5de122af5 Mon Sep 17 00:00:00 2001 From: Sankalp Rangare Date: Fri, 30 Sep 2022 07:53:19 +0200 Subject: [PATCH 226/489] add support for topology spread constaint for VMs (#1445) Signed-off-by: Sankalp Rangare Signed-off-by: Sankalp Rangare --- examples/kubevirt-machinedeployment.yaml | 7 + go.mod | 2 +- go.sum | 4 +- .../provider/kubevirt/provider.go | 140 ++++++++---------- .../provider/kubevirt/provider_test.go | 57 +++++++ .../provider/kubevirt/types/types.go | 22 ++- 6 files changed, 150 insertions(+), 82 deletions(-) create mode 100644 pkg/cloudprovider/provider/kubevirt/provider_test.go diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index c7d959a25..f2eecf71d 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -40,13 +40,20 @@ spec: size: "10Gi" storageClassName: kubermatic-fast affinity: + # Deprecated: Use topologySpreadConstraints instead. podAffinityPreset: "" # Allowed values: "", "soft", "hard" + # Deprecated: Use topologySpreadConstraints instead. podAntiAffinityPreset: "" # Allowed values: "", "soft", "hard" nodeAffinityPreset: type: "" # Allowed values: "", "soft", "hard" key: "foo" values: - bar + topologySpreadConstraints: + - maxSkew: "1" + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: "" # Allowed values: "DoNotSchedule", "ScheduleAnyway" + # Can also be `centos`, must align with he configured registryImage above operatingSystem: "ubuntu" operatingSystemSpec: diff --git a/go.mod b/go.mod index 6518d124e..26511e312 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( k8s.io/klog v1.0.0 k8s.io/kubelet v0.24.2 k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed - kubevirt.io/api v0.54.0 + kubevirt.io/api v0.57.1 kubevirt.io/containerized-data-importer-api v1.50.0 sigs.k8s.io/controller-runtime v0.12.1 sigs.k8s.io/yaml v1.3.0 diff --git a/go.sum b/go.sum index 0c6d23eaf..e5ecf23c7 100644 --- a/go.sum +++ b/go.sum @@ -1588,8 +1588,8 @@ k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -kubevirt.io/api v0.54.0 h1:rVHaKrsxpYf5Cu6rhASOxNTChS76Nvtn5tArtG2M2Ds= -kubevirt.io/api v0.54.0/go.mod h1:mK8ilpVLcZraqgo7hv2OSNQ5vdsA3G9Pxn8LY2/1+IY= +kubevirt.io/api v0.57.1 h1:z6ImWKCQL2efFYqMWmxEsDNyt8c6mbWk7oCY6ZAa06U= +kubevirt.io/api v0.57.1/go.mod h1:U0CQlZR0JoJCaC+Va0wz4dMOtYDdVywJ98OT1KmOkzI= kubevirt.io/containerized-data-importer-api v1.50.0 h1:O01F8L5K8qRLnkYICIfmAu0dU0P48jdO42uFPElht38= kubevirt.io/containerized-data-importer-api v1.50.0/go.mod h1:yjD8pGZVMCeqcN46JPUQdZ2JwRVoRCOXrTVyNuFvrLo= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 2523c62f1..fab927642 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -86,21 +86,22 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes } type Config struct { - Kubeconfig string - RestConfig *rest.Config - DNSConfig *corev1.PodDNSConfig - DNSPolicy corev1.DNSPolicy - CPUs string - Memory string - Namespace string - OsImage OSImage - StorageClassName string - PVCSize resource.Quantity - FlavorName string - SecondaryDisks []SecondaryDisks - PodAffinityPreset AffinityType - PodAntiAffinityPreset AffinityType - NodeAffinityPreset NodeAffinityPreset + Kubeconfig string + RestConfig *rest.Config + DNSConfig *corev1.PodDNSConfig + DNSPolicy corev1.DNSPolicy + CPUs string + Memory string + Namespace string + OsImage OSImage + StorageClassName string + PVCSize resource.Quantity + FlavorName string + SecondaryDisks []SecondaryDisks + PodAffinityPreset AffinityType + PodAntiAffinityPreset AffinityType + NodeAffinityPreset NodeAffinityPreset + TopologySpreadConstraints []corev1.TopologySpreadConstraint } type AffinityType string @@ -304,19 +305,14 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p }) } - // Affinity/AntiAffinity - config.PodAffinityPreset, err = p.affinityType(rawConfig.Affinity.PodAffinityPreset) - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "podAffinityPreset" field: %w`, err) - } - config.PodAntiAffinityPreset, err = p.affinityType(rawConfig.Affinity.PodAntiAffinityPreset) - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "podAntiAffinityPreset" field: %w`, err) - } config.NodeAffinityPreset, err = p.parseNodeAffinityPreset(rawConfig.Affinity.NodeAffinityPreset) if err != nil { return nil, nil, fmt.Errorf(`failed to parse "nodeAffinityPreset" field: %w`, err) } + config.TopologySpreadConstraints, err = p.parseTopologySpreadConstraint(rawConfig.TopologySpreadConstraints) + if err != nil { + return nil, nil, fmt.Errorf(`failed to parse "topologySpreadConstraints" field: %w`, err) + } return &config, pconfig, nil } @@ -343,6 +339,34 @@ func (p *provider) parseNodeAffinityPreset(nodeAffinityPreset kubevirttypes.Node return nodeAffinity, nil } +func (p *provider) parseTopologySpreadConstraint(topologyConstraints []kubevirttypes.TopologySpreadConstraint) ([]corev1.TopologySpreadConstraint, error) { + parsedTopologyConstraints := make([]corev1.TopologySpreadConstraint, 0, len(topologyConstraints)) + for _, constraint := range topologyConstraints { + maxSkewString, err := p.configVarResolver.GetConfigVarStringValue(constraint.MaxSkew) + if err != nil { + return nil, fmt.Errorf(`failed to parse "topologySpreadConstraint.maxSkew" field: %w`, err) + } + maxSkew, err := strconv.ParseInt(maxSkewString, 10, 32) + if err != nil { + return nil, fmt.Errorf(`failed to parse "topologySpreadConstraint.maxSkew" field: %w`, err) + } + topologyKey, err := p.configVarResolver.GetConfigVarStringValue(constraint.TopologyKey) + if err != nil { + return nil, fmt.Errorf(`failed to parse "topologySpreadConstraint.topologyKey" field: %w`, err) + } + whenUnsatisfiable, err := p.configVarResolver.GetConfigVarStringValue(constraint.WhenUnsatisfiable) + if err != nil { + return nil, fmt.Errorf(`failed to parse "topologySpreadConstraint.whenUnsatisfiable" field: %w`, err) + } + parsedTopologyConstraints = append(parsedTopologyConstraints, corev1.TopologySpreadConstraint{ + MaxSkew: int32(maxSkew), + TopologyKey: topologyKey, + WhenUnsatisfiable: corev1.UnsatisfiableConstraintAction(whenUnsatisfiable), + }) + } + return parsedTopologyConstraints, nil +} + // getNamespace returns the namespace where the VM is created. // VM is created in a dedicated namespace // which is the namespace where the machine-controller pod is running. @@ -572,6 +596,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, Volumes: getVMVolumes(c, dataVolumeName, userDataSecretName), DNSPolicy: c.DNSPolicy, DNSConfig: c.DNSConfig, + TopologySpreadConstraints: getTopologySpreadConstraints(c, map[string]string{machineDeploymentLabelKey: labels[machineDeploymentLabelKey]}), }, }, DataVolumeTemplates: getDataVolumeTemplates(c, dataVolumeName), @@ -781,30 +806,6 @@ func getDataVolumeSource(osImage OSImage) *cdiv1beta1.DataVolumeSource { func getAffinity(config *Config, matchKey, matchValue string) *corev1.Affinity { affinity := &corev1.Affinity{} - // PodAffinity - switch config.PodAffinityPreset { - case softAffinityType: - affinity.PodAffinity = &corev1.PodAffinity{ - PreferredDuringSchedulingIgnoredDuringExecution: hostnameWeightedAffinityTerm(matchKey, matchValue), - } - case hardAffinityType: - affinity.PodAffinity = &corev1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: hostnameAffinityTerm(matchKey, matchValue), - } - } - - // PodAntiAffinity - switch config.PodAntiAffinityPreset { - case softAffinityType: - affinity.PodAntiAffinity = &corev1.PodAntiAffinity{ - PreferredDuringSchedulingIgnoredDuringExecution: hostnameWeightedAffinityTerm(matchKey, matchValue), - } - case hardAffinityType: - affinity.PodAntiAffinity = &corev1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: hostnameAffinityTerm(matchKey, matchValue), - } - } - // NodeAffinity switch config.NodeAffinityPreset.Type { case softAffinityType: @@ -845,37 +846,26 @@ func getAffinity(config *Config, matchKey, matchValue string) *corev1.Affinity { return affinity } -func hostnameWeightedAffinityTerm(matchKey, matchValue string) []corev1.WeightedPodAffinityTerm { - return []corev1.WeightedPodAffinityTerm{ - { - Weight: 1, - PodAffinityTerm: corev1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - matchKey: matchValue, - }, - }, - TopologyKey: topologyKeyHostname, - }, - }, +func addPrefixToSecondaryDisk(secondaryDisks []SecondaryDisks, prefix string) { + for i := range secondaryDisks { + secondaryDisks[i].Name = fmt.Sprintf("%s-%s", prefix, secondaryDisks[i].Name) } } -func hostnameAffinityTerm(matchKey, matchValue string) []corev1.PodAffinityTerm { - return []corev1.PodAffinityTerm{ +func getTopologySpreadConstraints(config *Config, matchLabels map[string]string) []corev1.TopologySpreadConstraint { + if len(config.TopologySpreadConstraints) != 0 { + for i := range config.TopologySpreadConstraints { + config.TopologySpreadConstraints[i].LabelSelector = &metav1.LabelSelector{MatchLabels: matchLabels} + } + return config.TopologySpreadConstraints + } + // Return default TopologySpreadConstraint + return []corev1.TopologySpreadConstraint{ { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - matchKey: matchValue, - }, - }, - TopologyKey: topologyKeyHostname, + MaxSkew: 1, + TopologyKey: topologyKeyHostname, + WhenUnsatisfiable: corev1.ScheduleAnyway, + LabelSelector: &metav1.LabelSelector{MatchLabels: matchLabels}, }, } } - -func addPrefixToSecondaryDisk(secondaryDisks []SecondaryDisks, prefix string) { - for i := range secondaryDisks { - secondaryDisks[i].Name = fmt.Sprintf("%s-%s", prefix, secondaryDisks[i].Name) - } -} diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go new file mode 100644 index 000000000..b65178317 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -0,0 +1,57 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubevirt + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestTopologySpreadConstraint(t *testing.T) { + tests := []struct { + desc string + config Config + expected []corev1.TopologySpreadConstraint + }{ + { + desc: "default topology constraint", + config: Config{TopologySpreadConstraints: nil}, + expected: []corev1.TopologySpreadConstraint{ + {MaxSkew: 1, TopologyKey: topologyKeyHostname, WhenUnsatisfiable: corev1.ScheduleAnyway, LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"md": "test-md"}}}, + }, + }, + { + desc: "custom topology constraint", + config: Config{TopologySpreadConstraints: []corev1.TopologySpreadConstraint{{MaxSkew: 1, TopologyKey: "test-topology-key", WhenUnsatisfiable: corev1.DoNotSchedule}}}, + expected: []corev1.TopologySpreadConstraint{ + {MaxSkew: 1, TopologyKey: "test-topology-key", WhenUnsatisfiable: corev1.DoNotSchedule, LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"md": "test-md"}}}, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + result := getTopologySpreadConstraints(&test.config, map[string]string{"md": "test-md"}) + if !reflect.DeepEqual(result, test.expected) { + t.Errorf("expected ToplogySpreadConstraint: %v, got: %v", test.expected, result) + } + }) + } +} diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index 8eaab19b1..7cca0a96a 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -24,9 +24,10 @@ import ( ) type RawConfig struct { - Auth Auth `json:"auth,omitempty"` - VirtualMachine VirtualMachine `json:"virtualMachine,omitempty"` - Affinity Affinity `json:"affinity,omitempty"` + Auth Auth `json:"auth,omitempty"` + VirtualMachine VirtualMachine `json:"virtualMachine,omitempty"` + Affinity Affinity `json:"affinity,omitempty"` + TopologySpreadConstraints []TopologySpreadConstraint `json:"topologySpreadConstraints"` } // Auth. @@ -75,7 +76,9 @@ type Disk struct { // Affinity. type Affinity struct { - PodAffinityPreset providerconfigtypes.ConfigVarString `json:"podAffinityPreset,omitempty"` + // Deprecated: Use TopologySpreadConstraint instead. + PodAffinityPreset providerconfigtypes.ConfigVarString `json:"podAffinityPreset,omitempty"` + // Deprecated: Use TopologySpreadConstraint instead. PodAntiAffinityPreset providerconfigtypes.ConfigVarString `json:"podAntiAffinityPreset,omitempty"` NodeAffinityPreset NodeAffinityPreset `json:"nodeAffinityPreset,omitempty"` } @@ -87,6 +90,17 @@ type NodeAffinityPreset struct { Values []providerconfigtypes.ConfigVarString `json:"values,omitempty"` } +// TopologySpreadConstraint describes topology spread constraints for VMs. +type TopologySpreadConstraint struct { + // MaxSkew describes the degree to which VMs may be unevenly distributed. + MaxSkew providerconfigtypes.ConfigVarString `json:"maxSkew,omitempty"` + // TopologyKey is the key of infra-node labels. + TopologyKey providerconfigtypes.ConfigVarString `json:"topologyKey,omitempty"` + // WhenUnsatisfiable indicates how to deal with a VM if it doesn't satisfy + // the spread constraint. + WhenUnsatisfiable providerconfigtypes.ConfigVarString `json:"whenUnsatisfiable,omitempty"` +} + func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { rawConfig := &RawConfig{} From 406d1ef560157fddb2182e18c8ecb5ea5cc4a1d6 Mon Sep 17 00:00:00 2001 From: Marcin Franczyk Date: Tue, 4 Oct 2022 08:33:10 +0200 Subject: [PATCH 227/489] Add an option to enable Guest OS Features at GCP (#1453) Signed-off-by: Marcin Franczyk Signed-off-by: Marcin Franczyk --- examples/gce-machinedeployment.yaml | 3 +++ pkg/cloudprovider/provider/gce/config.go | 10 ++++++---- pkg/cloudprovider/provider/gce/provider_test.go | 4 ++++ pkg/cloudprovider/provider/gce/service.go | 5 +++++ pkg/cloudprovider/provider/gce/types/types.go | 3 ++- 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index 84f247077..41f5105c4 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -71,6 +71,9 @@ spec: disableMachineServiceAccount: false enableNestedVirtualization: false minCPUPlatform: "Intel Haswell" + guestOSFeatures: + - "VIRTIO_SCSI_MULTIQUEUE" + - "GVNIC" # Can be 'ubuntu' or 'rhel' operatingSystem: "ubuntu" operatingSystemSpec: diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index 28777faab..79bcd81d4 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -114,6 +114,7 @@ type config struct { disableMachineServiceAccount bool enableNestedVirtualization bool minCPUPlatform string + guestOSFeatures []string } // newConfig creates a Provider configuration out of the passed resolver and spec. @@ -126,10 +127,11 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide // Setup configuration. cfg := &config{ - providerConfig: providerConfig, - labels: cpSpec.Labels, - tags: cpSpec.Tags, - diskSize: cpSpec.DiskSize, + providerConfig: providerConfig, + labels: cpSpec.Labels, + tags: cpSpec.Tags, + diskSize: cpSpec.DiskSize, + guestOSFeatures: cpSpec.GuestOSFeatures, } cfg.serviceAccount, err = resolver.GetConfigVarStringValueOrEnv(cpSpec.ServiceAccount, envGoogleServiceAccount) diff --git a/pkg/cloudprovider/provider/gce/provider_test.go b/pkg/cloudprovider/provider/gce/provider_test.go index f3ffeddc9..d1217a2f8 100644 --- a/pkg/cloudprovider/provider/gce/provider_test.go +++ b/pkg/cloudprovider/provider/gce/provider_test.go @@ -57,6 +57,10 @@ func testProviderSpec() map[string]interface{} { "disableMachineServiceAccount": false, "enableNestedVirtualization": true, "minCPUPlatform": "Intel Haswell", + "guestOSFeatures": []string{ + "VIRTIO_SCSI_MULTIQUEUE", + "GVNIC", + }, }, "operatingSystem": "ubuntu", "operatingSystemSpec": map[string]interface{}{ diff --git a/pkg/cloudprovider/provider/gce/service.go b/pkg/cloudprovider/provider/gce/service.go index 840695c48..5b15873f2 100644 --- a/pkg/cloudprovider/provider/gce/service.go +++ b/pkg/cloudprovider/provider/gce/service.go @@ -125,6 +125,11 @@ func (svc *service) attachedDisks(cfg *config) ([]*compute.AttachedDisk, error) SourceImage: sourceImage, }, } + for _, v := range cfg.guestOSFeatures { + bootDisk.GuestOsFeatures = append(bootDisk.GuestOsFeatures, &compute.GuestOsFeature{ + Type: v, + }) + } return []*compute.AttachedDisk{bootDisk}, nil } diff --git a/pkg/cloudprovider/provider/gce/types/types.go b/pkg/cloudprovider/provider/gce/types/types.go index 0bc95de02..8b8736bbe 100644 --- a/pkg/cloudprovider/provider/gce/types/types.go +++ b/pkg/cloudprovider/provider/gce/types/types.go @@ -49,7 +49,8 @@ type CloudProviderSpec struct { CustomImage providerconfigtypes.ConfigVarString `json:"customImage,omitempty"` DisableMachineServiceAccount providerconfigtypes.ConfigVarBool `json:"disableMachineServiceAccount,omitempty"` EnableNestedVirtualization providerconfigtypes.ConfigVarBool `json:"enableNestedVirtualization,omitempty"` - MinCPUPlatform providerconfigtypes.ConfigVarString `json:"MinCPUPlatform,omitempty"` + MinCPUPlatform providerconfigtypes.ConfigVarString `json:"minCPUPlatform,omitempty"` + GuestOSFeatures []string `json:"guestOSFeatures,omitempty"` } // UpdateProviderSpec updates the given provider spec with changed From d41d7d9ce97ff62d7f3629495fce00c355e3ae53 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Tue, 4 Oct 2022 16:46:05 +0200 Subject: [PATCH 228/489] Rework OSM integration into generic external bootstrap "interface" (#1428) * Deprecate use-osm flag, remove osm dependency Signed-off-by: Marvin Beckers * Add a note to pkg/bootstrap/types.go Signed-off-by: Marvin Beckers * Remove provisioning secret as implementation detail of OSM Signed-off-by: Marvin Beckers * Add legacy label again but also set it if external bootstrapping is disabled Signed-off-by: Marvin Beckers * Fix linting issues Signed-off-by: Marvin Beckers * Annotate testdata with default OSPs Signed-off-by: Marvin Beckers Signed-off-by: Marvin Beckers --- cmd/machine-controller/main.go | 19 ++++---- cmd/webhook/main.go | 27 +++++------ go.mod | 5 +- go.sum | 27 +---------- pkg/admission/admission.go | 42 ++++++++--------- pkg/admission/machinedeployments.go | 10 +--- .../machinedeployments_validation.go | 34 +------------- pkg/admission/machines.go | 15 +++--- pkg/bootstrap/doc.go | 39 ++++++++++++++++ pkg/bootstrap/types.go | 37 +++++++++++++++ pkg/controller/machine/machine_controller.go | 46 +++++-------------- pkg/providerconfig/types.go | 4 +- pkg/userdata/flatcar/flatcar.go | 4 +- .../testdata/machinedeployment-alibaba.yaml | 2 + .../testdata/machinedeployment-anexia.yaml | 4 +- .../machinedeployment-aws-arm-machines.yaml | 2 + ...deployment-aws-ebs-encryption-enabled.yaml | 4 +- .../machinedeployment-aws-spot-instances.yaml | 4 +- .../testdata/machinedeployment-aws.yaml | 4 +- ...ployment-azure-custom-image-reference.yaml | 2 + ...hinedeployment-azure-redhat-satellite.yaml | 2 + .../testdata/machinedeployment-azure.yaml | 2 + ...achinedeployment-baremetal-tinkerbell.yaml | 2 + .../machinedeployment-digitalocean.yaml | 2 + .../machinedeployment-equinixmetal.yaml | 2 + .../testdata/machinedeployment-gce.yaml | 4 +- .../testdata/machinedeployment-hetzner.yaml | 2 + .../testdata/machinedeployment-kubevirt.yaml | 4 +- .../testdata/machinedeployment-linode.yaml | 2 + .../testdata/machinedeployment-nutanix.yaml | 2 + ...hinedeployment-openstack-project-auth.yaml | 2 + .../machinedeployment-openstack-upgrade.yml | 2 + .../testdata/machinedeployment-openstack.yaml | 2 + .../testdata/machinedeployment-scaleway.yaml | 2 + ...chinedeployment-vmware-cloud-director.yaml | 2 + ...edeployment-vsphere-datastore-cluster.yaml | 2 + ...chinedeployment-vsphere-resource-pool.yaml | 2 + .../machinedeployment-vsphere-static-ip.yaml | 14 +++--- .../testdata/machinedeployment-vsphere.yaml | 2 + 39 files changed, 209 insertions(+), 176 deletions(-) create mode 100644 pkg/bootstrap/doc.go create mode 100644 pkg/bootstrap/types.go diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 320e47052..177841b78 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -41,7 +41,6 @@ import ( "github.com/kubermatic/machine-controller/pkg/health" machinesv1alpha1 "github.com/kubermatic/machine-controller/pkg/machines/v1alpha1" "github.com/kubermatic/machine-controller/pkg/node" - osmv1alpha1 "k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/types" @@ -72,7 +71,8 @@ var ( enableLeaderElection bool leaderElectionNamespace string - useOSM bool + useOSM bool + useExternalBootstrap bool nodeCSRApprover bool nodeHTTPProxy string @@ -129,7 +129,8 @@ type controllerRunOptions struct { node machinecontroller.NodeSettings - useOSM bool + // Enable external bootstrap management by consuming secrets that are used to configure an instance's user-data. + useExternalBootstrap bool // A port range to reserve for services with NodePort visibility. nodePortRange string @@ -175,7 +176,8 @@ func main() { flag.StringVar(&podCIDR, "pod-cidr", "172.25.0.0/16", "WARNING: flag is unused, kept only for backwards compatibility") flag.StringVar(&nodePortRange, "node-port-range", "30000-32767", "A port range to reserve for services with NodePort visibility") flag.StringVar(&nodeRegistryCredentialsSecret, "node-registry-credentials-secret", "", "A Secret object reference, that contains auth info for image registry in namespace/secret-name form, example: kube-system/registry-credentials. See doc at https://github.com/kubermaric/machine-controller/blob/master/docs/registry-authentication.md") - flag.BoolVar(&useOSM, "use-osm", false, "use osm controller for node bootstrap") + flag.BoolVar(&useOSM, "use-osm", false, "DEPRECATED: use osm controller for node bootstrap [use use-external-bootstrap instead]") + flag.BoolVar(&useExternalBootstrap, "use-external-bootstrap", false, "use an external bootstrap provider for instance user-data (e.g. operating-system-manager, also known as OSM)") flag.StringVar(&overrideBootstrapKubeletAPIServer, "override-bootstrap-kubelet-apiserver", "", "Override for the API server address used in worker nodes bootstrap-kubelet.conf") flag.Parse() @@ -207,11 +209,6 @@ func main() { klog.Fatalf("failed to add clusterv1alpha1 api to scheme: %v", err) } - // needed for OSM - if err := osmv1alpha1.AddToScheme(scheme.Scheme); err != nil { - klog.Fatalf("failed to add osmv1alpha1 api to scheme: %v", err) - } - cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) if err != nil { klog.Fatalf("error building kubeconfig: %v", err) @@ -271,7 +268,7 @@ func main() { RegistryCredentialsSecretRef: nodeRegistryCredentialsSecret, ContainerRuntime: containerRuntimeConfig, }, - useOSM: useOSM, + useExternalBootstrap: useExternalBootstrap || useOSM, nodePortRange: nodePortRange, overrideBootstrapKubeletAPIServer: overrideBootstrapKubeletAPIServer, } @@ -409,7 +406,7 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { bs.opt.bootstrapTokenServiceAccountName, bs.opt.skipEvictionAfter, bs.opt.node, - bs.opt.useOSM, + bs.opt.useExternalBootstrap, bs.opt.nodePortRange, bs.opt.overrideBootstrapKubeletAPIServer, ); err != nil { diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 61c2bc904..c3d4af796 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -25,9 +25,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/node" userdatamanager "github.com/kubermatic/machine-controller/pkg/userdata/manager" - osmv1alpha1 "k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1" - "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/clientcmd" "k8s.io/klog" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -41,6 +39,7 @@ type options struct { admissionTLSKeyPath string caBundleFile string useOSM bool + useExternalBootstrap bool namespace string workerClusterKubeconfig string versionConstraint string @@ -66,7 +65,8 @@ func main() { flag.StringVar(&opt.versionConstraint, "kubernetes-version-constraints", ">=0.0.0", "") // OSM specific flags - flag.BoolVar(&opt.useOSM, "use-osm", false, "osm controller is enabled for node bootstrap") + flag.BoolVar(&opt.useOSM, "use-osm", false, "DEPRECATED: osm controller is enabled for node bootstrap [use use-external-bootstrap instead]") + flag.BoolVar(&opt.useExternalBootstrap, "use-external-bootstrap", false, "user-data is provided by external bootstrap mechanism (e.g. operating-system-manager, also known as OSM)") flag.Parse() opt.kubeconfig = flag.Lookup("kubeconfig").Value.(flag.Getter).Get().(string) @@ -78,11 +78,6 @@ func main() { } } - // Add osmv1alpha1 to scheme - if err := osmv1alpha1.AddToScheme(scheme.Scheme); err != nil { - klog.Fatalf("failed to add osmv1alpha1 api to scheme: %v", err) - } - cfg, err := clientcmd.BuildConfigFromFlags(opt.masterURL, opt.kubeconfig) if err != nil { klog.Fatalf("error building kubeconfig: %v", err) @@ -122,14 +117,14 @@ func main() { } srv, err := admission.Builder{ - ListenAddress: opt.admissionListenAddress, - Client: client, - WorkerClient: workerClient, - UserdataManager: um, - NodeFlags: nodeFlags, - UseOSM: opt.useOSM, - Namespace: opt.namespace, - VersionConstraints: constraint, + ListenAddress: opt.admissionListenAddress, + Client: client, + WorkerClient: workerClient, + UserdataManager: um, + UseExternalBootstrap: opt.useExternalBootstrap || opt.useOSM, + NodeFlags: nodeFlags, + Namespace: opt.namespace, + VersionConstraints: constraint, }.Build() if err != nil { klog.Fatalf("failed to create admission hook: %v", err) diff --git a/go.mod b/go.mod index 26511e312..fba2093ad 100644 --- a/go.mod +++ b/go.mod @@ -48,8 +48,6 @@ require ( google.golang.org/grpc v1.45.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 - // Please ensure that you update the image tags in `examples/operating-system-manager.yaml` as well. - k8c.io/operating-system-manager v1.0.0 k8s.io/api v0.25.0 k8s.io/apiextensions-apiserver v0.24.2 k8s.io/apimachinery v0.25.0 @@ -95,10 +93,9 @@ require ( github.com/docker/distribution v2.7.1+incompatible // indirect github.com/emicklei/go-restful/v3 v3.8.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/flatcar-linux/container-linux-config-transpiler v0.9.3 // indirect - github.com/flatcar-linux/ignition v0.36.1 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/zapr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.21.1 // indirect diff --git a/go.sum b/go.sum index e5ecf23c7..0f713e642 100644 --- a/go.sum +++ b/go.sum @@ -107,7 +107,6 @@ github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWX github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/ajeddeloh/go-json v0.0.0-20160803184958-73d058cf8437/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 h1:4SPQljF/GJ8Q+QlCWMWxRBepub4DresnOm4eI2ebFGc= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd h1:NlKlOv3aVJ5ODMC0JWPvddw05KENkL3cZttIuu8kJRo= @@ -118,7 +117,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1645 h1:IEL/Da0Dtg9j/36UnzyxD84n0eDj0JIoTKTKobN2eks= @@ -138,7 +136,6 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.16.12 h1:wbMYa2PlFysFx2GLIQojr6FJV5+OWCM/BwyHXARxETA= @@ -168,7 +165,6 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.16.15/go.mod h1:Y+BUV19q3OmQVqNUlbZ4 github.com/aws/smithy-go v1.13.0 h1:YfyEmSJLo7fAv8FbuDK4R8F9aAmi9DZ88Zb/KJJmUl0= github.com/aws/smithy-go v1.13.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -215,12 +211,10 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.1.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -288,10 +282,6 @@ github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flatcar-linux/container-linux-config-transpiler v0.9.3 h1:0Leh4HX8Wpe/PYuNidytk6v+2mIFHybK50DWipiCnng= -github.com/flatcar-linux/container-linux-config-transpiler v0.9.3/go.mod h1:AGVTulMzeIKwurV9ExYH3UiokET1Ur65g+EIeRDMwzM= -github.com/flatcar-linux/ignition v0.36.1 h1:yNvS9sQvm9HJ8VgxXskx88DsF73qdF35ALJkbTwcYhY= -github.com/flatcar-linux/ignition v0.36.1/go.mod h1:0jS5n4AopgOdwgi7QDo5MFgkMx/fQUDYjuxlGJC1Txg= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -312,7 +302,6 @@ github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aev github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= @@ -332,6 +321,7 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= +github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= @@ -366,7 +356,6 @@ github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8z github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kETFqjFoFlOc= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godror/godror v0.13.3/go.mod h1:2ouUT4kdhUBk7TAkHWD4SN0CdI0pgEQbo8FVHhbSKWg= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= @@ -545,7 +534,6 @@ github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -715,7 +703,6 @@ github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIw github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= @@ -727,7 +714,6 @@ github.com/peterhellberg/link v1.1.0 h1:s2+RH8EGuI/mI4QwrWGSYQCRz7uNgip9BaM04HKu github.com/peterhellberg/link v1.1.0/go.mod h1:gtSlOT4jmkY8P47hbTc8PTgiDDWpdPbFYl75keYyBB8= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pin/tftp v2.1.0+incompatible/go.mod h1:xVpZOMCXTy+A5QMjEVN0Glwa1sUvaJhFXbr/aAxuxGY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -806,8 +792,6 @@ github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sigma/bdoor v0.0.0-20160202064022-babf2a4017b0/go.mod h1:WBu7REWbxC/s/J06jsk//d+9DOz9BbsmcIrimuGRFbs= -github.com/sigma/vmw-guestinfo v0.0.0-20160204083807-95dd4126d6e8/go.mod h1:JrRFFC0veyh0cibh0DAhriSY7/gV3kDdNaVUOmfx01U= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -815,9 +799,7 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= @@ -876,8 +858,6 @@ github.com/vmware/go-vcloud-director/v2 v2.15.0 h1:idQ9NsHLr2dOSLBC8KIdBMq7XOvPi github.com/vmware/go-vcloud-director/v2 v2.15.0/go.mod h1:2BS1yw61VN34WI0/nUYoInFvBc3Zcuf84d4ESiAAl68= github.com/vmware/govmomi v0.28.0 h1:VgeQ/Rvz79U9G8QIKLdgpsN9AndHJL+5iMJLgYIrBGI= github.com/vmware/govmomi v0.28.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= -github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= -github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= @@ -956,7 +936,6 @@ go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go4.org v0.0.0-20160314031811-03efcb870d84/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1031,7 +1010,6 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1215,7 +1193,6 @@ golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20190321115727-fe223c5a2583/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -1546,8 +1523,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8c.io/operating-system-manager v1.0.0 h1:E1dCaLHypgaaLNgm50jcT3uwk3vok3xWYOnFcspXJ38= -k8c.io/operating-system-manager v1.0.0/go.mod h1:8Q1xpjJomTG9X6lfx/y3+yGHCackHtqxuYEk0TIPMfA= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= k8s.io/api v0.25.0 h1:H+Q4ma2U/ww0iGB78ijZx6DRByPz6/733jIuFpX70e0= diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go index a31ec388c..cf5ae12ea 100644 --- a/pkg/admission/admission.go +++ b/pkg/admission/admission.go @@ -42,37 +42,37 @@ import ( ) type admissionData struct { - client ctrlruntimeclient.Client - workerClient ctrlruntimeclient.Client - userDataManager *userdatamanager.Manager - nodeSettings machinecontroller.NodeSettings - useOSM bool - namespace string - constraints *semver.Constraints + client ctrlruntimeclient.Client + workerClient ctrlruntimeclient.Client + userDataManager *userdatamanager.Manager + nodeSettings machinecontroller.NodeSettings + useExternalBootstrap bool + namespace string + constraints *semver.Constraints } var jsonPatch = admissionv1.PatchTypeJSONPatch type Builder struct { - ListenAddress string - Client ctrlruntimeclient.Client - WorkerClient ctrlruntimeclient.Client - UserdataManager *userdatamanager.Manager - NodeFlags *node.Flags - UseOSM bool - Namespace string - VersionConstraints *semver.Constraints + ListenAddress string + Client ctrlruntimeclient.Client + WorkerClient ctrlruntimeclient.Client + UserdataManager *userdatamanager.Manager + UseExternalBootstrap bool + NodeFlags *node.Flags + Namespace string + VersionConstraints *semver.Constraints } func (build Builder) Build() (*http.Server, error) { mux := http.NewServeMux() ad := &admissionData{ - client: build.Client, - workerClient: build.WorkerClient, - userDataManager: build.UserdataManager, - useOSM: build.UseOSM, - namespace: build.Namespace, - constraints: build.VersionConstraints, + client: build.Client, + workerClient: build.WorkerClient, + userDataManager: build.UserdataManager, + useExternalBootstrap: build.UseExternalBootstrap, + namespace: build.Namespace, + constraints: build.VersionConstraints, } if err := build.NodeFlags.UpdateNodeSettings(&ad.nodeSettings); err != nil { diff --git a/pkg/admission/machinedeployments.go b/pkg/admission/machinedeployments.go index ac3c03eb6..a20fa5f53 100644 --- a/pkg/admission/machinedeployments.go +++ b/pkg/admission/machinedeployments.go @@ -22,7 +22,6 @@ import ( "fmt" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - mdvalidation "k8c.io/operating-system-manager/pkg/admission/machinedeployment/validation" admissionv1 "k8s.io/api/admission/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -37,7 +36,7 @@ func (ad *admissionData) mutateMachineDeployments(ctx context.Context, ar admiss machineDeploymentDefaultingFunction(&machineDeployment) - if err := mutationsForMachineDeployment(&machineDeployment, ad.useOSM); err != nil { + if err := mutationsForMachineDeployment(&machineDeployment); err != nil { return nil, fmt.Errorf("mutation failed: %w", err) } @@ -45,13 +44,6 @@ func (ad *admissionData) mutateMachineDeployments(ctx context.Context, ar admiss return nil, fmt.Errorf("validation failed: %v", errs) } - // If OSM is enabled then validate machine deployment against selected OSP - if ad.useOSM { - if errs := mdvalidation.ValidateMachineDeployment(ctx, machineDeployment, ad.client, ad.namespace); len(errs) > 0 { - return nil, fmt.Errorf("validation failed: %v", errs) - } - } - // Do not validate the spec if it hasn't changed machineSpecNeedsValidation := true if ar.Operation == admissionv1.Update { diff --git a/pkg/admission/machinedeployments_validation.go b/pkg/admission/machinedeployments_validation.go index e7b13c2b5..12a65c6c9 100644 --- a/pkg/admission/machinedeployments_validation.go +++ b/pkg/admission/machinedeployments_validation.go @@ -23,7 +23,6 @@ import ( "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - osmresources "k8c.io/operating-system-manager/pkg/controllers/osc/resources" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" @@ -33,8 +32,6 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" ) -const ospNamePattern = "osp-%s" - func validateMachineDeployment(md v1alpha1.MachineDeployment) field.ErrorList { allErrs := field.ErrorList{} allErrs = append(allErrs, validateMachineDeploymentSpec(&md.Spec, field.NewPath("spec"))...) @@ -117,19 +114,12 @@ func machineDeploymentDefaultingFunction(md *v1alpha1.MachineDeployment) { v1alpha1.PopulateDefaultsMachineDeployment(md) } -func mutationsForMachineDeployment(md *v1alpha1.MachineDeployment, useOSM bool) error { +func mutationsForMachineDeployment(md *v1alpha1.MachineDeployment) error { providerConfig, err := providerconfigtypes.GetConfig(md.Spec.Template.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to read MachineDeployment.Spec.Template.Spec.ProviderSpec: %w", err) } - if useOSM { - err = ensureOSPAnnotation(md, *providerConfig) - if err != nil { - return err - } - } - // Packet has been renamed to Equinix Metal if providerConfig.CloudProvider == cloudProviderPacket { err = migrateToEquinixMetal(providerConfig) @@ -146,25 +136,3 @@ func mutationsForMachineDeployment(md *v1alpha1.MachineDeployment, useOSM bool) return nil } - -func ensureOSPAnnotation(md *v1alpha1.MachineDeployment, providerConfig providerconfigtypes.Config) error { - // Check for existing annotation if it doesn't exist or if the value is empty - // inject the appropriate annotation. - if val, ok := md.Annotations[osmresources.MachineDeploymentOSPAnnotation]; !ok || val == "" { - if md.Annotations == nil { - md.Annotations = make(map[string]string) - } - // Annotation not specified, populate default OSP annotation - switch providerConfig.OperatingSystem { - case providerconfigtypes.OperatingSystemUbuntu, providerconfigtypes.OperatingSystemCentOS, providerconfigtypes.OperatingSystemFlatcar, - providerconfigtypes.OperatingSystemAmazonLinux2, providerconfigtypes.OperatingSystemRockyLinux, providerconfigtypes.OperatingSystemSLES, - providerconfigtypes.OperatingSystemRHEL: - md.Annotations[osmresources.MachineDeploymentOSPAnnotation] = fmt.Sprintf(ospNamePattern, providerConfig.OperatingSystem) - return nil - - default: - return fmt.Errorf("failed to populate OSP annotation for machinedeployment with unsupported Operating System %s", providerConfig.OperatingSystem) - } - } - return nil -} diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index 21e4f0111..86b730a31 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -98,12 +98,15 @@ func (ad *admissionData) mutateMachines(ctx context.Context, ar admissionv1.Admi common.SetOSLabel(&machine.Spec, string(providerConfig.OperatingSystem)) } - // Set LegacyMachineControllerUserDataLabel to false if OSM was used for managing the machine configuration. - if ad.useOSM { - if machine.Labels == nil { - machine.Labels = make(map[string]string) - } + if machine.Labels == nil { + machine.Labels = make(map[string]string) + } + + // Set LegacyMachineControllerUserDataLabel to false if external bootstrapping is expected for managing the machine configuration. + if ad.useExternalBootstrap { machine.Labels[controllerutil.LegacyMachineControllerUserDataLabel] = "false" + } else { + machine.Labels[controllerutil.LegacyMachineControllerUserDataLabel] = "true" } return createAdmissionResponse(machineOriginal, &machine) @@ -169,7 +172,7 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec providerConfig.OperatingSystem, providerConfig.CloudProvider, providerConfig.OperatingSystemSpec, - ad.useOSM, + ad.useExternalBootstrap, ) if err != nil { return err diff --git a/pkg/bootstrap/doc.go b/pkg/bootstrap/doc.go new file mode 100644 index 000000000..abceb4a6f --- /dev/null +++ b/pkg/bootstrap/doc.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +package bootstrap contains the necessary type definitions to implement the external bootstrap +mechanism that machine-controller can use instead of generating instance user-data itself. + +Any external bootstrap provider needs to implement the logic as laid out in this documentation. +This package can be imported to ensure the correct values and patterns are used. + +machine-controller will expect a Secret object in the namespace defined by `CloudInitSettingsNamespace`, +using `CloudConfigSecretNamePattern` as a pattern to determine the Secret name. This secret must provide +valid user-data that will be passed to the cloud provider instance on creation. + +Example code that determines the secret name for a specific Machine: + +``` +bootstrapSecretName := fmt.Sprintf(bootstrap.CloudConfigSecretNamePattern, + referencedMachineDeployment, + machine.Namespace, + bootstrap.BootstrapCloudConfig) +``` + +*/ + +package bootstrap diff --git a/pkg/bootstrap/types.go b/pkg/bootstrap/types.go new file mode 100644 index 000000000..7e2c96339 --- /dev/null +++ b/pkg/bootstrap/types.go @@ -0,0 +1,37 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +/* +Do NOT update existing consts in this file as they are used by external bootstrap providers. Instead, +introduce new consts (e.g. `CloudConfigSecretNamePatternV2`) and ensure that machine-controller still +supports the old "interface" (the existing consts) for a few releases, in addition to any new interfaces +you are introducing. +*/ + +type CloudConfigSecret string + +const ( + BootstrapCloudConfig CloudConfigSecret = "bootstrap" + + CloudConfigSecretNamePattern = "%s-%s-%s-config" + + // CloudInitSettingsNamespace is the namespace in which bootstrap secrets are created by an external mechanism. + CloudInitSettingsNamespace = "cloud-init-settings" + // MachineDeploymentRevision is the revision for Machine Deployment. + MachineDeploymentRevision = "k8c.io/machine-deployment-revision" +) diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index eaab0e49c..5c183c6b8 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -31,6 +31,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/apis/plugin" + "github.com/kubermatic/machine-controller/pkg/bootstrap" "github.com/kubermatic/machine-controller/pkg/cloudprovider" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" @@ -47,8 +48,6 @@ import ( userdatamanager "github.com/kubermatic/machine-controller/pkg/userdata/manager" userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" "github.com/kubermatic/machine-controller/pkg/userdata/rhel" - "k8c.io/operating-system-manager/pkg/controllers/osc" - osmresources "k8c.io/operating-system-manager/pkg/controllers/osc/resources" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -119,7 +118,7 @@ type Reconciler struct { redhatSubscriptionManager rhsm.RedHatSubscriptionManager satelliteSubscriptionManager rhsm.SatelliteSubscriptionManager - useOSM bool + useExternalBootstrap bool nodePortRange string overrideBootstrapKubeletAPIServer string } @@ -177,7 +176,7 @@ func Add( bootstrapTokenServiceAccountName *types.NamespacedName, skipEvictionAfter time.Duration, nodeSettings NodeSettings, - useOSM bool, + useExternalBootstrap bool, nodePortRange string, overrideBootstrapKubeletAPIServer string, ) error { @@ -196,7 +195,7 @@ func Add( redhatSubscriptionManager: rhsm.NewRedHatSubscriptionManager(), satelliteSubscriptionManager: rhsm.NewSatelliteSubscriptionManager(), - useOSM: useOSM, + useExternalBootstrap: useExternalBootstrap, nodePortRange: nodePortRange, overrideBootstrapKubeletAPIServer: overrideBootstrapKubeletAPIServer, } @@ -749,8 +748,8 @@ func (r *Reconciler) ensureInstanceExistsForMachine( var kubeconfig *clientcmdapi.Config - // OSM will take care of the bootstrap kubeconfig and token by itself. - if !r.useOSM { + // an external provider will take care of the bootstrap kubeconfig and token by itself. + if !r.useExternalBootstrap { kubeconfig, err = r.createBootstrapKubeconfig(ctx, machine.Name) if err != nil { return nil, fmt.Errorf("failed to create bootstrap kubeconfig: %w", err) @@ -815,49 +814,28 @@ func (r *Reconciler) ensureInstanceExistsForMachine( // Here we do stuff! var userdata string - if r.useOSM { + if r.useExternalBootstrap { referencedMachineDeployment, machineDeploymentRevision, err := controllerutil.GetMachineDeploymentNameAndRevisionForMachine(ctx, machine, r.client) if err != nil { return nil, fmt.Errorf("failed to find machine's MachineDployment: %w", err) } - // We need to ensure that both provisoning and bootstrapping secrets have been created. And that the revision - // matches with the machine deployment revision - provisioningSecretName := fmt.Sprintf(osmresources.CloudConfigSecretNamePattern, + bootstrapSecretName := fmt.Sprintf(bootstrap.CloudConfigSecretNamePattern, referencedMachineDeployment, machine.Namespace, - osmresources.ProvisioningCloudConfig) - - // Ensure that the provisioning secret exists - provisioningSecret := &corev1.Secret{} - if err := r.client.Get(ctx, - types.NamespacedName{Name: provisioningSecretName, Namespace: util.CloudInitNamespace}, - provisioningSecret); err != nil { - klog.Errorf(CloudInitNotReadyError, osmresources.ProvisioningCloudConfig, machine.Name) - return nil, err - } - - provisioningSecretRevision := provisioningSecret.Annotations[osc.MachineDeploymentRevision] - if provisioningSecretRevision != machineDeploymentRevision { - return nil, fmt.Errorf(CloudInitNotReadyError, osmresources.ProvisioningCloudConfig, machine.Name) - } - - bootstrapSecretName := fmt.Sprintf(osmresources.CloudConfigSecretNamePattern, - referencedMachineDeployment, - machine.Namespace, - osmresources.BootstrapCloudConfig) + bootstrap.BootstrapCloudConfig) bootstrapSecret := &corev1.Secret{} if err := r.client.Get(ctx, types.NamespacedName{Name: bootstrapSecretName, Namespace: util.CloudInitNamespace}, bootstrapSecret); err != nil { - klog.Errorf(CloudInitNotReadyError, osmresources.BootstrapCloudConfig, machine.Name) + klog.Errorf(CloudInitNotReadyError, bootstrap.BootstrapCloudConfig, machine.Name) return nil, err } - bootstrapSecretRevision := bootstrapSecret.Annotations[osc.MachineDeploymentRevision] + bootstrapSecretRevision := bootstrapSecret.Annotations[bootstrap.MachineDeploymentRevision] if bootstrapSecretRevision != machineDeploymentRevision { - return nil, fmt.Errorf(CloudInitNotReadyError, osmresources.BootstrapCloudConfig, machine.Name) + return nil, fmt.Errorf(CloudInitNotReadyError, bootstrap.BootstrapCloudConfig, machine.Name) } userdata = getOSMBootstrapUserdata(req.MachineSpec.Name, *bootstrapSecret) diff --git a/pkg/providerconfig/types.go b/pkg/providerconfig/types.go index 0b0879a80..1988d83d5 100644 --- a/pkg/providerconfig/types.go +++ b/pkg/providerconfig/types.go @@ -184,7 +184,7 @@ func DefaultOperatingSystemSpec( osys providerconfigtypes.OperatingSystem, cloudProvider providerconfigtypes.CloudProvider, operatingSystemSpec runtime.RawExtension, - operatingSystemManagerEnabled bool, + externalBootstrapEnabled bool, ) (runtime.RawExtension, error) { switch osys { case providerconfigtypes.OperatingSystemAmazonLinux2: @@ -192,7 +192,7 @@ func DefaultOperatingSystemSpec( case providerconfigtypes.OperatingSystemCentOS: return centos.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemFlatcar: - return flatcar.DefaultConfigForCloud(operatingSystemSpec, cloudProvider, operatingSystemManagerEnabled), nil + return flatcar.DefaultConfigForCloud(operatingSystemSpec, cloudProvider, externalBootstrapEnabled), nil case providerconfigtypes.OperatingSystemRHEL: return rhel.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemSLES: diff --git a/pkg/userdata/flatcar/flatcar.go b/pkg/userdata/flatcar/flatcar.go index c76c63a35..724223581 100644 --- a/pkg/userdata/flatcar/flatcar.go +++ b/pkg/userdata/flatcar/flatcar.go @@ -49,7 +49,7 @@ func DefaultConfig(operatingSystemSpec runtime.RawExtension) runtime.RawExtensio return DefaultConfigForCloud(operatingSystemSpec, "", true) } -func DefaultConfigForCloud(operatingSystemSpec runtime.RawExtension, cloudProvider types.CloudProvider, operatingSystemManagerEnabled bool) runtime.RawExtension { +func DefaultConfigForCloud(operatingSystemSpec runtime.RawExtension, cloudProvider types.CloudProvider, externalBootstrapEnabled bool) runtime.RawExtension { // If userdata is being used from machine-controller and selected cloud provider is AWS then we // force cloud-init. Because AWS has a very low cap for the maximum size of user-data. In case of ignition, // we always exceed that limit which prevents new ec2 instances from being created. @@ -58,7 +58,7 @@ func DefaultConfigForCloud(operatingSystemSpec runtime.RawExtension, cloudProvid _ = json.Unmarshal(operatingSystemSpec.Raw, &osSpec) } // In case of OSM this is not required. - if cloudProvider == types.CloudProviderAWS && !operatingSystemManagerEnabled { + if cloudProvider == types.CloudProviderAWS && !externalBootstrapEnabled { osSpec.ProvisioningUtility = CloudInit } diff --git a/test/e2e/provisioning/testdata/machinedeployment-alibaba.yaml b/test/e2e/provisioning/testdata/machinedeployment-alibaba.yaml index 38de62113..63e9637ee 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-alibaba.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-alibaba.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml b/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml index cca5f3dce..87e539fd8 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -21,7 +23,7 @@ spec: providerSpec: value: sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" + - "<< YOUR_PUBLIC_KEY >>" cloudProvider: anexia cloudProviderSpec: token: "<< ANEXIA_TOKEN >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml index e4c0d6375..b38cc05b5 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml index ba06debe1..68ff9cd82 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -35,7 +37,7 @@ spec: diskType: "gp2" ebsVolumeEncrypted: true securityGroupIDs: - - "sg-a2c195ca" + - "sg-a2c195ca" tags: # you have to set this flag to real clusterID when running against our dev or prod # otherwise you might have issues with your nodes not joining the cluster diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml index 258099e24..29f9c769d 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -40,7 +42,7 @@ spec: maxPrice: "<< MAX_PRICE >>" persistentRequest: false securityGroupIDs: - - "sg-a2c195ca" + - "sg-a2c195ca" tags: # you have to set this flag to real clusterID when running against our dev or prod # otherwise you might have issues with your nodes not joining the cluster diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws.yaml index ef32567d0..465f10146 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -38,7 +40,7 @@ spec: ebsVolumeEncrypted: false ami: "<< AMI >>" securityGroupIDs: - - "sg-a2c195ca" + - "sg-a2c195ca" tags: # you have to set this flag to real clusterID when running against our dev or prod # otherwise you might have issues with your nodes not joining the cluster diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure-custom-image-reference.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure-custom-image-reference.yaml index 499501237..18eb88c32 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure-custom-image-reference.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure-custom-image-reference.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml index 17dd230d4..728314a79 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure.yaml index ea6a910d7..3b6ed09d4 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-baremetal-tinkerbell.yaml b/test/e2e/provisioning/testdata/machinedeployment-baremetal-tinkerbell.yaml index 3cf0016f0..0a3797b8c 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-baremetal-tinkerbell.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-baremetal-tinkerbell.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-digitalocean.yaml b/test/e2e/provisioning/testdata/machinedeployment-digitalocean.yaml index 114c5d9e1..19479c97d 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-digitalocean.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-digitalocean.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml b/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml index 52ecd2f2f..398240b8d 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-gce.yaml b/test/e2e/provisioning/testdata/machinedeployment-gce.yaml index 5fb0a6c82..6b318f8b9 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-gce.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-gce.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -36,7 +38,7 @@ spec: # Can be 'pd-standard' or 'pd-ssd' diskType: "pd-standard" labels: - "kubernetes_cluster": "gce-test-cluster" + "kubernetes_cluster": "gce-test-cluster" assignPublicIPAddress: true customImage: "<< CUSTOM-IMAGE >>" disableMachineServiceAccount: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml b/test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml index 1e9dfadc9..66a5cc2ee 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index 37c585d33..184427fa8 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: paused: false replicas: 1 @@ -48,7 +50,7 @@ spec: type: "" # Allowed values: "", "soft", "hard" key: "foo" values: - - bar + - bar operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-linode.yaml b/test/e2e/provisioning/testdata/machinedeployment-linode.yaml index 3d82ec9c6..055b94175 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-linode.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-linode.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml b/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml index 5a2bea06a..f6315d4a0 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-openstack-project-auth.yaml b/test/e2e/provisioning/testdata/machinedeployment-openstack-project-auth.yaml index a97d47e6b..9d8b2c81f 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-openstack-project-auth.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-openstack-project-auth.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-openstack-upgrade.yml b/test/e2e/provisioning/testdata/machinedeployment-openstack-upgrade.yml index af6fcb47e..f25d1ce40 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-openstack-upgrade.yml +++ b/test/e2e/provisioning/testdata/machinedeployment-openstack-upgrade.yml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-openstack.yaml b/test/e2e/provisioning/testdata/machinedeployment-openstack.yaml index e40bfbfed..672188137 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-openstack.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-openstack.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-scaleway.yaml b/test/e2e/provisioning/testdata/machinedeployment-scaleway.yaml index 800fdab40..3927e59bd 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-scaleway.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-scaleway.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml b/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml index 969dfb3d5..c696987e0 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: paused: false replicas: 1 diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml index b42016a6c..b87ea6000 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml index 0e505c1fe..e58087ecb 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml index 131e6f913..a5407d5b7 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -41,11 +43,11 @@ spec: disableAutoUpdate: true rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" network: - cidr: "192.168.44.<< IP_OCTET >>/20" - gateway: "192.168.32.1" - dns: - servers: - - "192.168.32.1" - - "8.8.8.8" + cidr: "192.168.44.<< IP_OCTET >>/20" + gateway: "192.168.32.1" + dns: + servers: + - "192.168.32.1" + - "8.8.8.8" versions: kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml index 7ddc15a6b..5f1f969d2 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: From 2ccc22a0e315de2f5e75469923d4adc50d74a121 Mon Sep 17 00:00:00 2001 From: Helene Durand <90329641+hdurand0710@users.noreply.github.com> Date: Thu, 6 Oct 2022 15:52:38 +0200 Subject: [PATCH 229/489] Kubevirt switch to instancetype (#1454) * Kubevirt switch to instancetype Signed-off-by: Helene Durand * Fix example file Signed-off-by: Helene Durand * Code review comment Signed-off-by: Helene Durand Signed-off-by: Helene Durand --- examples/kubevirt-machinedeployment.yaml | 9 + go.mod | 3 +- go.sum | 22 +- .../provider/kubevirt/provider.go | 132 +++++--- .../provider/kubevirt/provider_test.go | 319 ++++++++++++++++++ .../provider/kubevirt/testdata/affinity.yaml | 81 +++++ .../kubevirt/testdata/custom-local-disk.yaml | 73 ++++ .../provider/kubevirt/testdata/flavor.yaml | 67 ++++ .../testdata/instancetype-flavor.yaml | 71 ++++ .../instancetype-preference-custom.yaml | 71 ++++ .../instancetype-preference-standard.yaml | 71 ++++ .../kubevirt/testdata/nominal-case.yaml | 72 ++++ .../kubevirt/testdata/secondary-disks.yaml | 110 ++++++ .../testdata/topologyspreadconstraints.yaml | 78 +++++ .../provider/kubevirt/types/types.go | 14 +- 15 files changed, 1143 insertions(+), 50 deletions(-) create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/flavor.yaml create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/instancetype-flavor.yaml create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index f2eecf71d..709916a3d 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -32,6 +32,15 @@ spec: # If instead specified directly, this value should be a base64 encoded kubeconfig. value: "<< KUBECONFIG_BASE64 >>" virtualMachine: + instancetype: + name: "standard-2" + kind: "VirtualMachineInstancetype" # Allowed values: "VirtualMachineInstancetype"/"VirtualMachineClusterInstancetype" + preference: + name: "sockets-advantage" + category: "VirtualMachinePreference" # Allowed values: "VirtualMachinePreference"/"VirtualMachineClusterPreference" + # will be deprecated: in favor instancetype and preference + flavor: + name: "kubermatic-standard" template: cpus: "1" memory: "2048M" diff --git a/go.mod b/go.mod index fba2093ad..f97b78c0e 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( k8s.io/kubelet v0.24.2 k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed kubevirt.io/api v0.57.1 - kubevirt.io/containerized-data-importer-api v1.50.0 + kubevirt.io/containerized-data-importer-api v1.54.0 sigs.k8s.io/controller-runtime v0.12.1 sigs.k8s.io/yaml v1.3.0 ) @@ -127,6 +127,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/go-digest v1.0.0-rc1 // indirect + github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/peterhellberg/link v1.1.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect diff --git a/go.sum b/go.sum index 0f713e642..400c1330e 100644 --- a/go.sum +++ b/go.sum @@ -230,6 +230,11 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= +github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= +github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= +github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -441,6 +446,7 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -686,6 +692,9 @@ github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2i github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 h1:+ShYlGoPriGahTTFTjQ0RtNXW0srxDodk2STdc238Rk= +github.com/openshift/api v0.0.0-20211217221424-8779abfbd571/go.mod h1:F/eU6jgr6Q2VhMu1mSpMmygxAELd7+BUxs3NHZ25jV4= +github.com/openshift/build-machinery-go v0.0.0-20211213093930-7e33a7eb4ce3/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= @@ -938,6 +947,7 @@ go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= +golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1095,6 +1105,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1256,6 +1267,7 @@ golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -1488,6 +1500,7 @@ gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= @@ -1523,12 +1536,14 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= k8s.io/api v0.25.0 h1:H+Q4ma2U/ww0iGB78ijZx6DRByPz6/733jIuFpX70e0= k8s.io/api v0.25.0/go.mod h1:ttceV1GyV1i1rnmvzT3BST08N6nGt+dudGrquzVQWPk= k8s.io/apiextensions-apiserver v0.24.2 h1:/4NEQHKlEz1MlaK/wHT5KMKC9UKYz6NZz6JE6ov4G6k= k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZfJiGOW6mm2hLQ= +k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apimachinery v0.25.0 h1:MlP0r6+3XbkUG2itd6vp3oxbtdQLQI94fD5gCS+gnoU= @@ -1536,6 +1551,7 @@ k8s.io/apimachinery v0.25.0/go.mod h1:qMx9eAk0sZQGsXGu86fab8tZdffHbwUfsvzqKn4mfB k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI= k8s.io/client-go v0.25.0 h1:CVWIaCETLMBNiTUta3d5nzRbXvY5Hy9Dpl+VvREpu5E= k8s.io/client-go v0.25.0/go.mod h1:lxykvypVfKilxhTklov0wz1FoaUZ8X4EwbhS6rpRfN8= +k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/code-generator v0.24.2/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= k8s.io/component-base v0.24.2 h1:kwpQdoSfbcH+8MPN4tALtajLDfSfYxBDYlXobNWI6OU= @@ -1559,14 +1575,15 @@ k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHU k8s.io/kubelet v0.24.2 h1:VAvULig8RiylCtyxudgHV7nhKsLnNIrdVBCRD4bXQ3Y= k8s.io/kubelet v0.24.2/go.mod h1:Xm9DkWQjwOs+uGOUIIGIPMvvmenvj0lDVOErvIKOOt0= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= kubevirt.io/api v0.57.1 h1:z6ImWKCQL2efFYqMWmxEsDNyt8c6mbWk7oCY6ZAa06U= kubevirt.io/api v0.57.1/go.mod h1:U0CQlZR0JoJCaC+Va0wz4dMOtYDdVywJ98OT1KmOkzI= -kubevirt.io/containerized-data-importer-api v1.50.0 h1:O01F8L5K8qRLnkYICIfmAu0dU0P48jdO42uFPElht38= -kubevirt.io/containerized-data-importer-api v1.50.0/go.mod h1:yjD8pGZVMCeqcN46JPUQdZ2JwRVoRCOXrTVyNuFvrLo= +kubevirt.io/containerized-data-importer-api v1.54.0 h1:0nIFScuAQNtD2OHNM3hNyBRrZwgOKIOUlD1JIG0PWxI= +kubevirt.io/containerized-data-importer-api v1.54.0/go.mod h1:92HiQEyzPoeMiCbgfG5Qe10JQVbtWMZOXucy56dKdGg= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= @@ -1580,6 +1597,7 @@ sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index fab927642..b84a0154a 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -97,6 +97,8 @@ type Config struct { StorageClassName string PVCSize resource.Quantity FlavorName string + Instancetype *kubevirtv1.InstancetypeMatcher + Preference *kubevirtv1.PreferenceMatcher SecondaryDisks []SecondaryDisks PodAffinityPreset AffinityType PodAntiAffinityPreset AffinityType @@ -265,11 +267,19 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "storageClassName" field: %w`, err) } + // Keep Flavor during migration. config.FlavorName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Flavor.Name) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "flavor.name" field: %w`, err) } + // Instancetype and Preference + config.Instancetype = rawConfig.VirtualMachine.Instancetype + config.Preference = rawConfig.VirtualMachine.Preference + if err != nil { + return nil, nil, fmt.Errorf(`failed to parse "preference" field: %w`, err) + } + dnsPolicyString, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.DNSPolicy) if err != nil { return nil, nil, fmt.Errorf(`failed to parse "dnsPolicy" field: %w`, err) @@ -328,7 +338,7 @@ func (p *provider) parseNodeAffinityPreset(nodeAffinityPreset kubevirttypes.Node if err != nil { return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.key" field: %w`, err) } - nodeAffinity.Values = make([]string, len(nodeAffinityPreset.Values)) + nodeAffinity.Values = make([]string, 0, len(nodeAffinityPreset.Values)) for _, v := range nodeAffinityPreset.Values { valueString, err := p.configVarResolver.GetConfigVarStringValue(v) if err != nil { @@ -445,8 +455,9 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe if err != nil { return fmt.Errorf("failed to parse config: %w", err) } - // If VMIPreset is specified, skip CPU and Memory validation. - if c.FlavorName == "" { + // If instancetype is specified (or flavor until deprecation), skip CPU and Memory validation. + // Values will come from instancetype. + if c.Instancetype == nil && c.FlavorName == "" { if _, err := parseResources(c.CPUs, c.Memory); err != nil { return err } @@ -504,6 +515,15 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } +type machineDeploymentNameGetter func() (string, error) + +func machineDeploymentNameAndRevisionForMachineGetter(ctx context.Context, machine *clusterv1alpha1.Machine, c client.Client) machineDeploymentNameGetter { + mdName, _, err := controllerutil.GetMachineDeploymentNameAndRevisionForMachine(ctx, machine, c) + return func() (string, error) { + return mdName, err + } +} + func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { @@ -513,26 +533,63 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, } } + sigClient, err := client.New(c.RestConfig, client.Options{}) + if err != nil { + return nil, fmt.Errorf("failed to get kubevirt client: %w", err) + } + + userDataSecretName := fmt.Sprintf("userdata-%s-%s", machine.Name, strconv.Itoa(int(time.Now().Unix()))) + + virtualMachine, err := p.newVirtualMachine(ctx, c, pc, machine, userDataSecretName, userdata, + machineDeploymentNameAndRevisionForMachineGetter(ctx, machine, data.Client), randomMacAddressGetter, sigClient) + if err != nil { + return nil, fmt.Errorf("could not create a VirtualMachine manifest %w", err) + } + + if err := sigClient.Create(ctx, virtualMachine); err != nil { + return nil, fmt.Errorf("failed to create vmi: %w", err) + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: userDataSecretName, + Namespace: virtualMachine.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(virtualMachine, kubevirtv1.VirtualMachineGroupVersionKind)}, + }, + Data: map[string][]byte{"userdata": []byte(userdata)}, + } + if err := sigClient.Create(ctx, secret); err != nil { + return nil, fmt.Errorf("failed to create secret for userdata: %w", err) + } + return &kubeVirtServer{}, nil +} + +func (p *provider) newVirtualMachine(ctx context.Context, c *Config, pc *providerconfigtypes.Config, machine *clusterv1alpha1.Machine, + userdataSecretName, userdata string, mdNameGetter machineDeploymentNameGetter, macAddressGetter macAddressGetter, sigClient client.Client) (*kubevirtv1.VirtualMachine, error) { // We add the timestamp because the secret name must be different when we recreate the VMI // because its pod got deleted // The secret has an ownerRef on the VMI so garbace collection will take care of cleaning up. terminationGracePeriodSeconds := int64(30) - userDataSecretName := fmt.Sprintf("userdata-%s-%s", machine.Name, strconv.Itoa(int(time.Now().Unix()))) resourceRequirements := kubevirtv1.ResourceRequirements{} labels := map[string]string{"kubevirt.io/vm": machine.Name} - // Add a common label to all VirtualMachines spawned by the same MachineDeployment (= MachineDeployment name). - if mdName, _, err := controllerutil.GetMachineDeploymentNameAndRevisionForMachine(ctx, machine, data.Client); err == nil { + //Add a common label to all VirtualMachines spawned by the same MachineDeployment (= MachineDeployment name). + if mdName, err := mdNameGetter(); err == nil { labels[machineDeploymentLabelKey] = mdName } - sigClient, err := client.New(c.RestConfig, client.Options{}) - if err != nil { - return nil, fmt.Errorf("failed to get kubevirt client: %w", err) - } - - // Add VMIPreset label if specified - if c.FlavorName != "" { + // Priority to instancetype. + // if no instancetype and no flavor, resources are from config. + if c.Instancetype == nil && c.FlavorName == "" { + requestsAndLimits, err := parseResources(c.CPUs, c.Memory) + if err != nil { + return nil, err + } + resourceRequirements.Requests = *requestsAndLimits + resourceRequirements.Limits = *requestsAndLimits + } else if c.FlavorName != "" && c.Instancetype == nil { + // if flavor is specified, then take it from flavor (if instancetype is not set!). + // Add VMIPreset label if specified. vmiPreset := kubevirtv1.VirtualMachineInstancePreset{} if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: c.FlavorName}, &vmiPreset); err != nil { return nil, err @@ -540,13 +597,6 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, for key, val := range vmiPreset.Spec.Selector.MatchLabels { labels[key] = val } - } else { - requestsAndLimits, err := parseResources(c.CPUs, c.Memory) - if err != nil { - return nil, err - } - resourceRequirements.Requests = *requestsAndLimits - resourceRequirements.Limits = *requestsAndLimits } var ( @@ -562,7 +612,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, } } - defaultBridgeNetwork, err := defaultBridgeNetwork() + defaultBridgeNetwork, err := defaultBridgeNetwork(macAddressGetter) if err != nil { return nil, fmt.Errorf("could not compute a random MAC address") } @@ -574,7 +624,9 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, Labels: labels, }, Spec: kubevirtv1.VirtualMachineSpec{ - Running: utilpointer.BoolPtr(true), + Running: utilpointer.BoolPtr(true), + Instancetype: c.Instancetype, + Preference: c.Preference, Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: annotations, @@ -593,7 +645,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, }, Affinity: getAffinity(c, machineDeploymentLabelKey, labels[machineDeploymentLabelKey]), TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, - Volumes: getVMVolumes(c, dataVolumeName, userDataSecretName), + Volumes: getVMVolumes(c, dataVolumeName, userdataSecretName), DNSPolicy: c.DNSPolicy, DNSConfig: c.DNSConfig, TopologySpreadConstraints: getTopologySpreadConstraints(c, map[string]string{machineDeploymentLabelKey: labels[machineDeploymentLabelKey]}), @@ -602,23 +654,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, DataVolumeTemplates: getDataVolumeTemplates(c, dataVolumeName), }, } - - if err := sigClient.Create(ctx, virtualMachine); err != nil { - return nil, fmt.Errorf("failed to create vmi: %w", err) - } - - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: userDataSecretName, - Namespace: virtualMachine.Namespace, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(virtualMachine, kubevirtv1.VirtualMachineGroupVersionKind)}, - }, - Data: map[string][]byte{"userdata": []byte(userdata)}, - } - if err := sigClient.Create(ctx, secret); err != nil { - return nil, fmt.Errorf("failed to create secret for userdata: %w", err) - } - return &kubeVirtServer{}, nil + return virtualMachine, nil } func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { @@ -700,13 +736,23 @@ func getVMDisks(config *Config) []kubevirtv1.Disk { return disks } -func defaultBridgeNetwork() (*kubevirtv1.Interface, error) { - defaultBridgeNetwork := kubevirtv1.DefaultBridgeNetworkInterface() +type macAddressGetter func() (string, error) + +func randomMacAddressGetter() (string, error) { mac, err := netutil.GenerateRandMAC() + if err != nil { + return "", err + } + return mac.String(), nil +} + +func defaultBridgeNetwork(macAddressGetter macAddressGetter) (*kubevirtv1.Interface, error) { + defaultBridgeNetwork := kubevirtv1.DefaultBridgeNetworkInterface() + mac, err := macAddressGetter() if err != nil { return nil, err } - defaultBridgeNetwork.MacAddress = mac.String() + defaultBridgeNetwork.MacAddress = mac return defaultBridgeNetwork, nil } diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index b65178317..3af0d314c 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -17,13 +17,332 @@ limitations under the License. package kubevirt import ( + "bytes" + "context" + "embed" + "html/template" + "path" "reflect" "testing" + kubevirtv1 "kubevirt.io/api/core/v1" + + cloudprovidertesting "github.com/kubermatic/machine-controller/pkg/cloudprovider/testing" + "github.com/kubermatic/machine-controller/pkg/providerconfig" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/diff" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var ( + //go:embed testdata + vmManifestsFS embed.FS + vmDir = "testdata" + fakeclient ctrlruntimeclient.WithWatch + expectedVms map[string]*kubevirtv1.VirtualMachine + flavorName = "to-deprecate-flavor" ) +func init() { + presets := []ctrlruntimeclient.Object{getPreset("77", "77Gi", flavorName)} + fakeclient = fakectrlruntimeclient.NewClientBuilder().WithObjects(presets...).Build() + objs := runtimeFromYaml(fakeclient, vmManifestsFS, vmDir) + expectedVms = toVirtualMachines(objs) +} + +func getPreset(cpu, memory, presetName string) *kubevirtv1.VirtualMachineInstancePreset { + cpuQuantity, err := resource.ParseQuantity(cpu) + if err != nil { + return nil + } + memoryQuantity, err := resource.ParseQuantity(memory) + if err != nil { + return nil + } + resourceList := corev1.ResourceList{ + corev1.ResourceMemory: memoryQuantity, + corev1.ResourceCPU: cpuQuantity, + } + + return &kubevirtv1.VirtualMachineInstancePreset{ + TypeMeta: metav1.TypeMeta{ + Kind: kubevirtv1.VirtualMachineInstancePresetGroupVersionKind.Kind, + APIVersion: kubevirtv1.GroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: presetName, + Namespace: testNamespace, + }, + Spec: kubevirtv1.VirtualMachineInstancePresetSpec{ + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{"kubevirt.io/flavor": presetName}, + }, + Domain: &kubevirtv1.DomainSpec{ + Resources: kubevirtv1.ResourceRequirements{ + Requests: resourceList, + Limits: resourceList, + }, + }, + }, + } +} + +type kubevirtProviderSpecConf struct { + OsImageDV string // if OsImage from DV and not from http source + Instancetype *kubevirtv1.InstancetypeMatcher + Preference *kubevirtv1.PreferenceMatcher + Flavor string // to remove when Flavor is deprecated + OperatingSystem string + TopologySpreadConstraint bool + Affinity bool + SecondaryDisks bool +} + +func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { + var out bytes.Buffer + tmpl, err := template.New("test").Parse(`{ + "cloudProvider": "kubevirt", + "cloudProviderSpec": { + "auth": { + "kubeconfig": "eyJhcGlWZXJzaW9uIjoidjEiLCJjbHVzdGVycyI6W3siY2x1c3RlciI6eyJjZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YSI6IiIsInNlcnZlciI6Imh0dHBzOi8vOTUuMjE2LjIwLjE0Njo2NDQzIn0sIm5hbWUiOiJrdWJlcm5ldGVzIn1dLCJjb250ZXh0cyI6W3siY29udGV4dCI6eyJjbHVzdGVyIjoia3ViZXJuZXRlcyIsIm5hbWVzcGFjZSI6Imt1YmUtc3lzdGVtIiwidXNlciI6Imt1YmVybmV0ZXMtYWRtaW4ifSwibmFtZSI6Imt1YmVybmV0ZXMtYWRtaW5Aa3ViZXJuZXRlcyJ9XSwiY3VycmVudC1jb250ZXh0Ijoia3ViZXJuZXRlcy1hZG1pbkBrdWJlcm5ldGVzIiwia2luZCI6IkNvbmZpZyIsInByZWZlcmVuY2VzIjp7fSwidXNlcnMiOlt7Im5hbWUiOiJrdWJlcm5ldGVzLWFkbWluIiwidXNlciI6eyJjbGllbnQtY2VydGlmaWNhdGUtZGF0YSI6IiIsImNsaWVudC1rZXktZGF0YSI6IiJ9fV19" + }, + {{- if .TopologySpreadConstraint }} + "topologySpreadConstraints": [{ + "maxSkew": "2", + "topologyKey": "key1", + "whenUnsatisfiable": "DoNotSchedule"},{ + "maxSkew": "3", + "topologyKey": "key2", + "whenUnsatisfiable": "ScheduleAnyway"}], + {{- end }} + {{- if .Affinity }} + "affinity": { + "nodeAffinityPreset": { + "type": "hard", + "key": "key1", + "values": [ + "foo1", "foo2" ] + } + }, + {{- end }} + "virtualMachine": { + {{- if .Instancetype }} + "instancetype": { + "name": "{{ .Instancetype.Name }}", + "kind": "{{ .Instancetype.Kind }}" + }, + {{- end }} + {{- if .Preference }} + "preference": { + "name": "{{ .Preference.Name }}", + "kind": "{{ .Preference.Kind }}" + }, + {{- end }} + {{- if .Flavor }} + "flavor": { + "name": "{{ .Flavor }}" + }, + {{- end }} + "template": { + "cpus": "2", + "memory": "2Gi", + {{- if .SecondaryDisks }} + "secondaryDisks": [{ + "size": "20Gi", + "storageClassName": "longhorn2"},{ + "size": "30Gi", + "storageClassName": "longhorn3"}], + {{- end }} + "primaryDisk": { + {{- if .OsImageDV }} + "osImage": "{{ .OsImageDV }}", + {{- else }} + "osImage": "http://x.y.z.t/ubuntu.img", + {{- end }} + "size": "10Gi", + "storageClassName": "longhorn" + } + } + } + }, + "operatingSystem": "ubuntu", + "operatingSystemSpec": { + "disableAutoUpdate": false, + "disableLocksmithD": true, + "disableUpdateEngine": false + } +}`) + if err != nil { + t.Fatalf("Error occurred while parsing kubevirt provider spec template: %v", err) + } + err = tmpl.Execute(&out, k) + if err != nil { + t.Fatalf("Error occurred while executing kubevirt provider spec template: %v", err) + } + t.Logf("Generated providerSpec: %s", out.String()) + return out.Bytes() +} + +var ( + userdata = "fake-userdata" + testNamespace = "test-namespace" +) + +func TestNewVirtualMachine(t *testing.T) { + tests := []struct { + name string + specConf kubevirtProviderSpecConf + }{ + { + name: "nominal-case", + specConf: kubevirtProviderSpecConf{}, + }, + { + name: "instancetype-preference-standard", + specConf: kubevirtProviderSpecConf{ + Instancetype: &kubevirtv1.InstancetypeMatcher{ + Name: "standard-it", + Kind: "VirtualMachineInstancetype", + }, + Preference: &kubevirtv1.PreferenceMatcher{ + Name: "standard-pref", + Kind: "VirtualMachinePreference", + }, + }, + }, + { + name: "instancetype-preference-custom", + specConf: kubevirtProviderSpecConf{ + Instancetype: &kubevirtv1.InstancetypeMatcher{ + Name: "custom-it", + Kind: "VirtualMachineClusterInstancetype", + }, + Preference: &kubevirtv1.PreferenceMatcher{ + Name: "custom-pref", + Kind: "VirtualMachineClusterPreference", + }, + }, + }, + { + name: "flavor", // to be deprecated when UI is switched to instancetype + specConf: kubevirtProviderSpecConf{ + Flavor: flavorName, + }, + }, + { + name: "instancetype-flavor", // to be deprecated when UI is switched to instancetype, instancetype wins + // no flavor labels + specConf: kubevirtProviderSpecConf{ + Flavor: flavorName, + Instancetype: &kubevirtv1.InstancetypeMatcher{ + Name: "standard-it", + Kind: "VirtualMachineInstancetype", + }, + Preference: &kubevirtv1.PreferenceMatcher{ + Name: "standard-pref", + Kind: "VirtualMachinePreference", + }, + }, + }, + { + name: "topologyspreadconstraints", + specConf: kubevirtProviderSpecConf{TopologySpreadConstraint: true}, + }, + { + name: "affinity", + specConf: kubevirtProviderSpecConf{Affinity: true}, + }, + { + name: "secondary-disks", + specConf: kubevirtProviderSpecConf{SecondaryDisks: true}, + }, + { + name: "custom-local-disk", + specConf: kubevirtProviderSpecConf{OsImageDV: "ns/dvname"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &provider{ + // Note that configVarResolver is not used in this test as the getConfigFunc is mocked. + configVarResolver: providerconfig.NewConfigVarResolver(context.Background(), fakeclient), + } + + machine := cloudprovidertesting.Creator{ + Name: tt.name, + Namespace: "kubevirt", + ProviderSpecGetter: tt.specConf.rawProviderSpec, + }.CreateMachine(t) + + c, pc, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + t.Fatalf("provider.getConfig() error %v", err) + } + // Do not rely on POD_NAMESPACE env variable, force to known value + c.Namespace = testNamespace + + // Check the created VirtualMachine + vm, _ := p.newVirtualMachine(context.TODO(), c, pc, machine, "udsn", userdata, fakeMachineDeploymentNameAndRevisionForMachineGetter(), fixedMacAddressGetter, fakeclient) + vm.TypeMeta.APIVersion, vm.TypeMeta.Kind = kubevirtv1.VirtualMachineGroupVersionKind.ToAPIVersionAndKind() + + if !equality.Semantic.DeepEqual(vm, expectedVms[tt.name]) { + t.Errorf("Diff %v", diff.ObjectGoPrintDiff(expectedVms[tt.name], vm)) + } + }) + } +} + +func fakeMachineDeploymentNameAndRevisionForMachineGetter() machineDeploymentNameGetter { + return func() (string, error) { + return "md-name", nil + } +} + +func toVirtualMachines(objects []runtime.Object) map[string]*kubevirtv1.VirtualMachine { + vms := make(map[string]*kubevirtv1.VirtualMachine) + for _, o := range objects { + if vm, ok := o.(*kubevirtv1.VirtualMachine); ok { + vms[vm.Name] = vm + } + } + return vms +} + +func fixedMacAddressGetter() (string, error) { + return "b6:f5:b4:fe:45:1d", nil +} + +// runtimeFromYaml returns a list of Kubernetes runtime objects from their yaml templates. +// It returns the objects for all files included in the ManifestFS folder, skipping (with error log) the yaml files +// that would not contain correct yaml files. +func runtimeFromYaml(client ctrlruntimeclient.Client, fs embed.FS, dir string) []runtime.Object { + decode := serializer.NewCodecFactory(client.Scheme()).UniversalDeserializer().Decode + + files, _ := fs.ReadDir(dir) + objects := make([]runtime.Object, 0, len(files)) + + for _, f := range files { + manifest, err := fs.ReadFile(path.Join(dir, f.Name())) + if err != nil { + continue + } + obj, _, err := decode(manifest, nil, nil) + // Skip and log but continue with others + if err != nil { + continue + } + objects = append(objects, obj) + } + + return objects +} func TestTopologySpreadConstraint(t *testing.T) { tests := []struct { desc string diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml new file mode 100644 index 000000000..4c1ffe470 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml @@ -0,0 +1,81 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: affinity + md: md-name + name: affinity + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + creationTimestamp: null + name: affinity + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + labels: + kubevirt.io/vm: affinity + md: md-name + spec: + affinity: + nodeAffinity: # Section present if nodeAffinityPreset.type != "" + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: key1 + operator: In + values: + - foo1 + - foo2 + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: affinity + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml new file mode 100644 index 000000000..a84d7c91a --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml @@ -0,0 +1,73 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: custom-local-disk + md: md-name + name: custom-local-disk + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: custom-local-disk + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + pvc: + namespace: ns + name: dvname + running: true + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: custom-local-disk + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: custom-local-disk + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/flavor.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/flavor.yaml new file mode 100644 index 000000000..d9e666f2a --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/flavor.yaml @@ -0,0 +1,67 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/flavor: to-deprecate-flavor + kubevirt.io/vm: flavor + md: md-name + name: flavor + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: flavor + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/flavor: to-deprecate-flavor + kubevirt.io/vm: flavor + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: flavor + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-flavor.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-flavor.yaml new file mode 100644 index 000000000..2880ba431 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-flavor.yaml @@ -0,0 +1,71 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: instancetype-flavor + md: md-name + name: instancetype-flavor + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: instancetype-flavor + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + instancetype: + kind: VirtualMachineInstancetype + name: standard-it + preference: + kind: VirtualMachinePreference + name: standard-pref + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: instancetype-flavor + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: instancetype-flavor + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml new file mode 100644 index 000000000..8d90a8747 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml @@ -0,0 +1,71 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: instancetype-preference-custom + md: md-name + name: instancetype-preference-custom + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + creationTimestamp: null + name: instancetype-preference-custom + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + instancetype: + kind: VirtualMachineClusterInstancetype + name: custom-it + preference: + kind: VirtualMachineClusterPreference + name: custom-pref + template: + metadata: + labels: + kubevirt.io/vm: instancetype-preference-custom + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: instancetype-preference-custom + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml new file mode 100644 index 000000000..709de1199 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml @@ -0,0 +1,71 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: instancetype-preference-standard + md: md-name + name: instancetype-preference-standard + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: instancetype-preference-standard + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + instancetype: + kind: VirtualMachineInstancetype + name: standard-it + preference: + kind: VirtualMachinePreference + name: standard-pref + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: instancetype-preference-standard + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: instancetype-preference-standard + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml new file mode 100644 index 000000000..bdb107a14 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml @@ -0,0 +1,72 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: nominal-case + md: md-name + name: nominal-case + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: nominal-case + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: nominal-case + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: nominal-case + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml new file mode 100644 index 000000000..b1137331a --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml @@ -0,0 +1,110 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: secondary-disks + md: md-name + name: secondary-disks + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: secondary-disks + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + - metadata: + name: secondary-disks-secondarydisk0 + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + storageClassName: longhorn2 + source: + http: + url: http://x.y.z.t/ubuntu.img + - metadata: + name: secondary-disks-secondarydisk1 + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 30Gi + storageClassName: longhorn3 + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: secondary-disks + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + - disk: + bus: virtio + name: secondary-disks-secondarydisk0 + - disk: + bus: virtio + name: secondary-disks-secondarydisk1 + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: secondary-disks + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + - dataVolume: + name: secondary-disks-secondarydisk0 + name: secondary-disks-secondarydisk0 + - dataVolume: + name: secondary-disks-secondarydisk1 + name: secondary-disks-secondarydisk1 diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml new file mode 100644 index 000000000..4f51eeb63 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml @@ -0,0 +1,78 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: topologyspreadconstraints + md: md-name + name: topologyspreadconstraints + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: topologyspreadconstraints + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: topologyspreadconstraints + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 2 + topologykey: key1 + whenunsatisfiable: DoNotSchedule + labelselector: + matchlabels: + md: md-name + - maxskew: 3 + topologykey: key2 + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: topologyspreadconstraints + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index 7cca0a96a..365171d08 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -17,6 +17,8 @@ limitations under the License. package types import ( + kubevirtv1 "kubevirt.io/api/core/v1" + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -37,10 +39,14 @@ type Auth struct { // VirtualMachine. type VirtualMachine struct { - Flavor Flavor `json:"flavor,omitempty"` - Template Template `json:"template,omitempty"` - DNSPolicy providerconfigtypes.ConfigVarString `json:"dnsPolicy,omitempty"` - DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` + Flavor Flavor `json:"flavor,omitempty"` + // Instancetype is optional. + Instancetype *kubevirtv1.InstancetypeMatcher `json:"instancetype,omitempty"` + // Preference is optional. + Preference *kubevirtv1.PreferenceMatcher `json:"preference,omitempty"` + Template Template `json:"template,omitempty"` + DNSPolicy providerconfigtypes.ConfigVarString `json:"dnsPolicy,omitempty"` + DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` } // Flavor. From 7784255e05d856f53399d591c3387dcc651f3ae9 Mon Sep 17 00:00:00 2001 From: Helene Durand <90329641+hdurand0710@users.noreply.github.com> Date: Thu, 6 Oct 2022 19:44:28 +0200 Subject: [PATCH 230/489] Bump KubeVirt CDI to 1.55.0 (#1456) --- examples/kubevirt-machinedeployment.yaml | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index 709916a3d..a3db171e3 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -37,7 +37,7 @@ spec: kind: "VirtualMachineInstancetype" # Allowed values: "VirtualMachineInstancetype"/"VirtualMachineClusterInstancetype" preference: name: "sockets-advantage" - category: "VirtualMachinePreference" # Allowed values: "VirtualMachinePreference"/"VirtualMachineClusterPreference" + kind: "VirtualMachinePreference" # Allowed values: "VirtualMachinePreference"/"VirtualMachineClusterPreference" # will be deprecated: in favor instancetype and preference flavor: name: "kubermatic-standard" diff --git a/go.mod b/go.mod index f97b78c0e..d1474b6c8 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( k8s.io/kubelet v0.24.2 k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed kubevirt.io/api v0.57.1 - kubevirt.io/containerized-data-importer-api v1.54.0 + kubevirt.io/containerized-data-importer-api v1.55.0 sigs.k8s.io/controller-runtime v0.12.1 sigs.k8s.io/yaml v1.3.0 ) diff --git a/go.sum b/go.sum index 400c1330e..2325c16c0 100644 --- a/go.sum +++ b/go.sum @@ -1582,8 +1582,8 @@ k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/l k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= kubevirt.io/api v0.57.1 h1:z6ImWKCQL2efFYqMWmxEsDNyt8c6mbWk7oCY6ZAa06U= kubevirt.io/api v0.57.1/go.mod h1:U0CQlZR0JoJCaC+Va0wz4dMOtYDdVywJ98OT1KmOkzI= -kubevirt.io/containerized-data-importer-api v1.54.0 h1:0nIFScuAQNtD2OHNM3hNyBRrZwgOKIOUlD1JIG0PWxI= -kubevirt.io/containerized-data-importer-api v1.54.0/go.mod h1:92HiQEyzPoeMiCbgfG5Qe10JQVbtWMZOXucy56dKdGg= +kubevirt.io/containerized-data-importer-api v1.55.0 h1:IQNc8PYVq1cTwKNPEJza5xSlcnXeYVNt76M5kZ8X7xo= +kubevirt.io/containerized-data-importer-api v1.55.0/go.mod h1:92HiQEyzPoeMiCbgfG5Qe10JQVbtWMZOXucy56dKdGg= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= From 1ad56fa6478516f28e838caa799b4effea4d33f6 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Thu, 13 Oct 2022 12:57:20 +0200 Subject: [PATCH 231/489] rename master branch to main (#1460) --- .prow/postsubmits.yaml | 4 ++-- CONTRIBUTING.md | 4 ++-- docs/howto-provider.md | 8 ++++---- hack/ci/download-gocache.sh | 4 ++-- hack/ci/upload-gocache.sh | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index b2ab4d12b..35281a3a5 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -18,7 +18,7 @@ postsubmits: decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" branches: - - ^master$ + - ^main$ # Match on tags - ^v\d+\.\d+\.\d+.* labels: @@ -49,7 +49,7 @@ postsubmits: decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" branches: - - ^master$ + - ^main$ labels: preset-goproxy: "true" spec: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7a975dc2e..d52315994 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,7 +25,7 @@ This can easily be done with the `--signoff` option to `git commit`. Note that we're requiring all commits in a PR to be signed-off. If you already created a PR, you can sign-off all existing commits by rebasing with the `--signoff` flag. ``` -git rebase --signoff origin/master +git rebase --signoff origin/main ``` By doing this you state that you can certify the following (from https://developercertificate.org/): @@ -53,7 +53,7 @@ Due to their public nature, GitHub and mailing lists are not appropriate places This is a rough outline of what a contributor's workflow looks like: -- Create a topic branch from where you want to base your work (usually master). +- Create a topic branch from where you want to base your work (usually `main`). - Make commits of logical units. - Make sure your commit messages are in the proper format (see below). - Push your changes to a topic branch in your fork of the repository. diff --git a/docs/howto-provider.md b/docs/howto-provider.md index 6fc598683..f797c4260 100644 --- a/docs/howto-provider.md +++ b/docs/howto-provider.md @@ -89,7 +89,7 @@ Now the provider is ready to be added into the project for CI tests. ## References -- [Cloud Provider Interface](https://github.com/kubermatic/machine-controller/blob/master/pkg/cloudprovider/cloud/provider.go) -- [Implementation for Hetzner](https://github.com/kubermatic/machine-controller/blob/master/pkg/cloudprovider/provider/hetzner/provider.go) -- [Cloud Provider Type Definition](https://github.com/kubermatic/machine-controller/blob/master/pkg/providerconfig/types.go) -- [Registration of supported Cloud Providers](https://github.com/kubermatic/machine-controller/blob/master/pkg/cloudprovider/provider.go) +- [Cloud Provider Interface](https://github.com/kubermatic/machine-controller/blob/main/pkg/cloudprovider/cloud/provider.go) +- [Implementation for Hetzner](https://github.com/kubermatic/machine-controller/blob/main/pkg/cloudprovider/provider/hetzner/provider.go) +- [Cloud Provider Type Definition](https://github.com/kubermatic/machine-controller/blob/main/pkg/providerconfig/types.go) +- [Registration of supported Cloud Providers](https://github.com/kubermatic/machine-controller/blob/main/pkg/cloudprovider/provider.go) diff --git a/hack/ci/download-gocache.sh b/hack/ci/download-gocache.sh index 650d67032..d9a94d119 100755 --- a/hack/ci/download-gocache.sh +++ b/hack/ci/download-gocache.sh @@ -52,7 +52,7 @@ CACHE_VERSION="${PULL_BASE_SHA:-}" # Periodics just use their head ref if [[ -z "${CACHE_VERSION}" ]]; then CACHE_VERSION="$(git rev-parse HEAD)" - GIT_BRANCH="master" + GIT_BRANCH="main" fi # normalize branch name to prevent accidental directories being created @@ -63,7 +63,7 @@ URL="${GOCACHE_MINIO_ADDRESS}/machine-controller/${GIT_BRANCH}/${ARCHIVE_NAME}" # Do not go through the retry loop when there is nothing, but do try the # first few parents if no cache was found. This is helpful for retests happening -# quickly after something got merged to master and no gocache for the most +# quickly after something got merged to main and no gocache for the most # recent commit exists yet. In this case, taking the previous commit's # cache is better than nothing. This also helps for postsubmits, where the current # commit (the one that got merged) cannot have a cache yet. diff --git a/hack/ci/upload-gocache.sh b/hack/ci/upload-gocache.sh index 5db05684f..c2d0a9865 100755 --- a/hack/ci/upload-gocache.sh +++ b/hack/ci/upload-gocache.sh @@ -37,7 +37,7 @@ export GIT_HEAD_HASH="$(git rev-parse HEAD | tr -d '\n')" # PULL_BASE_REF is the name of the current branch in case of a post-submit # or the name of the base branch in case of a PR. -GIT_BRANCH="${PULL_BASE_REF:-master}" +GIT_BRANCH="${PULL_BASE_REF:-main}" # normalize branch name to prevent accidental directories being created GIT_BRANCH="$(echo "$GIT_BRANCH" | sed 's#/#-#g')" From cbb247f362dfd259427b56021e665ff98214c83b Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Mon, 17 Oct 2022 14:00:52 +0200 Subject: [PATCH 232/489] Use Calico as CNI for e2e and disable node readiness checks (#1459) * Update containerized check Signed-off-by: Marvin Beckers * Test if kindnet CNI works Signed-off-by: Marvin Beckers * Revert "Test if kindnet CNI works" This reverts commit 35ad25ccd48f09339b87219b0ea4f18427946b96. * Replace Flannel with Cilium Signed-off-by: Marvin Beckers * Set operator.replicas=1 for cilium Signed-off-by: Marvin Beckers * Remove CentOS 7 tests Signed-off-by: Marvin Beckers * Add Calico as CNI Signed-off-by: Marvin Beckers * Do not check node readiness, just verify node exists Signed-off-by: Marvin Beckers * Revert "Remove CentOS 7 tests" This reverts commit 72b0d181de6529fb05a121cb7d2a26656726345e. * Add a test for node conditions except NodeReady Signed-off-by: Marvin Beckers * Fix yamllint issues Signed-off-by: Marvin Beckers * Fix linting issue Signed-off-by: Marvin Beckers * Disable upgrade Prow job Signed-off-by: Marvin Beckers Signed-off-by: Marvin Beckers --- .prow/e2e-features.yaml | 2 +- hack/ci/calico.yaml | 4724 +++++++++++++++++++++++++++++++ hack/ci/setup-kind-cluster.sh | 8 +- hack/lib.sh | 10 +- test/e2e/provisioning/verify.go | 15 +- 5 files changed, 4746 insertions(+), 13 deletions(-) create mode 100644 hack/ci/calico.yaml diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 537e8dd3d..321847a53 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -106,7 +106,7 @@ presubmits: memory: 7Gi - name: pull-machine-controller-e2e-deployment-upgrade - always_run: true + always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: diff --git a/hack/ci/calico.yaml b/hack/ci/calico.yaml new file mode 100644 index 000000000..f64820f0f --- /dev/null +++ b/hack/ci/calico.yaml @@ -0,0 +1,4724 @@ +--- +# Source: calico/templates/calico-kube-controllers.yaml +# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers +--- +# Source: calico/templates/calico-kube-controllers.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-kube-controllers + namespace: kube-system +--- +# Source: calico/templates/calico-node.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system +--- +# Source: calico/templates/calico-config.yaml +# This ConfigMap is used to configure a self-hosted Calico installation. +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-config + namespace: kube-system +data: + # Typha is disabled. + typha_service_name: "none" + # Configure the backend to use. + calico_backend: "bird" + + # Configure the MTU to use for workload interfaces and tunnels. + # By default, MTU is auto-detected, and explicitly setting this field should not be required. + # You can override auto-detection by providing a non-zero value. + veth_mtu: "0" + + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. + cni_network_config: |- + { + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "calico", + "log_level": "info", + "log_file_path": "/var/log/calico/cni/cni.log", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, + "ipam": { + "type": "calico-ipam" + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + }, + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} + }, + { + "type": "bandwidth", + "capabilities": {"bandwidth": true} + } + ] + } +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgpconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPConfiguration + listKind: BGPConfigurationList + plural: bgpconfigurations + singular: bgpconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: BGPConfiguration contains the configuration for any BGP routing. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPConfigurationSpec contains the values of the BGP configuration. + properties: + asNumber: + description: 'ASNumber is the default AS number used by a node. [Default: + 64512]' + format: int32 + type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string + communities: + description: Communities is a list of BGP community values and their + arbitrary names for tagging routes. + items: + description: Community contains standard or large community value + and its name. + properties: + name: + description: Name given to community value. + type: string + value: + description: Value must be of format `aa:nn` or `aa:nn:mm`. + For standard community use `aa:nn` format, where `aa` and + `nn` are 16 bit number. For large community use `aa:nn:mm` + format, where `aa`, `nn` and `mm` are 32 bit number. Where, + `aa` is an AS Number, `nn` and `mm` are per-AS identifier. + pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ + type: string + type: object + type: array + listenPort: + description: ListenPort is the port where BGP protocol should listen. + Defaults to 179 + maximum: 65535 + minimum: 1 + type: integer + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: INFO]' + type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object + nodeToNodeMeshEnabled: + description: 'NodeToNodeMeshEnabled sets whether full node to node + BGP mesh is enabled. [Default: true]' + type: boolean + prefixAdvertisements: + description: PrefixAdvertisements contains per-prefix advertisement + configuration. + items: + description: PrefixAdvertisement configures advertisement properties + for the specified CIDR. + properties: + cidr: + description: CIDR for which properties should be advertised. + type: string + communities: + description: Communities can be list of either community names + already defined in `Specs.Communities` or community value + of format `aa:nn` or `aa:nn:mm`. For standard community use + `aa:nn` format, where `aa` and `nn` are 16 bit number. For + large community use `aa:nn:mm` format, where `aa`, `nn` and + `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and + `mm` are per-AS identifier. + items: + type: string + type: array + type: object + type: array + serviceClusterIPs: + description: ServiceClusterIPs are the CIDR blocks from which service + cluster IPs are allocated. If specified, Calico will advertise these + blocks, as well as any cluster IPs within them. + items: + description: ServiceClusterIPBlock represents a single allowed ClusterIP + CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceExternalIPs: + description: ServiceExternalIPs are the CIDR blocks for Kubernetes + Service External IPs. Kubernetes Service ExternalIPs will only be + advertised if they are within one of these blocks. + items: + description: ServiceExternalIPBlock represents a single allowed + External IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceLoadBalancerIPs: + description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes + Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress + IPs will only be advertised if they are within one of these blocks. + items: + description: ServiceLoadBalancerIPBlock represents a single allowed + LoadBalancer IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgppeers.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPPeer + listKind: BGPPeerList + plural: bgppeers + singular: bgppeer + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPPeerSpec contains the specification for a BGPPeer resource. + properties: + asNumber: + description: The AS Number of the peer. + format: int32 + type: integer + keepOriginalNextHop: + description: Option to keep the original nexthop field when routes + are sent to a BGP Peer. Setting "true" configures the selected BGP + Peers node to use the "next hop keep;" instead of "next hop self;"(default) + in the specific branch of the Node on "bird.cfg". + type: boolean + maxRestartTime: + description: Time to allow for software restart. When specified, + this is configured as the graceful restart timeout. When not specified, + the BIRD default of 120s is used. + type: string + node: + description: The node name identifying the Calico node instance that + is targeted by this peer. If this is not set, and no nodeSelector + is specified, then this BGP peer selects all nodes in the cluster. + type: string + nodeSelector: + description: Selector for the nodes that should have this peering. When + this is set, the Node field must be empty. + type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer + password: + description: Optional BGP password for the peerings generated by this + BGPPeer resource. + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object + peerIP: + description: The IP address of the peer followed by an optional port + number to peer with. If port number is given, format should be `[]:port` + or `:` for IPv4. If optional port number is not set, + and this peer IP and ASNumber belongs to a calico/node with ListenPort + set in BGPConfiguration, then we use that port to peer. + type: string + peerSelector: + description: Selector for the remote nodes to peer with. When this + is set, the PeerIP and ASNumber fields must be empty. For each + peering between the local node and selected remote nodes, we configure + an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, + and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The + remote AS number comes from the remote node's NodeBGPSpec.ASNumber, + or the global default if that is not set. + type: string + sourceAddress: + description: Specifies whether and how to configure a source address + for the peerings generated by this BGPPeer resource. Default value + "UseNodeIP" means to configure the node IP as the source address. "None" + means not to configure a source address. + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: blockaffinities.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BlockAffinity + listKind: BlockAffinityList + plural: blockaffinities + singular: blockaffinity + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BlockAffinitySpec contains the specification for a BlockAffinity + resource. + properties: + cidr: + type: string + deleted: + description: Deleted indicates that this block affinity is being deleted. + This field is a string for compatibility with older releases that + mistakenly treat this field as a string. + type: string + node: + type: string + state: + type: string + required: + - cidr + - deleted + - node + - state + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null + name: caliconodestatuses.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: CalicoNodeStatus + listKind: CalicoNodeStatusList + plural: caliconodestatuses + singular: caliconodestatus + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: CalicoNodeStatusSpec contains the specification for a CalicoNodeStatus + resource. + properties: + classes: + description: Classes declares the types of information to monitor + for this calico/node, and allows for selective status reporting + about certain subsets of information. + items: + type: string + type: array + node: + description: The node name identifies the Calico node instance for + node status. + type: string + updatePeriodSeconds: + description: UpdatePeriodSeconds is the period at which CalicoNodeStatus + should be updated. Set to 0 to disable CalicoNodeStatus refresh. + Maximum update period is one day. + format: int32 + type: integer + type: object + status: + description: CalicoNodeStatusStatus defines the observed state of CalicoNodeStatus. + No validation needed for status since it is updated by Calico. + properties: + agent: + description: Agent holds agent status on the node. + properties: + birdV4: + description: BIRDV4 represents the latest observed status of bird4. + properties: + lastBootTime: + description: LastBootTime holds the value of lastBootTime + from bird.ctl output. + type: string + lastReconfigurationTime: + description: LastReconfigurationTime holds the value of lastReconfigTime + from bird.ctl output. + type: string + routerID: + description: Router ID used by bird. + type: string + state: + description: The state of the BGP Daemon. + type: string + version: + description: Version of the BGP daemon + type: string + type: object + birdV6: + description: BIRDV6 represents the latest observed status of bird6. + properties: + lastBootTime: + description: LastBootTime holds the value of lastBootTime + from bird.ctl output. + type: string + lastReconfigurationTime: + description: LastReconfigurationTime holds the value of lastReconfigTime + from bird.ctl output. + type: string + routerID: + description: Router ID used by bird. + type: string + state: + description: The state of the BGP Daemon. + type: string + version: + description: Version of the BGP daemon + type: string + type: object + type: object + bgp: + description: BGP holds node BGP status. + properties: + numberEstablishedV4: + description: The total number of IPv4 established bgp sessions. + type: integer + numberEstablishedV6: + description: The total number of IPv6 established bgp sessions. + type: integer + numberNotEstablishedV4: + description: The total number of IPv4 non-established bgp sessions. + type: integer + numberNotEstablishedV6: + description: The total number of IPv6 non-established bgp sessions. + type: integer + peersV4: + description: PeersV4 represents IPv4 BGP peers status on the node. + items: + description: CalicoNodePeer contains the status of BGP peers + on the node. + properties: + peerIP: + description: IP address of the peer whose condition we are + reporting. + type: string + since: + description: Since the state or reason last changed. + type: string + state: + description: State is the BGP session state. + type: string + type: + description: Type indicates whether this peer is configured + via the node-to-node mesh, or via en explicit global or + per-node BGPPeer object. + type: string + type: object + type: array + peersV6: + description: PeersV6 represents IPv6 BGP peers status on the node. + items: + description: CalicoNodePeer contains the status of BGP peers + on the node. + properties: + peerIP: + description: IP address of the peer whose condition we are + reporting. + type: string + since: + description: Since the state or reason last changed. + type: string + state: + description: State is the BGP session state. + type: string + type: + description: Type indicates whether this peer is configured + via the node-to-node mesh, or via en explicit global or + per-node BGPPeer object. + type: string + type: object + type: array + required: + - numberEstablishedV4 + - numberEstablishedV6 + - numberNotEstablishedV4 + - numberNotEstablishedV6 + type: object + lastUpdated: + description: LastUpdated is a timestamp representing the server time + when CalicoNodeStatus object last updated. It is represented in + RFC3339 form and is in UTC. + format: date-time + nullable: true + type: string + routes: + description: Routes reports routes known to the Calico BGP daemon + on the node. + properties: + routesV4: + description: RoutesV4 represents IPv4 routes on the node. + items: + description: CalicoNodeRoute contains the status of BGP routes + on the node. + properties: + destination: + description: Destination of the route. + type: string + gateway: + description: Gateway for the destination. + type: string + interface: + description: Interface for the destination + type: string + learnedFrom: + description: LearnedFrom contains information regarding + where this route originated. + properties: + peerIP: + description: If sourceType is NodeMesh or BGPPeer, IP + address of the router that sent us this route. + type: string + sourceType: + description: Type of the source where a route is learned + from. + type: string + type: object + type: + description: Type indicates if the route is being used for + forwarding or not. + type: string + type: object + type: array + routesV6: + description: RoutesV6 represents IPv6 routes on the node. + items: + description: CalicoNodeRoute contains the status of BGP routes + on the node. + properties: + destination: + description: Destination of the route. + type: string + gateway: + description: Gateway for the destination. + type: string + interface: + description: Interface for the destination + type: string + learnedFrom: + description: LearnedFrom contains information regarding + where this route originated. + properties: + peerIP: + description: If sourceType is NodeMesh or BGPPeer, IP + address of the router that sent us this route. + type: string + sourceType: + description: Type of the source where a route is learned + from. + type: string + type: object + type: + description: Type indicates if the route is being used for + forwarding or not. + type: string + type: object + type: array + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterinformations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: ClusterInformation + listKind: ClusterInformationList + plural: clusterinformations + singular: clusterinformation + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: ClusterInformation contains the cluster specific information. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ClusterInformationSpec contains the values of describing + the cluster. + properties: + calicoVersion: + description: CalicoVersion is the version of Calico that the cluster + is running + type: string + clusterGUID: + description: ClusterGUID is the GUID of the cluster + type: string + clusterType: + description: ClusterType describes the type of the cluster + type: string + datastoreReady: + description: DatastoreReady is used during significant datastore migrations + to signal to components such as Felix that it should wait before + accessing the datastore. + type: boolean + variant: + description: Variant declares which variant of Calico should be active. + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: felixconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: FelixConfiguration + listKind: FelixConfigurationList + plural: felixconfigurations + singular: felixconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: Felix Configuration contains the configuration for Felix. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FelixConfigurationSpec contains the values of the Felix configuration. + properties: + allowIPIPPacketsFromWorkloads: + description: 'AllowIPIPPacketsFromWorkloads controls whether Felix + will add a rule to drop IPIP encapsulated traffic from workloads + [Default: false]' + type: boolean + allowVXLANPacketsFromWorkloads: + description: 'AllowVXLANPacketsFromWorkloads controls whether Felix + will add a rule to drop VXLAN encapsulated traffic from workloads + [Default: false]' + type: boolean + awsSrcDstCheck: + description: 'Set source-destination-check on AWS EC2 instances. Accepted + value must be one of "DoNothing", "Enable" or "Disable". [Default: + DoNothing]' + enum: + - DoNothing + - Enable + - Disable + type: string + bpfConnectTimeLoadBalancingEnabled: + description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. The + connect-time load balancer is required for the host to be able to + reach Kubernetes services and it improves the performance of pod-to-service + connections. The only reason to disable it is for debugging purposes. [Default: + true]' + type: boolean + bpfDataIfacePattern: + description: BPFDataIfacePattern is a regular expression that controls + which interfaces Felix should attach BPF programs to in order to + catch traffic to/from the network. This needs to match the interfaces + that Calico workload traffic flows over as well as any interfaces + that handle incoming traffic to nodeports and services from outside + the cluster. It should not match the workload interfaces (usually + named cali...). + type: string + bpfDisableUnprivileged: + description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled + sysctl to disable unprivileged use of BPF. This ensures that unprivileged + users cannot access Calico''s BPF maps and cannot insert their own + BPF programs to interfere with Calico''s. [Default: true]' + type: boolean + bpfEnabled: + description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. + [Default: false]' + type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string + bpfExtToServiceConnmark: + description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + mark that is set on connections from an external client to a local + service. This mark allows us to control how packets of that connection + are routed within the host and how is routing interpreted by RPF + check. [Default: 0]' + type: integer + bpfExternalServiceMode: + description: 'BPFExternalServiceMode in BPF mode, controls how connections + from outside the cluster to services (node ports and cluster IPs) + are forwarded to remote workloads. If set to "Tunnel" then both + request and response traffic is tunneled to the remote node. If + set to "DSR", the request traffic is tunneled but the response traffic + is sent directly from the remote node. In "DSR" mode, the remote + node appears to use the IP of the ingress node; this requires a + permissive L2 network. [Default: Tunnel]' + type: string + bpfKubeProxyEndpointSlicesEnabled: + description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls + whether Felix's embedded kube-proxy accepts EndpointSlices or not. + type: boolean + bpfKubeProxyIptablesCleanupEnabled: + description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF + mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s + iptables chains. Should only be enabled if kube-proxy is not running. [Default: + true]' + type: boolean + bpfKubeProxyMinSyncPeriod: + description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the + minimum time between updates to the dataplane for Felix''s embedded + kube-proxy. Lower values give reduced set-up latency. Higher values + reduce Felix CPU usage by batching up more work. [Default: 1s]' + type: string + bpfLogLevel: + description: 'BPFLogLevel controls the log level of the BPF programs + when in BPF dataplane mode. One of "Off", "Info", or "Debug". The + logs are emitted to the BPF trace pipe, accessible with the command + `tc exec bpf debug`. [Default: Off].' + type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeIfState: + description: BPFMapSizeIfState sets the size for ifstate map. The + ifstate map must be large enough to hold an entry for each device + (host + workloads) on a host. + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true + bpfPolicyDebugEnabled: + description: BPFPolicyDebugEnabled when true, Felix records detailed + information about the BPF policy programs, which can be examined + with the calico-bpf command-line tool. + type: boolean + chainInsertMode: + description: 'ChainInsertMode controls whether Felix hooks the kernel''s + top-level iptables chains by inserting a rule at the top of the + chain or by appending a rule at the bottom. insert is the safe default + since it prevents Calico''s rules from being bypassed. If you switch + to append mode, be sure that the other rules in the chains signal + acceptance by falling through to the Calico rules, otherwise the + Calico policy will be bypassed. [Default: insert]' + type: string + dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' + type: string + debugDisableLogDropping: + type: boolean + debugMemoryProfilePath: + type: string + debugSimulateCalcGraphHangAfter: + type: string + debugSimulateDataplaneHangAfter: + type: string + defaultEndpointToHostAction: + description: 'DefaultEndpointToHostAction controls what happens to + traffic that goes from a workload endpoint to the host itself (after + the traffic hits the endpoint egress policy). By default Calico + blocks traffic from workload endpoints to the host itself with an + iptables "DROP" action. If you want to allow some or all traffic + from endpoint to host, set this parameter to RETURN or ACCEPT. Use + RETURN if you have your own rules in the iptables "INPUT" chain; + Calico will insert its rules at the top of that chain, then "RETURN" + packets to the "INPUT" chain once it has completed processing workload + endpoint egress policy. Use ACCEPT to unconditionally accept packets + from workloads after processing workload endpoint egress policy. + [Default: Drop]' + type: string + deviceRouteProtocol: + description: This defines the route protocol added to programmed device + routes, by default this will be RTPROT_BOOT when left blank. + type: integer + deviceRouteSourceAddress: + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + disableConntrackInvalidCheck: + type: boolean + endpointReportingDelay: + type: string + endpointReportingEnabled: + type: boolean + externalNodesList: + description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes + which may source tunnel traffic and have the tunneled traffic be + accepted at calico nodes. + items: + type: string + type: array + failsafeInboundHostPorts: + description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow incoming traffic to host endpoints + on irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all inbound host ports, use the value + none. The default value allows ssh access and DHCP. [Default: tcp:22, + udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' + items: + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + type: array + failsafeOutboundHostPorts: + description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow outgoing traffic from host endpoints + to irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all outbound host ports, use the value + none. The default value opens etcd''s standard ports to ensure that + Felix does not get cut off from etcd as well as allowing DHCP and + DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, + tcp:6667, udp:53, udp:67]' + items: + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + type: array + featureDetectOverride: + description: FeatureDetectOverride is used to override the feature + detection. Values are specified in a comma separated list with no + spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + "true" or "false" will force the feature, empty or omitted values + are auto-detected. + type: string + floatingIPs: + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string + genericXDPEnabled: + description: 'GenericXDPEnabled enables Generic XDP so network cards + that don''t support XDP offload or driver modes can use XDP. This + is not recommended since it doesn''t provide better performance + than iptables. [Default: false]' + type: boolean + healthEnabled: + type: boolean + healthHost: + type: string + healthPort: + type: integer + interfaceExclude: + description: 'InterfaceExclude is a comma-separated list of interfaces + that Felix should exclude when monitoring for host endpoints. The + default value ensures that Felix ignores Kubernetes'' IPVS dummy + interface, which is used internally by kube-proxy. If you want to + exclude multiple interface names using a single value, the list + supports regular expressions. For regular expressions you must wrap + the value with ''/''. For example having values ''/^kube/,veth1'' + will exclude all interfaces that begin with ''kube'' and also the + interface ''veth1''. [Default: kube-ipvs0]' + type: string + interfacePrefix: + description: 'InterfacePrefix is the interface name prefix that identifies + workload endpoints and so distinguishes them from host endpoint + interfaces. Note: in environments other than bare metal, the orchestrators + configure this appropriately. For example our Kubernetes and Docker + integrations set the ''cali'' value, and our OpenStack integration + sets the ''tap'' value. [Default: cali]' + type: string + interfaceRefreshInterval: + description: InterfaceRefreshInterval is the period at which Felix + rescans local interfaces to verify their state. The rescan can be + disabled by setting the interval to 0. + type: string + ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' + type: boolean + ipipMTU: + description: 'IPIPMTU is the MTU to set on the tunnel device. See + Configuring MTU [Default: 1440]' + type: integer + ipsetsRefreshInterval: + description: 'IpsetsRefreshInterval is the period at which Felix re-checks + all iptables state to ensure that no other process has accidentally + broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: + 90s]' + type: string + iptablesBackend: + description: IptablesBackend specifies which backend of iptables will + be used. The default is legacy. + type: string + iptablesFilterAllowAction: + type: string + iptablesLockFilePath: + description: 'IptablesLockFilePath is the location of the iptables + lock file. You may need to change this if the lock file is not in + its standard location (for example if you have mapped it into Felix''s + container at a different path). [Default: /run/xtables.lock]' + type: string + iptablesLockProbeInterval: + description: 'IptablesLockProbeInterval is the time that Felix will + wait between attempts to acquire the iptables lock if it is not + available. Lower values make Felix more responsive when the lock + is contended, but use more CPU. [Default: 50ms]' + type: string + iptablesLockTimeout: + description: 'IptablesLockTimeout is the time that Felix will wait + for the iptables lock, or 0, to disable. To use this feature, Felix + must share the iptables lock file with all other processes that + also take the lock. When running Felix inside a container, this + requires the /run directory of the host to be mounted into the calico/node + or calico/felix container. [Default: 0s disabled]' + type: string + iptablesMangleAllowAction: + type: string + iptablesMarkMask: + description: 'IptablesMarkMask is the mask that Felix selects its + IPTables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xff000000]' + format: int32 + type: integer + iptablesNATOutgoingInterfaceFilter: + type: string + iptablesPostWriteCheckInterval: + description: 'IptablesPostWriteCheckInterval is the period after Felix + has done a write to the dataplane that it schedules an extra read + back in order to check the write was not clobbered by another process. + This should only occur if another application on the system doesn''t + respect the iptables lock. [Default: 1s]' + type: string + iptablesRefreshInterval: + description: 'IptablesRefreshInterval is the period at which Felix + re-checks the IP sets in the dataplane to ensure that no other process + has accidentally broken Calico''s rules. Set to 0 to disable IP + sets refresh. Note: the default for this value is lower than the + other refresh intervals as a workaround for a Linux kernel bug that + was fixed in kernel version 4.11. If you are using v4.11 or greater + you may want to set this to, a higher value to reduce Felix CPU + usage. [Default: 10s]' + type: string + ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). + type: boolean + kubeNodePortRanges: + description: 'KubeNodePortRanges holds list of port ranges used for + service node ports. Only used if felix detects kube-proxy running + in ipvs mode. Felix uses these ranges to separate host and workload + traffic. [Default: 30000:32767].' + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string + logFilePath: + description: 'LogFilePath is the full path to the Felix log. Set to + none to disable file logging. [Default: /var/log/calico/felix.log]' + type: string + logPrefix: + description: 'LogPrefix is the log prefix that Felix uses when rendering + LOG rules. [Default: calico-packet]' + type: string + logSeverityFile: + description: 'LogSeverityFile is the log severity above which logs + are sent to the log file. [Default: Info]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + type: string + logSeveritySys: + description: 'LogSeveritySys is the log severity above which logs + are sent to the syslog. Set to None for no logging to syslog. [Default: + Info]' + type: string + maxIpsetSize: + type: integer + metadataAddr: + description: 'MetadataAddr is the IP address or domain name of the + server that can answer VM queries for cloud-init metadata. In OpenStack, + this corresponds to the machine running nova-api (or in Ubuntu, + nova-api-metadata). A value of none (case insensitive) means that + Felix should not set up any NAT rule for the metadata path. [Default: + 127.0.0.1]' + type: string + metadataPort: + description: 'MetadataPort is the port of the metadata server. This, + combined with global.MetadataAddr (if not ''None''), is used to + set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. + In most cases this should not need to be changed [Default: 8775].' + type: integer + mtuIfacePattern: + description: MTUIfacePattern is a regular expression that controls + which interfaces Felix should scan in order to calculate the host's + MTU. This should not match workload interfaces (usually named cali...). + type: string + natOutgoingAddress: + description: NATOutgoingAddress specifies an address to use when performing + source NAT for traffic in a natOutgoing pool that is leaving the + network. By default the address used is an address on the interface + the traffic is leaving on (ie it uses the iptables MASQUERADE target) + type: string + natPortRange: + anyOf: + - type: integer + - type: string + description: NATPortRange specifies the range of ports that is used + for port mapping when doing outgoing NAT. When unset the default + behavior of the network stack is used. + pattern: ^.* + x-kubernetes-int-or-string: true + netlinkTimeout: + type: string + openstackRegion: + description: 'OpenstackRegion is the name of the region that a particular + Felix belongs to. In a multi-region Calico/OpenStack deployment, + this must be configured somehow for each Felix (here in the datamodel, + or in felix.cfg or the environment on each compute node), and must + match the [calico] openstack_region value configured in neutron.conf + on each node. [Default: Empty]' + type: string + policySyncPathPrefix: + description: 'PolicySyncPathPrefix is used to by Felix to communicate + policy changes to external services, like Application layer policy. + [Default: Empty]' + type: string + prometheusGoMetricsEnabled: + description: 'PrometheusGoMetricsEnabled disables Go runtime metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + prometheusMetricsEnabled: + description: 'PrometheusMetricsEnabled enables the Prometheus metrics + server in Felix if set to true. [Default: false]' + type: boolean + prometheusMetricsHost: + description: 'PrometheusMetricsHost is the host that the Prometheus + metrics server should bind to. [Default: empty]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. [Default: 9091]' + type: integer + prometheusProcessMetricsEnabled: + description: 'PrometheusProcessMetricsEnabled disables process metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + prometheusWireGuardMetricsEnabled: + description: 'PrometheusWireGuardMetricsEnabled disables wireguard + metrics collection, which the Prometheus client does by default, + when set to false. This reduces the number of metrics reported, + reducing Prometheus load. [Default: true]' + type: boolean + removeExternalRoutes: + description: Whether or not to remove device routes that have not + been programmed by Felix. Disabling this will allow external applications + to also add device routes. This is enabled by default which means + we will remove externally added routes. + type: boolean + reportingInterval: + description: 'ReportingInterval is the interval at which Felix reports + its status into the datastore or 0 to disable. Must be non-zero + in OpenStack deployments. [Default: 30s]' + type: string + reportingTTL: + description: 'ReportingTTL is the time-to-live setting for process-wide + status reports. [Default: 90s]' + type: string + routeRefreshInterval: + description: 'RouteRefreshInterval is the period at which Felix re-checks + the routes in the dataplane to ensure that no other process has + accidentally broken Calico''s rules. Set to 0 to disable route refresh. + [Default: 90s]' + type: string + routeSource: + description: 'RouteSource configures where Felix gets its routing + information. - WorkloadIPs: use workload endpoints to construct + routes. - CalicoIPAM: the default - use IPAM data to construct routes.' + type: string + routeSyncDisabled: + description: RouteSyncDisabled will disable all operations performed + on the route table. Set to true to run in network-policy mode only. + type: boolean + routeTableRange: + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array + serviceLoopPrevention: + description: 'When service IP advertisement is enabled, prevent routing + loops to service IPs that are not in use, by dropping or rejecting + packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", + in which case such routing loops continue to be allowed. [Default: + Drop]' + type: string + sidecarAccelerationEnabled: + description: 'SidecarAccelerationEnabled enables experimental sidecar + acceleration [Default: false]' + type: boolean + usageReportingEnabled: + description: 'UsageReportingEnabled reports anonymous Calico version + number and cluster size to projectcalico.org. Logs warnings returned + by the usage server. For example, if a significant security vulnerability + has been discovered in the version of Calico being used. [Default: + true]' + type: boolean + usageReportingInitialDelay: + description: 'UsageReportingInitialDelay controls the minimum delay + before Felix makes a report. [Default: 300s]' + type: string + usageReportingInterval: + description: 'UsageReportingInterval controls the interval at which + Felix makes reports. [Default: 86400s]' + type: string + useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. + type: boolean + vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' + type: boolean + vxlanMTU: + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' + type: integer + vxlanPort: + type: integer + vxlanVNI: + type: integer + wireguardEnabled: + description: 'WireguardEnabled controls whether Wireguard is enabled + for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network). + [Default: false]' + type: boolean + wireguardEnabledV6: + description: 'WireguardEnabledV6 controls whether Wireguard is enabled + for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network). + [Default: false]' + type: boolean + wireguardHostEncryptionEnabled: + description: 'WireguardHostEncryptionEnabled controls whether Wireguard + host-to-host encryption is enabled. [Default: false]' + type: boolean + wireguardInterfaceName: + description: 'WireguardInterfaceName specifies the name to use for + the IPv4 Wireguard interface. [Default: wireguard.cali]' + type: string + wireguardInterfaceNameV6: + description: 'WireguardInterfaceNameV6 specifies the name to use for + the IPv6 Wireguard interface. [Default: wg-v6.cali]' + type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string + wireguardListeningPort: + description: 'WireguardListeningPort controls the listening port used + by IPv4 Wireguard. [Default: 51820]' + type: integer + wireguardListeningPortV6: + description: 'WireguardListeningPortV6 controls the listening port + used by IPv6 Wireguard. [Default: 51821]' + type: integer + wireguardMTU: + description: 'WireguardMTU controls the MTU on the IPv4 Wireguard + interface. See Configuring MTU [Default: 1440]' + type: integer + wireguardMTUV6: + description: 'WireguardMTUV6 controls the MTU on the IPv6 Wireguard + interface. See Configuring MTU [Default: 1420]' + type: integer + wireguardRoutingRulePriority: + description: 'WireguardRoutingRulePriority controls the priority value + to use for the Wireguard routing rule. [Default: 99]' + type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string + xdpEnabled: + description: 'XDPEnabled enables XDP acceleration for suitable untracked + incoming deny rules. [Default: true]' + type: boolean + xdpRefreshInterval: + description: 'XDPRefreshInterval is the period at which Felix re-checks + all XDP state to ensure that no other process has accidentally broken + Calico''s BPF maps or attached programs. Set to 0 to disable XDP + refresh. [Default: 90s]' + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: globalnetworkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkPolicy + listKind: GlobalNetworkPolicyList + plural: globalnetworkpolicies + singular: globalnetworkpolicy + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + applyOnForward: + description: ApplyOnForward indicates to apply the rules in this policy + on forward traffic. + type: boolean + doNotTrack: + description: DoNotTrack indicates whether packets matched by the rules + in this policy should go through the data plane's connection tracking, + such as Linux conntrack. If True, the rules in this policy are + applied before any data plane connection tracking, and packets allowed + by this policy are marked as not to be tracked. + type: boolean + egress: + description: The ordered set of egress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: The ordered set of ingress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + namespaceSelector: + description: NamespaceSelector is an optional field for an expression + used to select a pod based on namespaces. + type: string + order: + description: Order is an optional field that specifies the order in + which the policy is applied. Policies with higher "order" are applied + after those with lower order. If the order is omitted, it may be + considered to be "infinite" - i.e. the policy will be applied last. Policies + with identical order will be applied in alphanumerical order based + on the Policy "Name". + type: number + preDNAT: + description: PreDNAT indicates to apply the rules in this policy before + any DNAT. + type: boolean + selector: + description: "The selector is an expression used to pick pick out + the endpoints that the policy should be applied to. \n Selector + expressions follow this syntax: \n \tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present \tlabel in + { \"a\", \"b\", \"c\", ... } -> true if the value of label X is + one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", + ... } -> true if the value of label X is not one of \"a\", \"b\", + \"c\" \thas(label_name) -> True if that label is present \t! expr + -> negation of expr \texpr && expr -> Short-circuit and \texpr + || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() + or the empty selector -> matches all endpoints. \n Label names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive but they do not support escape characters. \n Examples + (with made-up labels): \n \ttype == \"webserver\" && deployment + == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != + \"dev\" \t! has(label_name)" + type: string + serviceAccountSelector: + description: ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + types: + description: "Types indicates whether this policy applies to ingress, + or to egress, or to both. When not explicitly specified (and so + the value on creation is empty or nil), Calico defaults Types according + to what Ingress and Egress rules are present in the policy. The + default is: \n - [ PolicyTypeIngress ], if there are no Egress rules + (including the case where there are also no Ingress rules) \n + - [ PolicyTypeEgress ], if there are Egress rules but no Ingress + rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are + both Ingress and Egress rules. \n When the policy is read back again, + Types will always be one of these values, never empty or nil." + items: + description: PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: globalnetworksets.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkSet + listKind: GlobalNetworkSetList + plural: globalnetworksets + singular: globalnetworkset + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs + that share labels to allow rules to refer to them via selectors. The labels + of GlobalNetworkSet are not namespaced. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: GlobalNetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: hostendpoints.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: HostEndpoint + listKind: HostEndpointList + plural: hostendpoints + singular: hostendpoint + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: HostEndpointSpec contains the specification for a HostEndpoint + resource. + properties: + expectedIPs: + description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. + If \"InterfaceName\" is not present, Calico will look for an interface + matching any of the IPs in the list and apply policy to that. Note: + \tWhen using the selector match criteria in an ingress or egress + security Policy \tor Profile, Calico converts the selector into + a set of IP addresses. For host \tendpoints, the ExpectedIPs field + is used for that purpose. (If only the interface \tname is specified, + Calico does not learn the IPs of the interface for use in match + \tcriteria.)" + items: + type: string + type: array + interfaceName: + description: "Either \"*\", or the name of a specific Linux interface + to apply policy to; or empty. \"*\" indicates that this HostEndpoint + governs all traffic to, from or through the default network namespace + of the host named by the \"Node\" field; entering and leaving that + namespace via any interface, including those from/to non-host-networked + local workloads. \n If InterfaceName is not \"*\", this HostEndpoint + only governs traffic that enters or leaves the host through the + specific interface named by InterfaceName, or - when InterfaceName + is empty - through the specific interface that has one of the IPs + in ExpectedIPs. Therefore, when InterfaceName is empty, at least + one expected IP must be specified. Only external interfaces (such + as \"eth0\") are supported here; it isn't possible for a HostEndpoint + to protect traffic through a specific local workload interface. + \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; + initially just pre-DNAT policy. Please check Calico documentation + for the latest position." + type: string + node: + description: The node name identifying the Calico node instance. + type: string + ports: + description: Ports contains the endpoint's named ports, which may + be referenced in security policy rules. + items: + properties: + name: + type: string + port: + type: integer + protocol: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + required: + - name + - port + - protocol + type: object + type: array + profiles: + description: A list of identifiers of security Profile objects that + apply to this endpoint. Each profile is applied in the order that + they appear in this list. Profile rules are applied after the selector-based + security policy. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamblocks.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMBlock + listKind: IPAMBlockList + plural: ipamblocks + singular: ipamblock + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMBlockSpec contains the specification for an IPAMBlock + resource. + properties: + affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. + type: string + allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. + items: + type: integer + # TODO: This nullable is manually added in. We should update controller-gen + # to handle []*int properly itself. + nullable: true + type: array + attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. + items: + properties: + handle_id: + type: string + secondary: + additionalProperties: + type: string + type: object + type: object + type: array + cidr: + description: The block's CIDR. + type: string + deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. + type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object + strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. + type: boolean + unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. + items: + type: integer + type: array + required: + - allocations + - attributes + - cidr + - strictAffinity + - unallocated + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamconfigs.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMConfig + listKind: IPAMConfigList + plural: ipamconfigs + singular: ipamconfig + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMConfigSpec contains the specification for an IPAMConfig + resource. + properties: + autoAllocateBlocks: + type: boolean + maxBlocksPerHost: + description: MaxBlocksPerHost, if non-zero, is the max number of blocks + that can be affine to each host. + maximum: 2147483647 + minimum: 0 + type: integer + strictAffinity: + type: boolean + required: + - autoAllocateBlocks + - strictAffinity + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamhandles.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMHandle + listKind: IPAMHandleList + plural: ipamhandles + singular: ipamhandle + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMHandleSpec contains the specification for an IPAMHandle + resource. + properties: + block: + additionalProperties: + type: integer + type: object + deleted: + type: boolean + handleID: + type: string + required: + - block + - handleID + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ippools.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPPool + listKind: IPPoolList + plural: ippools + singular: ippool + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPPoolSpec contains the specification for an IPPool resource. + properties: + allowedUses: + description: AllowedUse controls what the IP pool will be used for. If + not specified or empty, defaults to ["Tunnel", "Workload"] for back-compatibility + items: + type: string + type: array + blockSize: + description: The block size to use for IP address assignments from + this pool. Defaults to 26 for IPv4 and 122 for IPv6. + type: integer + cidr: + description: The pool CIDR. + type: string + disableBGPExport: + description: 'Disable exporting routes from this IP Pool''s CIDR over + BGP. [Default: false]' + type: boolean + disabled: + description: When disabled is true, Calico IPAM will not assign addresses + from this pool. + type: boolean + ipip: + description: 'Deprecated: this field is only used for APIv1 backwards + compatibility. Setting this field is not allowed, this field is + for internal use only.' + properties: + enabled: + description: When enabled is true, ipip tunneling will be used + to deliver packets to destinations within this pool. + type: boolean + mode: + description: The IPIP mode. This can be one of "always" or "cross-subnet". A + mode of "always" will also use IPIP tunneling for routing to + destination IP addresses within this pool. A mode of "cross-subnet" + will only use IPIP tunneling when the destination node is on + a different subnet to the originating node. The default value + (if not specified) is "always". + type: string + type: object + ipipMode: + description: Contains configuration for IPIP tunneling for this pool. + If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling + is disabled). + type: string + nat-outgoing: + description: 'Deprecated: this field is only used for APIv1 backwards + compatibility. Setting this field is not allowed, this field is + for internal use only.' + type: boolean + natOutgoing: + description: When nat-outgoing is true, packets sent from Calico networked + containers in this pool to destinations outside of this pool will + be masqueraded. + type: boolean + nodeSelector: + description: Allows IPPool to allocate for a specific node by label + selector. + type: string + vxlanMode: + description: Contains configuration for VXLAN tunneling for this pool. + If not specified, then this is defaulted to "Never" (i.e. VXLAN + tunneling is disabled). + type: string + required: + - cidr + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null + name: ipreservations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPReservation + listKind: IPReservationList + plural: ipreservations + singular: ipreservation + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPReservationSpec contains the specification for an IPReservation + resource. + properties: + reservedCIDRs: + description: ReservedCIDRs is a list of CIDRs and/or IP addresses + that Calico IPAM will exclude from new allocations. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: kubecontrollersconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: KubeControllersConfiguration + listKind: KubeControllersConfigurationList + plural: kubecontrollersconfigurations + singular: kubecontrollersconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KubeControllersConfigurationSpec contains the values of the + Kubernetes controllers configuration. + properties: + controllers: + description: Controllers enables and configures individual Kubernetes + controllers + properties: + namespace: + description: Namespace enables and configures the namespace controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + node: + description: Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: HostEndpoint controls syncing nodes to host endpoints. + Disabled by default, set to nil to disable. + properties: + autoCreate: + description: 'AutoCreate enables automatic creation of + host endpoints for every node. [Default: Disabled]' + type: string + type: object + leakGracePeriod: + description: 'LeakGracePeriod is the period used by the controller + to determine if an IP address has been leaked. Set to 0 + to disable IP garbage collection. [Default: 15m]' + type: string + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + syncLabels: + description: 'SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]' + type: string + type: object + policy: + description: Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + serviceAccount: + description: ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + workloadEndpoint: + description: WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer + etcdV3CompactionPeriod: + description: 'EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]' + type: string + healthChecks: + description: 'HealthChecks enables or disables support for health + checks [Default: Enabled]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: 9094]' + type: integer + required: + - controllers + type: object + status: + description: KubeControllersConfigurationStatus represents the status + of the configuration. It's useful for admins to be able to see the actual + config that was applied, which can be modified by environment variables + on the kube-controllers process. + properties: + environmentVars: + additionalProperties: + type: string + description: EnvironmentVars contains the environment variables on + the kube-controllers that influenced the RunningConfig. + type: object + runningConfig: + description: RunningConfig contains the effective config that is running + in the kube-controllers pod, after merging the API resource with + any environment variables. + properties: + controllers: + description: Controllers enables and configures individual Kubernetes + controllers + properties: + namespace: + description: Namespace enables and configures the namespace + controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + node: + description: Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: HostEndpoint controls syncing nodes to host + endpoints. Disabled by default, set to nil to disable. + properties: + autoCreate: + description: 'AutoCreate enables automatic creation + of host endpoints for every node. [Default: Disabled]' + type: string + type: object + leakGracePeriod: + description: 'LeakGracePeriod is the period used by the + controller to determine if an IP address has been leaked. + Set to 0 to disable IP garbage collection. [Default: + 15m]' + type: string + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + syncLabels: + description: 'SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]' + type: string + type: object + policy: + description: Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + serviceAccount: + description: ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + workloadEndpoint: + description: WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer + etcdV3CompactionPeriod: + description: 'EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]' + type: string + healthChecks: + description: 'HealthChecks enables or disables support for health + checks [Default: Enabled]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which + logs are sent to the stdout. [Default: Info]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: + 9094]' + type: integer + required: + - controllers + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: networkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: NetworkPolicy + listKind: NetworkPolicyList + plural: networkpolicies + singular: networkpolicy + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + egress: + description: The ordered set of egress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: The ordered set of ingress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + order: + description: Order is an optional field that specifies the order in + which the policy is applied. Policies with higher "order" are applied + after those with lower order. If the order is omitted, it may be + considered to be "infinite" - i.e. the policy will be applied last. Policies + with identical order will be applied in alphanumerical order based + on the Policy "Name". + type: number + selector: + description: "The selector is an expression used to pick pick out + the endpoints that the policy should be applied to. \n Selector + expressions follow this syntax: \n \tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present \tlabel in + { \"a\", \"b\", \"c\", ... } -> true if the value of label X is + one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", + ... } -> true if the value of label X is not one of \"a\", \"b\", + \"c\" \thas(label_name) -> True if that label is present \t! expr + -> negation of expr \texpr && expr -> Short-circuit and \texpr + || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() + or the empty selector -> matches all endpoints. \n Label names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive but they do not support escape characters. \n Examples + (with made-up labels): \n \ttype == \"webserver\" && deployment + == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != + \"dev\" \t! has(label_name)" + type: string + serviceAccountSelector: + description: ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + types: + description: "Types indicates whether this policy applies to ingress, + or to egress, or to both. When not explicitly specified (and so + the value on creation is empty or nil), Calico defaults Types according + to what Ingress and Egress are present in the policy. The default + is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including + the case where there are also no Ingress rules) \n - [ PolicyTypeEgress + ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, + PolicyTypeEgress ], if there are both Ingress and Egress rules. + \n When the policy is read back again, Types will always be one + of these values, never empty or nil." + items: + description: PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: networksets.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: NetworkSet + listKind: NetworkSetList + plural: networksets + singular: networkset + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/calico-kube-controllers-rbac.yaml +# Include a clusterrole for the kube-controllers component, +# and bind it to the calico-kube-controllers serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +rules: + # Nodes are watched to monitor for deletions. + - apiGroups: [""] + resources: + - nodes + verbs: + - watch + - list + - get + # Pods are watched to check for existence as part of IPAM controller. + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - watch + # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipreservations + verbs: + - list + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - watch + # Pools are watched to maintain a mapping of blocks to IP pools. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - watch + # kube-controllers manages hostendpoints. + - apiGroups: ["crd.projectcalico.org"] + resources: + - hostendpoints + verbs: + - get + - list + - create + - update + - delete + # Needs access to update clusterinformations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - clusterinformations + verbs: + - get + - list + - create + - update + - watch + # KubeControllersConfiguration is where it gets its config + - apiGroups: ["crd.projectcalico.org"] + resources: + - kubecontrollersconfigurations + verbs: + # read its own config + - get + # create a default if none exists + - create + # update status + - update + # watch for changes + - watch +--- +# Source: calico/templates/calico-node-rbac.yaml +# Include a clusterrole for the calico-node DaemonSet, +# and bind it to the calico-node serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-node +rules: + # Used for creating service account tokens to be used by the CNI plugin + - apiGroups: [""] + resources: + - serviceaccounts/token + resourceNames: + - calico-node + verbs: + - create + # The CNI plugin needs to get pods, nodes, and namespaces. + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + # EndpointSlices are used for Service-based network policy rule + # enforcement. + - apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: + - watch + - list + - apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get + # Pod CIDR auto-detection on kubeadm needs access to config maps. + - apiGroups: [""] + resources: + - configmaps + verbs: + - get + - apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update + # Watch for changes to Kubernetes NetworkPolicies. + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list + # Used by Calico for policy information. + - apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch + # The CNI plugin patches pods/status. + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + # Calico monitors various CRDs for config. + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - ipreservations + - ipamblocks + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - networksets + - clusterinformations + - hostendpoints + - blockaffinities + - caliconodestatuses + verbs: + - get + - list + - watch + # Calico must create and update some CRDs on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update + # Calico must update some CRDs. + - apiGroups: ["crd.projectcalico.org"] + resources: + - caliconodestatuses + verbs: + - update + # Calico stores some configuration information on the node. + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch + # These permissions are only required for upgrade from v2.6, and can + # be removed after upgrade or on fresh installations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update + # These permissions are required for Calico CNI to perform IPAM allocations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + # The CNI plugin and calico/node need to be able to create a default + # IPAMConfiguration + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipamconfigs + verbs: + - get + - create + # Block affinities must also be watchable by confd for route aggregation. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + verbs: + - watch + # The Calico IPAM migration needs to get daemonsets. These permissions can be + # removed if not upgrading from an installation using host-local IPAM. + - apiGroups: ["apps"] + resources: + - daemonsets + verbs: + - get +--- +# Source: calico/templates/calico-kube-controllers-rbac.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-kube-controllers +subjects: +- kind: ServiceAccount + name: calico-kube-controllers + namespace: kube-system +--- +# Source: calico/templates/calico-node-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system +--- +# Source: calico/templates/calico-node.yaml +# This manifest installs the calico-node container, as well +# as the CNI plugins and network config on +# each master and worker node in a Kubernetes cluster. +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: calico-node + namespace: kube-system + labels: + k8s-app: calico-node +spec: + selector: + matchLabels: + k8s-app: calico-node + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + k8s-app: calico-node + spec: + nodeSelector: + kubernetes.io/os: linux + hostNetwork: true + tolerations: + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + serviceAccountName: calico-node + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + priorityClassName: system-node-critical + initContainers: + # This container performs upgrade from host-local IPAM to calico-ipam. + # It can be deleted if this is a fresh installation, or if you have already + # upgraded to use calico-ipam. + - name: upgrade-ipam + image: docker.io/calico/cni:v3.24.1 + imagePullPolicy: IfNotPresent + command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + volumeMounts: + - mountPath: /var/lib/cni/networks + name: host-local-net-dir + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + securityContext: + privileged: true + # This container installs the CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: docker.io/calico/cni:v3.24.1 + imagePullPolicy: IfNotPresent + command: ["/opt/cni/bin/install"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # CNI MTU Config variable + - name: CNI_MTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + securityContext: + privileged: true + # This init container mounts the necessary filesystems needed by the BPF data plane + # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed + # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. + - name: "mount-bpffs" + image: docker.io/calico/node:v3.24.1 + imagePullPolicy: IfNotPresent + command: ["calico-node", "-init", "-best-effort"] + volumeMounts: + - mountPath: /sys/fs + name: sys-fs + # Bidirectional is required to ensure that the new mount we make at /sys/fs/bpf propagates to the host + # so that it outlives the init container. + mountPropagation: Bidirectional + - mountPath: /var/run/calico + name: var-run-calico + # Bidirectional is required to ensure that the new mount we make at /run/calico/cgroup propagates to the host + # so that it outlives the init container. + mountPropagation: Bidirectional + # Mount /proc/ from host which usually is an init program at /nodeproc. It's needed by mountns binary, + # executed by calico-node, to mount root cgroup2 fs at /run/calico/cgroup to attach CTLB programs correctly. + - mountPath: /nodeproc + name: nodeproc + readOnly: true + securityContext: + privileged: true + containers: + # Runs calico-node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: docker.io/calico/node:v3.24.1 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Choose the backend to use. + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s,bgp" + # Auto-detect the BGP IP address. + - name: IP + value: "autodetect" + # Enable IPIP + - name: CALICO_IPV4POOL_IPIP + value: "Always" + # Enable or Disable VXLAN on the default IP pool. + - name: CALICO_IPV4POOL_VXLAN + value: "Never" + # Enable or Disable VXLAN on the default IPv6 IP pool. + - name: CALICO_IPV6POOL_VXLAN + value: "Never" + # Set MTU for tunnel device used if ipip is enabled + - name: FELIX_IPINIPMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Set MTU for the VXLAN tunnel device. + - name: FELIX_VXLANMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Set MTU for the Wireguard tunnel device. + - name: FELIX_WIREGUARDMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within `--cluster-cidr`. + - name: CALICO_IPV4POOL_CIDR + value: "10.244.0.0/16" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + - name: FELIX_HEALTHENABLED + value: "true" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + lifecycle: + preStop: + exec: + command: + - /bin/calico-node + - -shutdown + livenessProbe: + exec: + command: + - /bin/calico-node + - -felix-live + - -bird-live + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + - -bird-ready + periodSeconds: 10 + timeoutSeconds: 10 + volumeMounts: + # For maintaining CNI plugin API credentials. + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + readOnly: false + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false + - name: policysync + mountPath: /var/run/nodeagent + # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the + # parent directory. + - name: bpffs + mountPath: /sys/fs/bpf + - name: cni-log-dir + mountPath: /var/log/calico/cni + readOnly: true + volumes: + # Used by calico-node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: sys-fs + hostPath: + path: /sys/fs/ + type: DirectoryOrCreate + - name: bpffs + hostPath: + path: /sys/fs/bpf + type: Directory + # mount /proc at /nodeproc to be used by mount-bpffs initContainer to mount root cgroup2 fs. + - name: nodeproc + hostPath: + path: /proc + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + # Used to access CNI logs. + - name: cni-log-dir + hostPath: + path: /var/log/calico/cni + # Mount in the directory for host-local IPAM allocations. This is + # used when upgrading from host-local to calico-ipam, and can be removed + # if not using the upgrade-ipam init container. + - name: host-local-net-dir + hostPath: + path: /var/lib/cni/networks + # Used to create per-pod Unix Domain Sockets + - name: policysync + hostPath: + type: DirectoryOrCreate + path: /var/run/nodeagent +--- +# Source: calico/templates/calico-kube-controllers.yaml +# See https://github.com/projectcalico/kube-controllers +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + # The controllers can only have a single active instance. + replicas: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + strategy: + type: Recreate + template: + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + spec: + nodeSelector: + kubernetes.io/os: linux + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + serviceAccountName: calico-kube-controllers + priorityClassName: system-cluster-critical + containers: + - name: calico-kube-controllers + image: docker.io/calico/kube-controllers:v3.24.1 + imagePullPolicy: IfNotPresent + env: + # Choose which controllers to run. + - name: ENABLED_CONTROLLERS + value: node + - name: DATASTORE_TYPE + value: kubernetes + livenessProbe: + exec: + command: + - /usr/bin/check-status + - -l + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r + periodSeconds: 10 diff --git a/hack/ci/setup-kind-cluster.sh b/hack/ci/setup-kind-cluster.sh index 063817c1d..5637126a0 100755 --- a/hack/ci/setup-kind-cluster.sh +++ b/hack/ci/setup-kind-cluster.sh @@ -140,13 +140,7 @@ echodate "Kind cluster $KIND_CLUSTER_NAME is up and running." if [ ! -f cni-plugin-deployed ]; then echodate "Installing CNI plugin." - ( - # Install CNI plugins since they are not installed by default in KIND. Also, kube-flannel doesn't install - # CNI plugins unlike other plugins so we have to do it manually. - setup_cni_in_kind=$(cat hack/ci/setup-cni-in-kind.sh) - docker exec $KIND_CLUSTER_NAME-control-plane bash -c "$setup_cni_in_kind &" - ) - kubectl create -f https://raw.githubusercontent.com/flannel-io/flannel/v0.19.2/Documentation/kube-flannel.yml + kubectl apply -f hack/ci/calico.yaml touch cni-plugin-deployed fi diff --git a/hack/lib.sh b/hack/lib.sh index 9f0789c57..e04b7a3b4 100644 --- a/hack/lib.sh +++ b/hack/lib.sh @@ -45,9 +45,14 @@ appendTrap() { fi } +is_containerized() { + # we're inside a Kubernetes pod/container or inside a container launched by containerize() + [ -n "${KUBERNETES_SERVICE_HOST:-}" ] || [ -n "${CONTAINERIZED:-}" ] +} + containerize() { local cmd="$1" - local image="${CONTAINERIZE_IMAGE:-quay.io/kubermatic/util:2.0.0}" + local image="${CONTAINERIZE_IMAGE:-quay.io/kubermatic/util:2.2.0}" local gocache="${CONTAINERIZE_GOCACHE:-/tmp/.gocache}" local gomodcache="${CONTAINERIZE_GOMODCACHE:-/tmp/.gomodcache}" local skip="${NO_CONTAINERIZE:-}" @@ -55,7 +60,7 @@ containerize() { # short-circuit containerize when in some cases it needs to be avoided [ -n "$skip" ] && return - if ! [ -f /.dockerenv ]; then + if ! is_containerized; then echodate "Running $cmd in a Docker container using $image..." mkdir -p "$gocache" mkdir -p "$gomodcache" @@ -67,6 +72,7 @@ containerize() { -w /go/src/k8c.io/kubermatic \ -e "GOCACHE=$gocache" \ -e "GOMODCACHE=$gomodcache" \ + -e "CONTAINERIZED=true" \ -u "$(id -u):$(id -g)" \ --entrypoint="$cmd" \ --rm \ diff --git a/test/e2e/provisioning/verify.go b/test/e2e/provisioning/verify.go index c6b6d22da..2dc7233db 100644 --- a/test/e2e/provisioning/verify.go +++ b/test/e2e/provisioning/verify.go @@ -209,11 +209,20 @@ func hasMachineReadyNode(machine *clusterv1alpha1.Machine, client ctrlruntimecli } for _, node := range nodes.Items { if isNodeForMachine(&node, machine) { + foundConditions := make(map[corev1.NodeConditionType]corev1.ConditionStatus) + for _, condition := range node.Status.Conditions { - if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { - return true, nil - } + foundConditions[condition.Type] = condition.Status + // TODO: re-enable this once you figure out how to properly run a CNI + // if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { + // return true, nil + // } } + + // ensure that kubelet self-reported resource health + return foundConditions[corev1.NodeMemoryPressure] == corev1.ConditionFalse && + foundConditions[corev1.NodeDiskPressure] == corev1.ConditionFalse && + foundConditions[corev1.NodePIDPressure] == corev1.ConditionFalse, nil } } return false, nil From 64bfee3ac5e14f088c0616849feb4462c5de010f Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Tue, 18 Oct 2022 09:19:27 +0200 Subject: [PATCH 233/489] Use 'main' as KKP branch instead of 'master' (#1464) Signed-off-by: Marvin Beckers Signed-off-by: Marvin Beckers --- hack/ci/setup-kind-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/ci/setup-kind-cluster.sh b/hack/ci/setup-kind-cluster.sh index 5637126a0..b30c78820 100755 --- a/hack/ci/setup-kind-cluster.sh +++ b/hack/ci/setup-kind-cluster.sh @@ -157,7 +157,7 @@ if [ -z "${DISABLE_CLUSTER_EXPOSER:-}" ]; then cd /tmp/kubermatic echodate "Cloning cluster exposer" KKP_REPO_URL="${KKP_REPO_URL:-https://github.com/kubermatic/kubermatic.git}" - KKP_REPO_TAG="${KKP_REPO_BRANCH:-master}" + KKP_REPO_TAG="${KKP_REPO_BRANCH:-main}" git clone --depth 1 --branch "${KKP_REPO_TAG}" "${KKP_REPO_URL}" . echodate "Building cluster exposer" From debfca7b52db265625167e1285c9fa7879c4f2d6 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Mon, 24 Oct 2022 10:17:50 +0200 Subject: [PATCH 234/489] Use official Nutanix SDK for Go (#1457) Signed-off-by: Marvin Beckers Signed-off-by: Marvin Beckers --- go.mod | 8 +- go.sum | 110 ++++++++++++++++++- pkg/cloudprovider/provider/nutanix/client.go | 6 +- 3 files changed, 112 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index d1474b6c8..2560f8da1 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,6 @@ require ( github.com/coreos/container-linux-config-transpiler v0.9.0 github.com/davecgh/go-spew v1.1.1 github.com/digitalocean/godo v1.81.0 - github.com/embik/nutanix-client-go v0.1.0 github.com/ghodss/yaml v1.0.0 github.com/go-test/deep v1.0.8 github.com/google/uuid v1.3.0 @@ -29,6 +28,7 @@ require ( github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 github.com/hetznercloud/hcloud-go v1.34.0 github.com/linode/linodego v1.8.0 + github.com/nutanix-cloud-native/prism-go-client v0.3.0 github.com/packethost/packngo v0.25.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pborman/uuid v1.2.1 @@ -41,7 +41,7 @@ require ( github.com/vmware/go-vcloud-director/v2 v2.15.0 github.com/vmware/govmomi v0.28.0 go.anx.io/go-anxcloud v0.4.4 - golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e + golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb gomodules.xyz/jsonpatch/v2 v2.2.0 google.golang.org/api v0.74.0 @@ -73,6 +73,8 @@ require ( github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect + github.com/PaesslerAG/gval v1.0.0 // indirect + github.com/PaesslerAG/jsonpath v0.1.1 // indirect github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect @@ -133,7 +135,7 @@ require ( github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.35.0 // indirect github.com/prometheus/procfs v0.7.3 // indirect - github.com/rogpeppe/go-internal v1.6.1 // indirect + github.com/rogpeppe/go-internal v1.8.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index 2325c16c0..34adc05c4 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/99designs/gqlgen v0.15.1 h1:48bRXecwlCNTa/n2bMSp2rQsXNxwZ54QHbiULNf78ec= +github.com/99designs/gqlgen v0.15.1/go.mod h1:nbeSjFkqphIqpZsYe1ULVz0yfH8hjpJdJIQoX/e0G2I= github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= @@ -101,12 +103,20 @@ github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXn github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PaesslerAG/gval v1.0.0 h1:GEKnRwkWDdf9dOmKcNrar9EA1bz1z9DqPIO1+iLzhd8= +github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= +github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8= +github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk= +github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/agnivade/levenshtein v1.1.0 h1:n6qGwyHG61v3ABce1rPVZklEYRT8NFpCMrpZdBUbYGM= +github.com/agnivade/levenshtein v1.1.0/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 h1:4SPQljF/GJ8Q+QlCWMWxRBepub4DresnOm4eI2ebFGc= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd h1:NlKlOv3aVJ5ODMC0JWPvddw05KENkL3cZttIuu8kJRo= @@ -121,6 +131,8 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAu github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1645 h1:IEL/Da0Dtg9j/36UnzyxD84n0eDj0JIoTKTKobN2eks= github.com/aliyun/alibaba-cloud-sdk-go v1.61.1645/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= @@ -128,6 +140,7 @@ github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195 h1:c4mLfegoDw6OhSJXTd2jUEQgZUQuJWtocudb97Qn9EM= github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI= +github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -137,6 +150,7 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6l github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.42.23/go.mod h1:gyRszuZ/icHmHAVE4gc/r+cfCmhA1AD+vqfWbgI+eHs= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.16.12 h1:wbMYa2PlFysFx2GLIQojr6FJV5+OWCM/BwyHXARxETA= github.com/aws/aws-sdk-go-v2 v1.16.12/go.mod h1:C+Ym0ag2LIghJbXhfXZ0YEEp49rBWowxKzJLUoob0ts= @@ -165,6 +179,7 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.16.15/go.mod h1:Y+BUV19q3OmQVqNUlbZ4 github.com/aws/smithy-go v1.13.0 h1:YfyEmSJLo7fAv8FbuDK4R8F9aAmi9DZ88Zb/KJJmUl0= github.com/aws/smithy-go v1.13.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= +github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -173,6 +188,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bnkamalesh/webgo/v4 v4.1.11/go.mod h1:taIAonQTzao8G5rnB22WgKmQuIOWHpQ0n/YLAidBXlM= +github.com/bnkamalesh/webgo/v6 v6.2.2/go.mod h1:2Y+dEdTp1xC/ra+3PAVZV6hh4sCI+iPK7mcHt+t9bfM= github.com/briandowns/spinner v1.8.0/go.mod h1://Zf9tMcxfRUA36V23M6YGEAv+kECGfvpnLTnb8n4XQ= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= @@ -230,6 +247,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creasty/defaults v1.5.2 h1:/VfB6uxpyp6h0fr7SPp7n8WJBoV8jfxQXPCnkVSjyls= +github.com/creasty/defaults v1.5.2/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= @@ -238,9 +257,12 @@ github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/digitalocean/godo v1.81.0 h1:sjb3fOfPfSlUQUK22E87BcI8Zx2qtnF7VUCCO4UK3C8= github.com/digitalocean/godo v1.81.0/go.mod h1:BPCqvwbjbGqxuUnIKB4EvS/AX7IDnNmt5fwvIkWo+ew= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= @@ -262,8 +284,6 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/embik/nutanix-client-go v0.1.0 h1:yPcozUczE2a12RRD/mfk8CehhKPAJWVpisPgqjILpas= -github.com/embik/nutanix-client-go v0.1.0/go.mod h1:gkKNSxfEt3QtYG3S/wKiN8OmrJ4fpU7JbTlbnrMDOL8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -303,7 +323,13 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= +github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs= +github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg= +github.com/go-chi/cors v1.2.0/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= +github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8= +github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -343,8 +369,15 @@ github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrK github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-playground/validator/v10 v10.10.1 h1:uA0+amWMiglNZKZ9FJRKUAe9U3RX91eVn1JYXMWt7ig= +github.com/go-playground/validator/v10 v10.10.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 h1:JVrqSeQfdhYRFk24TvhTZWU0q8lfCojxZQFi3Ou7+uY= github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= @@ -361,6 +394,7 @@ github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8z github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kETFqjFoFlOc= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/goccy/go-json v0.9.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godror/godror v0.13.3/go.mod h1:2ouUT4kdhUBk7TAkHWD4SN0CdI0pgEQbo8FVHhbSKWg= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= @@ -371,6 +405,7 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= @@ -412,6 +447,7 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -565,11 +601,18 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/keploy/go-sdk v0.4.3 h1:dCsmfANlZH94It+JKWx8/JEEC6dn8W7KIRRKRZwCPZQ= +github.com/keploy/go-sdk v0.4.3/go.mod h1:tn62gQ8a/AD7mY51DvQfhudiBPTlD+w3XtXemDcbON4= github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kevinmbeaulieu/eq-go v1.0.0/go.mod h1:G3S8ajA56gKBZm4UB9AOyoOS37JO3roToPzKNM8dtdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -585,13 +628,24 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/labstack/echo/v4 v4.6.1/go.mod h1:RnjgMWNDB9g/HucVWhQYNQP9PvbYf6adqftqryo7s9k= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y= +github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ= +github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++0Gf8MBnAvE= +github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= +github.com/lestrrat-go/jwx v1.2.20/go.mod h1:tLE1XszaFgd7zaS5wHe4NxA+XVhu7xgdRvDpNyi3kNM= +github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.1-0.20191011153232-f91d3411e481/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/linode/linodego v1.8.0 h1:7B2UaWu6C48tZZZrtINWRElAcwzk4TLnL9USjKf3xm0= github.com/linode/linodego v1.8.0/go.mod h1:heqhl91D8QTPVm2k9qZHP78zzbOdTFLXE9NJc3bcc50= +github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -600,13 +654,17 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matryer/moq v0.2.3/go.mod h1:9RtPYjTnH1bSBIkpvtHkFN7nbWAnO7oRpdJkEIn6UtE= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-oci8 v0.0.7/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -628,6 +686,7 @@ github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS4 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -641,6 +700,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -657,6 +717,8 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nutanix-cloud-native/prism-go-client v0.3.0 h1:4N6L8qLpEl7Y4jKmhGQNk+fMVYLc9FZCINApfuhrA+4= +github.com/nutanix-cloud-native/prism-go-client v0.3.0/go.mod h1:mwZsRrdiXVDtz8G1+Z79wbVHIJ41LB44xRN13/HlGPM= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -674,6 +736,7 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= @@ -723,6 +786,7 @@ github.com/peterhellberg/link v1.1.0 h1:s2+RH8EGuI/mI4QwrWGSYQCRz7uNgip9BaM04HKu github.com/peterhellberg/link v1.1.0/go.mod h1:gtSlOT4jmkY8P47hbTc8PTgiDDWpdPbFYl75keYyBB8= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -779,8 +843,9 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= @@ -847,7 +912,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/testcontainers/testcontainers-go v0.9.0/go.mod h1:b22BFXhRbg4PJmeMVWh6ftqjyZHgiIl3w274e9r3C2E= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -861,6 +927,14 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.35.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/vektah/gqlparser/v2 v2.2.0 h1:bAc3slekAAJW6sZTi07aGq0OrfaCjj4jxARAaC7g2EM= +github.com/vektah/gqlparser/v2 v2.2.0/go.mod h1:i3mQIGIrbK2PD1RrCeMTlVbkF2FJ6WkU1KJlJlC+3F4= github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vmware/go-vcloud-director/v2 v2.15.0 h1:idQ9NsHLr2dOSLBC8KIdBMq7XOvPiWmfxgWNaf580mk= @@ -868,11 +942,15 @@ github.com/vmware/go-vcloud-director/v2 v2.15.0/go.mod h1:2BS1yw61VN34WI0/nUYoIn github.com/vmware/govmomi v0.28.0 h1:VgeQ/Rvz79U9G8QIKLdgpsN9AndHJL+5iMJLgYIrBGI= github.com/vmware/govmomi v0.28.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -897,8 +975,12 @@ go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46O go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= +go.keploy.io/server v0.1.8 h1:b50vAt1+WKMscYVP5Bm8gx/iSaR7mpHox8VpaxjrQ88= +go.keploy.io/server v0.1.8/go.mod h1:ZqhwTZOBb+dzx5t30Wt6eUGI6kO5QizvPg6coNPtbow= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.mongodb.org/mongo-driver v1.8.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= +go.mongodb.org/mongo-driver v1.8.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -929,6 +1011,7 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -943,6 +1026,7 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= @@ -965,13 +1049,16 @@ golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1061,6 +1148,7 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1126,6 +1214,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1183,6 +1272,7 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1197,6 +1287,7 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= @@ -1215,6 +1306,7 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= @@ -1225,6 +1317,7 @@ golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1233,6 +1326,7 @@ golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= @@ -1274,6 +1368,7 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200815165600-90abf76919f3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1458,6 +1553,7 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= @@ -1485,6 +1581,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -1546,6 +1643,7 @@ k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZf k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= +k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apimachinery v0.25.0 h1:MlP0r6+3XbkUG2itd6vp3oxbtdQLQI94fD5gCS+gnoU= k8s.io/apimachinery v0.25.0/go.mod h1:qMx9eAk0sZQGsXGu86fab8tZdffHbwUfsvzqKn4mfB0= k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI= diff --git a/pkg/cloudprovider/provider/nutanix/client.go b/pkg/cloudprovider/provider/nutanix/client.go index 0ebf070e4..ca09c90ca 100644 --- a/pkg/cloudprovider/provider/nutanix/client.go +++ b/pkg/cloudprovider/provider/nutanix/client.go @@ -25,8 +25,8 @@ import ( "strings" "time" - nutanixclient "github.com/embik/nutanix-client-go/pkg/client" - nutanixv3 "github.com/embik/nutanix-client-go/pkg/client/v3" + nutanixclient "github.com/nutanix-cloud-native/prism-go-client" + nutanixv3 "github.com/nutanix-cloud-native/prism-go-client/v3" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" @@ -226,7 +226,7 @@ func createVM(ctx context.Context, client *ClientSet, name string, conf Config, func getSubnetByName(ctx context.Context, client *ClientSet, name, clusterID string) (*nutanixv3.SubnetIntentResponse, error) { filter := fmt.Sprintf("name==%s", name) - subnets, err := client.Prism.V3.ListAllSubnet(ctx, filter) + subnets, err := client.Prism.V3.ListAllSubnet(ctx, filter, nil) if err != nil { return nil, wrapNutanixError(err) From cc66623474011e702e7268759c9f584eb0a6a290 Mon Sep 17 00:00:00 2001 From: Sachin Tiptur <56350000+sachintiptur@users.noreply.github.com> Date: Tue, 25 Oct 2022 11:34:51 +0200 Subject: [PATCH 235/489] Dualstack support with AWS CCM (#1466) Signed-off-by: Sachin Tiptur Signed-off-by: Sachin Tiptur --- pkg/cloudprovider/provider/aws/types/cloudconfig.go | 4 ++++ pkg/cloudprovider/provider/aws/types/cloudconfig_test.go | 1 + .../provider/aws/types/testdata/simple-config.golden | 2 ++ 3 files changed, 7 insertions(+) diff --git a/pkg/cloudprovider/provider/aws/types/cloudconfig.go b/pkg/cloudprovider/provider/aws/types/cloudconfig.go index f70761532..2fca4788e 100644 --- a/pkg/cloudprovider/provider/aws/types/cloudconfig.go +++ b/pkg/cloudprovider/provider/aws/types/cloudconfig.go @@ -37,6 +37,9 @@ KubernetesClusterID={{ .Global.KubernetesClusterID | iniEscape }} DisableSecurityGroupIngress={{ .Global.DisableSecurityGroupIngress }} ElbSecurityGroup={{ .Global.ElbSecurityGroup | iniEscape }} DisableStrictZoneCheck={{ .Global.DisableStrictZoneCheck }} +{{- range .Global.NodeIPFamilies }} +NodeIPFamilies={{ . | iniEscape}} +{{- end }} ` ) @@ -55,6 +58,7 @@ type GlobalOpts struct { ElbSecurityGroup string DisableSecurityGroupIngress bool DisableStrictZoneCheck bool + NodeIPFamilies []string } func CloudConfigToString(c *CloudConfig) (string, error) { diff --git a/pkg/cloudprovider/provider/aws/types/cloudconfig_test.go b/pkg/cloudprovider/provider/aws/types/cloudconfig_test.go index ed6423d29..f9eaa1cfb 100644 --- a/pkg/cloudprovider/provider/aws/types/cloudconfig_test.go +++ b/pkg/cloudprovider/provider/aws/types/cloudconfig_test.go @@ -46,6 +46,7 @@ func TestCloudConfigToString(t *testing.T) { KubernetesClusterTag: "some-tag", RoleARN: "some-arn", RouteTableID: "some-rt", + NodeIPFamilies: []string{"ipv4", "ipv6"}, }, }, }, diff --git a/pkg/cloudprovider/provider/aws/types/testdata/simple-config.golden b/pkg/cloudprovider/provider/aws/types/testdata/simple-config.golden index 4642120f9..57bffe19e 100644 --- a/pkg/cloudprovider/provider/aws/types/testdata/simple-config.golden +++ b/pkg/cloudprovider/provider/aws/types/testdata/simple-config.golden @@ -8,3 +8,5 @@ KubernetesClusterID="some-tag" DisableSecurityGroupIngress=true ElbSecurityGroup="some-sg" DisableStrictZoneCheck=true +NodeIPFamilies="ipv4" +NodeIPFamilies="ipv6" From 04a5ee34bcb26079135c4e6a35f003c2bdcfdac3 Mon Sep 17 00:00:00 2001 From: Mara Sophie Grosch Date: Sat, 29 Oct 2022 16:45:08 +0200 Subject: [PATCH 236/489] Extend containerd registry mirror config (#1467) containerd can be configured for multiple registry mirrors per registry, this extends machine-controller to configure mirrors for registries other than docker.io, reusing the existing command line flag. Signed-off-by: Mara Sophie Grosch Signed-off-by: Mara Sophie Grosch --- pkg/containerruntime/config.go | 32 ++++--- pkg/containerruntime/config_test.go | 134 ++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+), 10 deletions(-) create mode 100644 pkg/containerruntime/config_test.go diff --git a/pkg/containerruntime/config.go b/pkg/containerruntime/config.go index 551befbb6..90bce0306 100644 --- a/pkg/containerruntime/config.go +++ b/pkg/containerruntime/config.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "net/url" + "regexp" "strings" corev1 "k8s.io/api/core/v1" @@ -45,11 +46,25 @@ func BuildConfig(opts Opts) (Config, error) { } } - var registryMirrors []string + // we want to match e.g. docker.io=registry.docker-cn.com, having docker.io as the first + // match group and registry.docker-cn.com as the second one. + registryMirrorRegexp := regexp.MustCompile(`^([a-zA-Z0-9\.-]+)=(.*)`) + + if opts.ContainerdRegistryMirrors == nil { + opts.ContainerdRegistryMirrors = make(RegistryMirrorsFlags) + } + for _, mirror := range strings.Split(opts.RegistryMirrors, ",") { if trimmedMirror := strings.TrimSpace(mirror); trimmedMirror != "" { - if !strings.HasPrefix(mirror, "http") { - trimmedMirror = "https://" + mirror + registry := "docker.io" + + if matches := registryMirrorRegexp.FindStringSubmatch(trimmedMirror); matches != nil { + registry = matches[1] + trimmedMirror = matches[2] + } + + if !strings.HasPrefix(trimmedMirror, "http") { + trimmedMirror = "https://" + trimmedMirror } _, err := url.Parse(trimmedMirror) @@ -57,15 +72,12 @@ func BuildConfig(opts Opts) (Config, error) { return Config{}, fmt.Errorf("incorrect mirror provided: %w", err) } - registryMirrors = append(registryMirrors, trimmedMirror) - } - } + if opts.ContainerdRegistryMirrors[registry] == nil { + opts.ContainerdRegistryMirrors[registry] = make([]string, 0, 1) + } - if len(registryMirrors) > 0 { - if opts.ContainerdRegistryMirrors == nil { - opts.ContainerdRegistryMirrors = make(RegistryMirrorsFlags) + opts.ContainerdRegistryMirrors[registry] = append(opts.ContainerdRegistryMirrors[registry], trimmedMirror) } - opts.ContainerdRegistryMirrors["docker.io"] = registryMirrors } // Only validate registry credential here diff --git a/pkg/containerruntime/config_test.go b/pkg/containerruntime/config_test.go new file mode 100644 index 000000000..4ee6ecd79 --- /dev/null +++ b/pkg/containerruntime/config_test.go @@ -0,0 +1,134 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package containerruntime + +import ( + "errors" + "fmt" + "testing" +) + +func TestContainerdRegistryMirror(t *testing.T) { + type testCase struct { + desc string + flag string + expectedMirrors map[string][]string + expectedError error + } + + testCases := []testCase{ + { + desc: "no registry mirrors set", + flag: "", + expectedMirrors: map[string][]string{}, + expectedError: nil, + }, + + { + desc: "registry mirror without name and protocol", + flag: "registry-v1.docker.io", + expectedMirrors: map[string][]string{ + "docker.io": {"https://registry-v1.docker.io/"}, + }, + expectedError: nil, + }, + { + desc: "multiple registry mirrors without name, with and without protocol", + flag: "registry-v1.docker.io,http://registry.docker-cn.com", + expectedMirrors: map[string][]string{ + "docker.io": { + "https://registry-v1.docker.io/", + "http://registry.docker-cn.com/", + }, + }, + expectedError: nil, + }, + + { + desc: "registry mirror with name and without protocol", + flag: "quay.io=my-quay-io-mirror.example.com", + expectedMirrors: map[string][]string{ + "quay.io": {"https://my-quay-io-mirror.example.com/"}, + }, + expectedError: nil, + }, + { + desc: "registry mirror with name and protocol", + flag: "quay.io=http://my-quay-io-mirror.example.com", + expectedMirrors: map[string][]string{ + "quay.io": {"http://my-quay-io-mirror.example.com/"}, + }, + expectedError: nil, + }, + { + desc: "multiple registry mirrors with same name", + flag: "quay.io=http://my-quay-io-mirror.example.com,quay.io=example.net", + expectedMirrors: map[string][]string{ + "quay.io": { + "http://my-quay-io-mirror.example.com/", + "https://example.net/", + }, + }, + expectedError: nil, + }, + + { + desc: "complex example", + flag: "quay.io=http://my-quay-io-mirror.example.com,quay.io=example.net," + + "registry-v1.docker.io,http://registry.docker-cn.com," + + "ghcr.io=http://foo/bar", + expectedMirrors: map[string][]string{ + "quay.io": { + "http://my-quay-io-mirror.example.com/", + "https://example.net/", + }, + "docker.io": { + "https://registry-v1.docker.io/", + "http://registry.docker-cn.com/", + }, + "ghcr.io": { + "http://foo/bar", + }, + }, + expectedError: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + opts := Opts{ + ContainerRuntime: containerdName, + RegistryMirrors: tc.flag, + } + + config, err := BuildConfig(opts) + if tc.expectedError != nil { + if !errors.Is(err, tc.expectedError) { + t.Errorf("expected error %q but got %q", tc.expectedError, err) + } + } + + if err != nil { + t.Errorf("expected success but got error: %q", err) + } + + if fmt.Sprint(config.RegistryMirrors) != fmt.Sprint(tc.expectedMirrors) { + t.Errorf("expected to get %v instead got: %v", tc.expectedMirrors, config.RegistryMirrors) + } + }) + } +} From c36a7d77ae7a78afcf936c4fa7b6825e3a5a9225 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 2 Nov 2022 15:24:26 +0500 Subject: [PATCH 237/489] Fix static network rendering for flatcar (#1473) Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- pkg/userdata/flatcar/provider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index 2e04d9d68..7ffc944c3 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -170,7 +170,7 @@ const userDataIgnitionTemplate = `passwd: {{end}} {{- end }} -{{- if .ProviderSpec.Network }} +{{- if .ProviderSpec.Network.IsStaticIPConfig }} networkd: units: - name: static-nic.network @@ -544,7 +544,7 @@ users: coreos: units: -{{- if .ProviderSpec.Network }} +{{- if .ProviderSpec.Network.IsStaticIPConfig }} - name: static-nic.network content: | [Match] From ff2451aec0a12e99f567376a22aca5427343c604 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 2 Nov 2022 17:30:23 +0500 Subject: [PATCH 238/489] Update supported k8s versions to 1.23,1.24, and 1.25 (#1474) Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- README.md | 2 +- examples/alibaba-machinedeployment.yaml | 2 +- examples/anexia-machinedeployment.yaml | 6 +- examples/aws-machinedeployment.yaml | 2 +- examples/azure-machinedeployment.yaml | 2 +- examples/digitalocean-machinedeployment.yaml | 2 +- examples/equinixmetal-machinedeployment.yaml | 2 +- examples/gce-machinedeployment.yaml | 2 +- examples/hetzner-machinedeployment.yaml | 2 +- examples/kubevirt-machinedeployment.yaml | 2 +- examples/linode-machinedeployment.yaml | 2 +- examples/nutanix-machinedeployment.yaml | 8 +- examples/openstack-machinedeployment.yaml | 2 +- examples/scaleway-machinedeployment.yaml | 2 +- ...ware-cloud-director-machinedeployment.yaml | 2 +- ...e-datastore-cluster-machinedeployment.yaml | 6 +- examples/vsphere-machinedeployment.yaml | 6 +- pkg/userdata/amzn2/provider_test.go | 18 +- ...-v1.22-aws.yaml => kubelet-v1.25-aws.yaml} | 51 +- pkg/userdata/centos/provider_test.go | 18 +- ...-v1.22-aws.yaml => kubelet-v1.25-aws.yaml} | 51 +- pkg/userdata/flatcar/provider_test.go | 8 +- ...t_v1.22.7.yaml => cloud-init_v1.25.0.yaml} | 46 +- ...ion_v1.22.7.json => ignition_v1.25.0.json} | 2 +- pkg/userdata/helper/common_test.go | 6 +- .../helper/download_binaries_script_test.go | 4 +- .../download_binaries_v1.23.13.golden | 17 + ...olden => download_binaries_v1.24.7.golden} | 2 +- .../testdata/download_binaries_v1.25.3.golden | 17 + ...emd_unit_version-v1.23.13-external.golden} | 2 - ...blet_systemd_unit_version-v1.23.13.golden} | 2 - ...stemd_unit_version-v1.24.7-external.golden | 36 + ...kublet_systemd_unit_version-v1.24.7.golden | 35 + ...stemd_unit_version-v1.25.3-external.golden | 36 + ...kublet_systemd_unit_version-v1.25.3.golden | 35 + ...=> safe_download_binaries_v1.23.13.golden} | 2 +- pkg/userdata/rhel/provider_test.go | 10 +- ...-v1.22-aws.yaml => kubelet-v1.25-aws.yaml} | 51 +- ...utanix.yaml => kubelet-v1.25-nutanix.yaml} | 51 +- .../rhel/testdata/pod-cidr-azure-rhel.yaml | 51 +- pkg/userdata/rockylinux/provider_test.go | 4 +- ...-v1.22-aws.yaml => kubelet-v1.25-aws.yaml} | 51 +- pkg/userdata/sles/provider_test.go | 8 +- .../sles/testdata/dist-upgrade-on-boot.yaml | 4 +- .../kubelet-version-without-v-prefix.yaml | 4 +- .../sles/testdata/multiple-dns-servers.yaml | 4 +- .../sles/testdata/multiple-ssh-keys.yaml | 4 +- .../openstack-overwrite-cloud-config.yaml | 4 +- pkg/userdata/sles/testdata/openstack.yaml | 4 +- ...rsion-1.22.7.yaml => version-1.23.13.yaml} | 4 +- .../sles/testdata/version-1.23.5.yaml | 816 ++++++++-------- .../sles/testdata/version-1.24.0.yaml | 847 ++++++++--------- .../sles/testdata/version-1.24.7.yaml | 439 +++++++++ .../sles/testdata/version-1.25.0.yaml | 415 ++++++++ .../sles/testdata/version-1.25.3.yaml | 439 +++++++++ .../sles/testdata/vsphere-mirrors.yaml | 4 +- pkg/userdata/sles/testdata/vsphere-proxy.yaml | 4 +- pkg/userdata/sles/testdata/vsphere.yaml | 4 +- pkg/userdata/ubuntu/provider_test.go | 6 +- ...rsion-1.22.7.yaml => version-1.23.13.yaml} | 4 +- .../ubuntu/testdata/version-1.23.5.yaml | 862 +++++++++-------- .../ubuntu/testdata/version-1.24.0.yaml | 893 +++++++++--------- .../ubuntu/testdata/version-1.24.7.yaml | 462 +++++++++ .../ubuntu/testdata/version-1.25.0.yaml | 438 +++++++++ .../ubuntu/testdata/version-1.25.3.yaml | 462 +++++++++ test/e2e/provisioning/helper.go | 6 +- ...hinedeployment-azure-redhat-satellite.yaml | 2 +- 67 files changed, 4833 insertions(+), 1964 deletions(-) rename pkg/userdata/amzn2/testdata/{kubelet-v1.22-aws.yaml => kubelet-v1.25-aws.yaml} (92%) rename pkg/userdata/centos/testdata/{kubelet-v1.22-aws.yaml => kubelet-v1.25-aws.yaml} (92%) rename pkg/userdata/flatcar/testdata/{cloud-init_v1.22.7.yaml => cloud-init_v1.25.0.yaml} (92%) rename pkg/userdata/flatcar/testdata/{ignition_v1.22.7.json => ignition_v1.25.0.json} (76%) create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.23.13.golden rename pkg/userdata/helper/testdata/{download_binaries_v1.22.7.golden => download_binaries_v1.24.7.golden} (92%) create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.25.3.golden rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.22.7-external.golden => kublet_systemd_unit_version-v1.23.13-external.golden} (90%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.22.7.golden => kublet_systemd_unit_version-v1.23.13.golden} (90%) create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.7-external.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.7.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.3-external.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.3.golden rename pkg/userdata/helper/testdata/{safe_download_binaries_v1.22.5.golden => safe_download_binaries_v1.23.13.golden} (98%) rename pkg/userdata/rhel/testdata/{kubelet-v1.22-aws.yaml => kubelet-v1.25-aws.yaml} (93%) rename pkg/userdata/rhel/testdata/{kubelet-v1.22-nutanix.yaml => kubelet-v1.25-nutanix.yaml} (93%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.22-aws.yaml => kubelet-v1.25-aws.yaml} (92%) rename pkg/userdata/sles/testdata/{version-1.22.7.yaml => version-1.23.13.yaml} (98%) create mode 100644 pkg/userdata/sles/testdata/version-1.24.7.yaml create mode 100644 pkg/userdata/sles/testdata/version-1.25.0.yaml create mode 100644 pkg/userdata/sles/testdata/version-1.25.3.yaml rename pkg/userdata/ubuntu/testdata/{version-1.22.7.yaml => version-1.23.13.yaml} (98%) create mode 100644 pkg/userdata/ubuntu/testdata/version-1.24.7.yaml create mode 100644 pkg/userdata/ubuntu/testdata/version-1.25.0.yaml create mode 100644 pkg/userdata/ubuntu/testdata/version-1.25.3.yaml diff --git a/README.md b/README.md index 2627af999..5197fc1b1 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,9 @@ machine-controller tries to follow the Kubernetes version Currently supported K8S versions are: +- 1.25 - 1.24 - 1.23 -- 1.22 ## What does not work diff --git a/examples/alibaba-machinedeployment.yaml b/examples/alibaba-machinedeployment.yaml index df5c0ccdb..940198f90 100644 --- a/examples/alibaba-machinedeployment.yaml +++ b/examples/alibaba-machinedeployment.yaml @@ -62,4 +62,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.22.5 + kubelet: 1.23.13 diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index f5210a441..8702e2a29 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -39,8 +39,8 @@ spec: # only a single disk is currently supported, but support for multiple disks is planned already disks: - - size: 60 - performanceType: ENT6 + - size: 60 + performanceType: ENT6 # You may have this old disk config attribute in your config - please migrate to the disks attribute. # For now it is still recognized though. @@ -53,4 +53,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.22.5 + kubelet: 1.23.13 diff --git a/examples/aws-machinedeployment.yaml b/examples/aws-machinedeployment.yaml index 6404fff68..b0ad7ea7a 100644 --- a/examples/aws-machinedeployment.yaml +++ b/examples/aws-machinedeployment.yaml @@ -80,4 +80,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.22.5 + kubelet: 1.23.13 diff --git a/examples/azure-machinedeployment.yaml b/examples/azure-machinedeployment.yaml index abbf780e5..c19b27b2e 100644 --- a/examples/azure-machinedeployment.yaml +++ b/examples/azure-machinedeployment.yaml @@ -92,4 +92,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.22.5 + kubelet: 1.23.13 diff --git a/examples/digitalocean-machinedeployment.yaml b/examples/digitalocean-machinedeployment.yaml index 5887aaf50..d6373cb5a 100644 --- a/examples/digitalocean-machinedeployment.yaml +++ b/examples/digitalocean-machinedeployment.yaml @@ -57,4 +57,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.22.5 + kubelet: 1.23.13 diff --git a/examples/equinixmetal-machinedeployment.yaml b/examples/equinixmetal-machinedeployment.yaml index c119de9d9..eae33e2d5 100644 --- a/examples/equinixmetal-machinedeployment.yaml +++ b/examples/equinixmetal-machinedeployment.yaml @@ -51,4 +51,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.22.5 + kubelet: 1.23.13 diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index 41f5105c4..acb73e180 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -86,4 +86,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.22.5 + kubelet: 1.23.13 diff --git a/examples/hetzner-machinedeployment.yaml b/examples/hetzner-machinedeployment.yaml index 6ca5784dc..a2da21a0f 100644 --- a/examples/hetzner-machinedeployment.yaml +++ b/examples/hetzner-machinedeployment.yaml @@ -63,4 +63,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.22.5 + kubelet: 1.23.13 diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index a3db171e3..41fe29c57 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -76,4 +76,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.22.5 + kubelet: 1.23.13 diff --git a/examples/linode-machinedeployment.yaml b/examples/linode-machinedeployment.yaml index 7caec78e0..da08f50de 100644 --- a/examples/linode-machinedeployment.yaml +++ b/examples/linode-machinedeployment.yaml @@ -53,4 +53,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.22.5 + kubelet: 1.23.13 diff --git a/examples/nutanix-machinedeployment.yaml b/examples/nutanix-machinedeployment.yaml index d9c42139c..5dc8ad9a8 100644 --- a/examples/nutanix-machinedeployment.yaml +++ b/examples/nutanix-machinedeployment.yaml @@ -38,13 +38,13 @@ spec: cloudProvider: "nutanix" cloudProviderSpec: # Can also be set via the env var 'NUTANIX_USERNAME' on the machine-controller - username: '<< NUTANIX_USERNAME >>' + username: "<< NUTANIX_USERNAME >>" # Can also be set via the env var 'NUTANIX_ENDPOINT' on the machine-controller # example: 'your-nutanix-host' or '10.0.1.5'. No protocol or port should be passed. - endpoint: '<< NUTANIX_ENDPOINT >>' + endpoint: "<< NUTANIX_ENDPOINT >>" # Can also be set via the env var 'NUTANIX_PORT' on the machine-controller # if not set, defaults to 9440 (default Nutanix port) - port: '<< NUTANIX_PORT >>' + port: "<< NUTANIX_PORT >>" # Optional: Allow insecure connections to endpoint if no valid TLS certificate is presented allowInsecure: true # Can also be set via the env var 'NUTANIX_PASSWORD' on the machine-controller @@ -78,4 +78,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.22.5 + kubelet: 1.23.13 diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index a5bc3eedc..8fa5eecf5 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -166,4 +166,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.22.5 + kubelet: 1.23.13 diff --git a/examples/scaleway-machinedeployment.yaml b/examples/scaleway-machinedeployment.yaml index 0cea3c70c..76bdb3cbf 100644 --- a/examples/scaleway-machinedeployment.yaml +++ b/examples/scaleway-machinedeployment.yaml @@ -61,4 +61,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.22.5 + kubelet: 1.23.13 diff --git a/examples/vmware-cloud-director-machinedeployment.yaml b/examples/vmware-cloud-director-machinedeployment.yaml index 5b5106dda..ec5baaac2 100644 --- a/examples/vmware-cloud-director-machinedeployment.yaml +++ b/examples/vmware-cloud-director-machinedeployment.yaml @@ -76,4 +76,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.22.5 + kubelet: 1.23.13 diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml index 6d2302fe8..b34ff9880 100644 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ b/examples/vsphere-datastore-cluster-machinedeployment.yaml @@ -38,10 +38,10 @@ spec: cloudProvider: "vsphere" cloudProviderSpec: # Can also be set via the env var 'VSPHERE_USERNAME' on the machine-controller - username: '<< VSPHERE_USERNAME >>' + username: "<< VSPHERE_USERNAME >>" # Can also be set via the env var 'VSPHERE_ADDRESS' on the machine-controller # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically - vsphereURL: '<< VSPHERE_ADDRESS >>' + vsphereURL: "<< VSPHERE_ADDRESS >>" # Can also be set via the env var 'VSPHERE_PASSWORD' on the machine-controller password: secretKeyRef: @@ -74,4 +74,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.22.5 + kubelet: 1.23.13 diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index f0442d72c..3425b432a 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -38,10 +38,10 @@ spec: cloudProvider: "vsphere" cloudProviderSpec: # Can also be set via the env var 'VSPHERE_USERNAME' on the machine-controller - username: '<< VSPHERE_USERNAME >>' + username: "<< VSPHERE_USERNAME >>" # Can also be set via the env var 'VSPHERE_ADDRESS' on the machine-controller # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically - vsphereURL: '<< VSPHERE_ADDRESS >>' + vsphereURL: "<< VSPHERE_ADDRESS >>" # Can also be set via the env var 'VSPHERE_PASSWORD' on the machine-controller password: secretKeyRef: @@ -74,4 +74,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.22.5 + kubelet: 1.23.13 diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index 816e2752f..b81db4cdb 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -99,15 +99,6 @@ func TestUserDataGeneration(t *testing.T) { t.Parallel() tests := []userDataTestCase{ - { - name: "kubelet-v1.22-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.5", - }, - }, - }, { name: "kubelet-v1.23-aws", spec: clusterv1alpha1.MachineSpec{ @@ -174,6 +165,15 @@ func TestUserDataGeneration(t *testing.T) { }, }, }, + { + name: "kubelet-v1.25-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.25.0", + }, + }, + }, } defaultCloudProvider := &fakeCloudConfigProvider{ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml similarity index 92% rename from pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml index 3eaa0477a..4e1dc46c2 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml @@ -80,22 +80,25 @@ write_files: ipvsadm - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + mkdir -p /etc/systemd/system/containerd.service.d - cat < /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment + - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + - path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + - path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + - path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + systemctl restart systemd-modules-load.service + sysctl --system + + zypper --non-interactive --quiet --color install ebtables \ + ceph-common \ + e2fsprogs \ + jq \ + socat \ + ipvsadm + + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v1.1.1}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.23.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + + - path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + + - path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + + - path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --network-plugin=cni \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + + - path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" + + - path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + - path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + - path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + - path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + - path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + + - path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + - path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + + - path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + + - path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + - path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + + - path: /etc/systemd/system/docker.service.d/environment.conf + permissions: "0644" + content: | + [Service] + EnvironmentFile=-/etc/environment runcmd: -- systemctl start setup.service + - systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.24.0.yaml b/pkg/userdata/sles/testdata/version-1.24.0.yaml index 579f094d2..c658c8962 100644 --- a/pkg/userdata/sles/testdata/version-1.24.0.yaml +++ b/pkg/userdata/sles/testdata/version-1.24.0.yaml @@ -2,438 +2,427 @@ hostname: node1 - ssh_pwauth: false ssh_authorized_keys: -- "ssh-rsa AAABBB" + - "ssh-rsa AAABBB" write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.0}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment + - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + - path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + - path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + - path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + systemctl restart systemd-modules-load.service + sysctl --system + + zypper --non-interactive --quiet --color install ebtables \ + ceph-common \ + e2fsprogs \ + jq \ + socat \ + ipvsadm + + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v1.1.1}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.23.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.24.0}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + + - path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + + - path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + + - path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + + - path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" + + - path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + - path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + - path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + - path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + - path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + + - path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + - path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + + - path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + - path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + - path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=containerd.service + After=containerd.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + + - path: /etc/systemd/system/docker.service.d/environment.conf + permissions: "0644" + content: | + [Service] + EnvironmentFile=-/etc/environment runcmd: -- systemctl start setup.service + - systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.24.7.yaml b/pkg/userdata/sles/testdata/version-1.24.7.yaml new file mode 100644 index 000000000..dffeefcae --- /dev/null +++ b/pkg/userdata/sles/testdata/version-1.24.7.yaml @@ -0,0 +1,439 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + systemctl restart systemd-modules-load.service + sysctl --system + + zypper --non-interactive --quiet --color install ebtables \ + ceph-common \ + e2fsprogs \ + jq \ + socat \ + ipvsadm + + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v1.1.1}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.24.7}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=containerd.service + After=containerd.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + +- path: /etc/systemd/system/docker.service.d/environment.conf + permissions: "0644" + content: | + [Service] + EnvironmentFile=-/etc/environment + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.25.0.yaml b/pkg/userdata/sles/testdata/version-1.25.0.yaml new file mode 100644 index 000000000..39d7f4c37 --- /dev/null +++ b/pkg/userdata/sles/testdata/version-1.25.0.yaml @@ -0,0 +1,415 @@ +#cloud-config + +hostname: node1 + +ssh_pwauth: false +ssh_authorized_keys: + - "ssh-rsa AAABBB" + +write_files: + - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + - path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + - path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + - path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + systemctl restart systemd-modules-load.service + sysctl --system + + zypper --non-interactive --quiet --color install ebtables \ + ceph-common \ + e2fsprogs \ + jq \ + socat \ + ipvsadm + + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v1.1.1}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.23.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + + - path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + + - path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + + - path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + + - path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" + + - path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + - path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + - path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + - path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + - path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + + - path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + - path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + + - path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + + - path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + - path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=docker.service + After=docker.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + + - path: /etc/systemd/system/docker.service.d/environment.conf + permissions: "0644" + content: | + [Service] + EnvironmentFile=-/etc/environment + +runcmd: + - systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.25.3.yaml b/pkg/userdata/sles/testdata/version-1.25.3.yaml new file mode 100644 index 000000000..30681b8d1 --- /dev/null +++ b/pkg/userdata/sles/testdata/version-1.25.3.yaml @@ -0,0 +1,439 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + systemctl restart systemd-modules-load.service + sysctl --system + + zypper --non-interactive --quiet --color install ebtables \ + ceph-common \ + e2fsprogs \ + jq \ + socat \ + ipvsadm + + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v1.1.1}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.25.3}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now docker + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl enable --now --no-block docker-healthcheck.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + cp /etc/fstab /etc/fstab.orig + cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap + mv /etc/fstab.noswap /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: /etc/systemd/system/docker-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=containerd.service + After=containerd.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh container-runtime + + [Install] + WantedBy=multi-user.target + +- path: /etc/systemd/system/docker.service.d/environment.conf + permissions: "0644" + content: | + [Service] + EnvironmentFile=-/etc/environment + +runcmd: +- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml index 3b4368b66..14f503d83 100644 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml @@ -111,7 +111,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.13}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -201,8 +201,6 @@ write_files: --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml index d94216edf..0ea26f723 100644 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/sles/testdata/vsphere-proxy.yaml @@ -111,7 +111,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.13}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -201,8 +201,6 @@ write_files: --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml index 73b893b14..df58c104e 100644 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ b/pkg/userdata/sles/testdata/vsphere.yaml @@ -102,7 +102,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.13}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -191,8 +191,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 14c230c61..404381e3a 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -128,9 +128,9 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.22.7"), - semver.MustParse("v1.23.5"), - semver.MustParse("v1.24.0"), + semver.MustParse("v1.23.13"), + semver.MustParse("v1.24.7"), + semver.MustParse("v1.25.3"), } var tests []userDataTestCase diff --git a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.13.yaml similarity index 98% rename from pkg/userdata/ubuntu/testdata/version-1.22.7.yaml rename to pkg/userdata/ubuntu/testdata/version-1.23.13.yaml index fef7938b8..fbe60e5d9 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.22.7.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.23.13.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" + KUBE_VERSION="${KUBE_VERSION:-v1.23.13}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -230,8 +230,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=docker \ --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml index 6ad303d59..81699ddb4 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.23.5.yaml @@ -2,445 +2,435 @@ hostname: node1 - ssh_pwauth: false ssh_authorized_keys: -- "ssh-rsa AAABBB" + - "ssh-rsa AAABBB" write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - + - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + - path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + - path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + - path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + + - path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + + - path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + + - path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + - path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --network-plugin=cni \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + + - path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + + - path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + - path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + - path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + - path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + - path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + + - path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + + - path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + + - path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + - path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target runcmd: -- systemctl enable --now setup.service + - systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml index d0773d79f..5e5aac8e7 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml @@ -2,461 +2,450 @@ hostname: node1 - ssh_pwauth: false ssh_authorized_keys: -- "ssh-rsa AAABBB" + - "ssh-rsa AAABBB" write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - + - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + - path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + - path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + - path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + + - path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + + - path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + + - path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + - path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + + - path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + + - path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + - path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + - path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + - path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + - path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + + - path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + + - path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + - path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + - path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target runcmd: -- systemctl enable --now setup.service + - systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.7.yaml b/pkg/userdata/ubuntu/testdata/version-1.24.7.yaml new file mode 100644 index 000000000..bfc934f34 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.24.7.yaml @@ -0,0 +1,462 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.25.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.25.0.yaml new file mode 100644 index 000000000..5d6f8d440 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.25.0.yaml @@ -0,0 +1,438 @@ +#cloud-config + +hostname: node1 + +ssh_pwauth: false +ssh_authorized_keys: + - "ssh-rsa AAABBB" + +write_files: + - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + - path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + - path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + - path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + + - path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + + - path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + + - path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + - path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + + - path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + + - path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + - path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + - path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + - path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + - path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + + - path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + + - path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + + - path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + - path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + +runcmd: + - systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.25.3.yaml b/pkg/userdata/ubuntu/testdata/version-1.25.3.yaml new file mode 100644 index 000000000..524eb9fa6 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.25.3.yaml @@ -0,0 +1,462 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 6c506ad6a..c747de3b1 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,9 +33,9 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.22.14"), - semver.MustParse("v1.23.11"), - semver.MustParse("v1.24.5"), + semver.MustParse("v1.23.13"), + semver.MustParse("v1.24.7"), + semver.MustParse("v1.25.3"), } operatingSystems = []providerconfigtypes.OperatingSystem{ diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml index 728314a79..5ed66030c 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml @@ -55,4 +55,4 @@ spec: rhelOrganizationName: "" rhelActivationKey: "" versions: - kubelet: 1.22.5 + kubelet: 1.23.13 From 66821866afa5a3dbed1cec35e34b1bba76224882 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 2 Nov 2022 21:11:57 +0500 Subject: [PATCH 239/489] Upgrade to Go 1.19.3 (#1475) Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 20 ++++++++++---------- .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 6 +++--- .prow/verify.yaml | 10 +++++----- Dockerfile | 2 +- Makefile | 2 +- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- 22 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 321847a53..dd0540094 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -91,7 +91,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -118,7 +118,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 35281a3a5..2622cf92f 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index 8745901bb..a9d76b58d 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 78da9e24f..dbf5a22bc 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 99038333a..8d5906f80 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -57,7 +57,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -88,7 +88,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -116,7 +116,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -144,7 +144,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -173,7 +173,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -201,7 +201,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -229,7 +229,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -257,7 +257,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -285,7 +285,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 075df153b..d0fe23112 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -57,7 +57,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -87,7 +87,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index d58f2a01f..cb6984b8a 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 20a2ea8ee..2fe932093 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index ccd1180f8..fe4aa3c4a 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index d712ce4ae..54063d31f 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index c54c43661..43c8ab7f8 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 0289bb263..2a7b6b0a0 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index c0b955224..1e5832317 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 74ab54f68..5076ab630 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -57,7 +57,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index c56328077..ff2c6a48a 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 361c1f1a6..0a6511dc7 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 682c8678e..9ce2522f6 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -57,7 +57,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -86,7 +86,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index a6953b3ed..8584ef439 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.19.0 + - image: golang:1.19.3 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.19.0 + - image: golang:1.19.3 command: - make args: @@ -63,7 +63,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golangci/golangci-lint:v1.49.0 + - image: golangci/golangci-lint:v1.50.1 command: - make args: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.19.0 + - image: golang:1.19.3 command: - make args: diff --git a/Dockerfile b/Dockerfile index 3e04a892c..157690265 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.19.0 +ARG GO_VERSION=1.19.3 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index 21e8bcf74..80b57afed 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.19.0 +GO_VERSION ?= 1.19.3 GOOS ?= $(shell go env GOOS) diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 077ab9420..9422adf73 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.19.0 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.19.3 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index cf0783724..470109588 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.19-node-18-kind-0.14-0 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 containerize ./hack/verify-licenses.sh go mod vendor From fb0531deda46630c7de80dfe63cdfefeff3e69d6 Mon Sep 17 00:00:00 2001 From: lucakuendig Date: Thu, 3 Nov 2022 16:16:13 +0100 Subject: [PATCH 240/489] add optional additional subnets to Nutanix provider (#1471) * add optional storageSubnet to Nutanix provider Signed-off-by: Luca Kuendig * add additionalSubnetNames field to ntnx-provider Signed-off-by: Luca Kuendig Signed-off-by: Luca Kuendig --- examples/nutanix-machinedeployment.yaml | 4 +++ pkg/cloudprovider/provider/nutanix/client.go | 32 ++++++++++++++----- .../provider/nutanix/provider.go | 17 +++++++--- .../provider/nutanix/types/types.go | 9 +++--- test/e2e/provisioning/all_e2e_test.go | 2 ++ .../testdata/machinedeployment-nutanix.yaml | 1 + 6 files changed, 49 insertions(+), 16 deletions(-) diff --git a/examples/nutanix-machinedeployment.yaml b/examples/nutanix-machinedeployment.yaml index 5dc8ad9a8..ed4d147ae 100644 --- a/examples/nutanix-machinedeployment.yaml +++ b/examples/nutanix-machinedeployment.yaml @@ -60,6 +60,10 @@ spec: projectName: project1 # Sets the subnet that the VM is connected to. Must exist in the given Nutanix cluster subnetName: subnet1 + # Optional: Sets multiple additional subnets that the VM is connected to. Must exist in the given Nutanix cluster + # additionalSubnetNames: + # - subnet2 + # - subnet3 # Provides the image used to create the VM imageName: ubuntu-20.04 # Sets the vCPU count for this VM diff --git a/pkg/cloudprovider/provider/nutanix/client.go b/pkg/cloudprovider/provider/nutanix/client.go index ca09c90ca..332769823 100644 --- a/pkg/cloudprovider/provider/nutanix/client.go +++ b/pkg/cloudprovider/provider/nutanix/client.go @@ -104,6 +104,29 @@ func createVM(ctx context.Context, client *ClientSet, name string, conf Config, return nil, err } + nicList := []*nutanixv3.VMNic{ + { + SubnetReference: &nutanixv3.Reference{ + Kind: pointer.String(nutanixtypes.SubnetKind), + UUID: subnet.Metadata.UUID, + }, + }, + } + + for _, subnet := range conf.AdditionalSubnetNames { + additionalSubnet, err := getSubnetByName(ctx, client, subnet, *cluster.Metadata.UUID) + if err != nil { + return nil, err + } + additionalSubnetNic := &nutanixv3.VMNic{ + SubnetReference: &nutanixv3.Reference{ + Kind: pointer.String(nutanixtypes.SubnetKind), + UUID: additionalSubnet.Metadata.UUID, + }, + } + nicList = append(nicList, additionalSubnetNic) + } + image, err := getImageByName(ctx, client, conf.ImageName) if err != nil { return nil, err @@ -127,14 +150,7 @@ func createVM(ctx context.Context, client *ClientSet, name string, conf Config, PowerState: pointer.String("ON"), NumSockets: pointer.Int64(conf.CPUs), MemorySizeMib: pointer.Int64(conf.MemoryMB), - NicList: []*nutanixv3.VMNic{ - { - SubnetReference: &nutanixv3.Reference{ - Kind: pointer.String(nutanixtypes.SubnetKind), - UUID: subnet.Metadata.UUID, - }, - }, - }, + NicList: nicList, DiskList: []*nutanixv3.VMDisk{ { DeviceProperties: &nutanixv3.VMDiskDeviceProperties{ diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 70c2b701e..8e43ce3ed 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -45,10 +45,11 @@ type Config struct { AllowInsecure bool ProxyURL string - ClusterName string - ProjectName string - SubnetName string - ImageName string + ClusterName string + ProjectName string + SubnetName string + AdditionalSubnetNames []string + ImageName string Categories map[string]string @@ -181,6 +182,8 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, nil, err } + c.AdditionalSubnetNames = append(c.AdditionalSubnetNames, rawConfig.AdditionalSubnetNames...) + c.ImageName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ImageName) if err != nil { return nil, nil, nil, err @@ -227,6 +230,12 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return fmt.Errorf("failed to get subnet: %w", err) } + for _, subnet := range config.AdditionalSubnetNames { + if _, err := getSubnetByName(ctx, client, subnet, *cluster.Metadata.UUID); err != nil { + return fmt.Errorf("failed to get subnet: %w", err) + } + } + image, err := getImageByName(ctx, client, config.ImageName) if err != nil { return fmt.Errorf("failed to get image: %w", err) diff --git a/pkg/cloudprovider/provider/nutanix/types/types.go b/pkg/cloudprovider/provider/nutanix/types/types.go index 007e270b4..a2283b721 100644 --- a/pkg/cloudprovider/provider/nutanix/types/types.go +++ b/pkg/cloudprovider/provider/nutanix/types/types.go @@ -38,10 +38,11 @@ type RawConfig struct { AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` ProxyURL providerconfigtypes.ConfigVarString `json:"proxyURL,omitempty"` - ClusterName providerconfigtypes.ConfigVarString `json:"clusterName"` - ProjectName *providerconfigtypes.ConfigVarString `json:"projectName,omitempty"` - SubnetName providerconfigtypes.ConfigVarString `json:"subnetName"` - ImageName providerconfigtypes.ConfigVarString `json:"imageName"` + ClusterName providerconfigtypes.ConfigVarString `json:"clusterName"` + ProjectName *providerconfigtypes.ConfigVarString `json:"projectName,omitempty"` + SubnetName providerconfigtypes.ConfigVarString `json:"subnetName"` + AdditionalSubnetNames []string `json:"additionalSubnetNames,omitempty"` + ImageName providerconfigtypes.ConfigVarString `json:"imageName"` // VM sizing configuration CPUs int64 `json:"cpus"` diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 9cec18e1d..3716f473f 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -949,6 +949,7 @@ func getNutanixTestParams(t *testing.T) []string { cluster := os.Getenv("NUTANIX_E2E_CLUSTER_NAME") project := os.Getenv("NUTANIX_E2E_PROJECT_NAME") subnet := os.Getenv("NUTANIX_E2E_SUBNET_NAME") + additionalSubnetNames := os.Getenv("NUTANIX_E2E_ADDITIONAL_SUBNET_NAMES") endpoint := os.Getenv("NUTANIX_E2E_ENDPOINT") if password == "" || username == "" || endpoint == "" || cluster == "" || project == "" || subnet == "" { @@ -963,6 +964,7 @@ func getNutanixTestParams(t *testing.T) []string { fmt.Sprintf("<< NUTANIX_CLUSTER >>=%s", cluster), fmt.Sprintf("<< NUTANIX_PROJECT >>=%s", project), fmt.Sprintf("<< NUTANIX_SUBNET >>=%s", subnet), + fmt.Sprintf("<< NUTANIX_ADDITIONAL_SUBNETS >>=%s", additionalSubnetNames), } return params } diff --git a/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml b/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml index f6315d4a0..2ac36dd59 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml @@ -33,6 +33,7 @@ spec: clusterName: '<< NUTANIX_CLUSTER >>' projectName: '<< NUTANIX_PROJECT >>' subnetName: '<< NUTANIX_SUBNET >>' + additionalSubnetNames: [] imageName: 'machine-controller-e2e-<< OS_NAME >>' cpus: 2 memoryMB: 2048 From b70f5d2a46c238fffa15ce0fa35a38c2f5e1df0a Mon Sep 17 00:00:00 2001 From: lucakuendig Date: Mon, 7 Nov 2022 10:01:49 +0100 Subject: [PATCH 241/489] restart containerd for flatcar so the drop in conf will be considered (#1476) * restart containerd for flatcar so the drop in conf will be considered Signed-off-by: Luca Kuendig * update fixture data Signed-off-by: Luca Kuendig Signed-off-by: Luca Kuendig --- pkg/containerruntime/containerd.go | 2 +- pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml | 2 +- pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml | 2 +- pkg/userdata/flatcar/testdata/containerd.yaml | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.24.0.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.25.0.json | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index 248ff0bb4..6337e04ad 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -106,7 +106,7 @@ ExecStart=/usr/bin/env PATH=\${TORCX_BINDIR}:\${PATH} \${TORCX_BINDIR}/container EOF systemctl daemon-reload -systemctl enable --now containerd +systemctl restart containerd `)) containerdAmzn2Template = template.Must(template.New("containerd-yum-amzn2").Parse(` diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml index e075a0484..acab7c412 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml @@ -462,7 +462,7 @@ write_files: EOF systemctl daemon-reload - systemctl enable --now containerd + systemctl restart containerd systemctl disable download-script.service diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml index 1a313495a..a7b1db9bd 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml @@ -462,7 +462,7 @@ write_files: EOF systemctl daemon-reload - systemctl enable --now containerd + systemctl restart containerd systemctl disable download-script.service diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index 6c9e0cfe5..1880058a8 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -445,7 +445,7 @@ write_files: EOF systemctl daemon-reload - systemctl enable --now containerd + systemctl restart containerd systemctl disable download-script.service diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json index 9cfe46ecf..d44e3c5be 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.1.1%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.1.1%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json index 9c0695a35..178620395 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.1.1%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.25.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20enable%20--now%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fdynamic-config-dir%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.1.1%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.25.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file From c259e08a4583cc5fe945d940b83ced9254d34914 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 7 Nov 2022 22:05:00 +0500 Subject: [PATCH 242/489] Update github URL references to use main branch (#1477) Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- cmd/machine-controller/main.go | 2 +- code-of-conduct.md | 2 +- docs/kubevirt.md | 2 +- pkg/cloudprovider/provider/vsphere/network.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 177841b78..171327cc5 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -175,7 +175,7 @@ func main() { flag.BoolVar(&nodeCSRApprover, "node-csr-approver", true, "Enable NodeCSRApprover controller to automatically approve node serving certificate requests") flag.StringVar(&podCIDR, "pod-cidr", "172.25.0.0/16", "WARNING: flag is unused, kept only for backwards compatibility") flag.StringVar(&nodePortRange, "node-port-range", "30000-32767", "A port range to reserve for services with NodePort visibility") - flag.StringVar(&nodeRegistryCredentialsSecret, "node-registry-credentials-secret", "", "A Secret object reference, that contains auth info for image registry in namespace/secret-name form, example: kube-system/registry-credentials. See doc at https://github.com/kubermaric/machine-controller/blob/master/docs/registry-authentication.md") + flag.StringVar(&nodeRegistryCredentialsSecret, "node-registry-credentials-secret", "", "A Secret object reference, that contains auth info for image registry in namespace/secret-name form, example: kube-system/registry-credentials. See doc at https://github.com/kubermaric/machine-controller/blob/main/docs/registry-authentication.md") flag.BoolVar(&useOSM, "use-osm", false, "DEPRECATED: use osm controller for node bootstrap [use use-external-bootstrap instead]") flag.BoolVar(&useExternalBootstrap, "use-external-bootstrap", false, "use an external bootstrap provider for instance user-data (e.g. operating-system-manager, also known as OSM)") flag.StringVar(&overrideBootstrapKubeletAPIServer, "override-bootstrap-kubelet-apiserver", "", "Override for the API server address used in worker nodes bootstrap-kubelet.conf") diff --git a/code-of-conduct.md b/code-of-conduct.md index dcddd3fe0..66c44de25 100644 --- a/code-of-conduct.md +++ b/code-of-conduct.md @@ -34,4 +34,4 @@ when an individual is representing the project or its community. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the Kubermatic Conduct Committee via coc@kubermatic.com. -This Code of Conduct is adapted from the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md) and [Contributor Covenant](http://contributor-covenant.org/version/1/2/0/), version 1.2.0. +This Code of Conduct is adapted from the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md) and [Contributor Covenant](http://contributor-covenant.org/version/1/2/0/), version 1.2.0. diff --git a/docs/kubevirt.md b/docs/kubevirt.md index dd14ba441..118663546 100644 --- a/docs/kubevirt.md +++ b/docs/kubevirt.md @@ -4,7 +4,7 @@ In order to use the machine-controller to create machines using [Kubevirt](https you must first install the latter. We provide a manifest for this, simply run `kubectl apply -f examples/kubevirt-operator-0.19.0.yaml`. We strongly recommend installing a version which is equal or higher than `0.19.0`. Machine Controller also uses the KubeVirt CDI which can be found under `examples/cdi-operator.yaml` to provision storage. It is important to have a basic understanding of Kubernetes storage. For more -information regarding which types of storage can be used please refer to [KubeVirt documentation](https://github.com/kubevirt/containerized-data-importer/blob/master/doc/basic_pv_pvc_dv.md). +information regarding which types of storage can be used please refer to [KubeVirt documentation](https://github.com/kubevirt/containerized-data-importer/blob/main/doc/basic_pv_pvc_dv.md). Afterwards, you can use the provided `exampes/examples/kubevirt-machinedeployment.yaml` as base. There diff --git a/pkg/cloudprovider/provider/vsphere/network.go b/pkg/cloudprovider/provider/vsphere/network.go index 6bb4ecbcb..e38d11135 100644 --- a/pkg/cloudprovider/provider/vsphere/network.go +++ b/pkg/cloudprovider/provider/vsphere/network.go @@ -28,7 +28,7 @@ const ( ethCardType = "vmxnet3" ) -// Based on https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/master/pkg/cloud/vsphere/services/govmomi/vcenter/clone.go#L158 +// Based on https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/pkg/cloud/vsphere/services/govmomi/vcenter/clone.go#L158 func GetNetworkSpecs(ctx context.Context, session *Session, devices object.VirtualDeviceList, network string) ([]types.BaseVirtualDeviceConfigSpec, error) { var deviceSpecs []types.BaseVirtualDeviceConfigSpec From e94dfa5fd85fe92aba3fd930458da52fd52a5cf8 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 8 Nov 2022 18:56:04 +0100 Subject: [PATCH 243/489] Refactor overwriteCloudConfigSpec for MDs (#1478) * refactor overwriteCloudConfigSpec for MDs Signed-off-by: Moath Qasim * generate fixtures Signed-off-by: Moath Qasim Signed-off-by: Moath Qasim --- pkg/controller/machine/machine_controller.go | 50 ++++++++++--------- pkg/userdata/amzn2/provider.go | 2 + .../testdata/kubelet-v1.23-aws-external.yaml | 1 - .../amzn2/testdata/kubelet-v1.23-aws.yaml | 1 - .../kubelet-v1.23-vsphere-mirrors.yaml | 1 - .../testdata/kubelet-v1.23-vsphere-proxy.yaml | 1 - .../amzn2/testdata/kubelet-v1.23-vsphere.yaml | 1 - .../amzn2/testdata/kubelet-v1.24-aws.yaml | 1 - .../amzn2/testdata/kubelet-v1.25-aws.yaml | 1 - pkg/userdata/centos/provider.go | 2 + .../testdata/kubelet-v1.23-aws-external.yaml | 1 - .../centos/testdata/kubelet-v1.23-aws.yaml | 1 - .../testdata/kubelet-v1.23-nutanix.yaml | 1 - .../kubelet-v1.23-vsphere-mirrors.yaml | 1 - .../testdata/kubelet-v1.23-vsphere-proxy.yaml | 1 - .../testdata/kubelet-v1.23-vsphere.yaml | 1 - .../centos/testdata/kubelet-v1.24-aws.yaml | 1 - .../centos/testdata/kubelet-v1.25-aws.yaml | 1 - pkg/userdata/flatcar/provider.go | 2 + pkg/userdata/rhel/provider.go | 2 + .../testdata/kubelet-v1.23-aws-external.yaml | 1 - .../rhel/testdata/kubelet-v1.23-aws.yaml | 1 - .../kubelet-v1.23-vsphere-mirrors.yaml | 1 - .../testdata/kubelet-v1.23-vsphere-proxy.yaml | 1 - .../rhel/testdata/kubelet-v1.23-vsphere.yaml | 1 - .../testdata/kubelet-v1.24-aws-external.yaml | 1 - .../rhel/testdata/kubelet-v1.24-aws.yaml | 1 - .../rhel/testdata/kubelet-v1.25-aws.yaml | 1 - .../rhel/testdata/kubelet-v1.25-nutanix.yaml | 1 - .../rhel/testdata/pod-cidr-azure-rhel.yaml | 1 - pkg/userdata/rockylinux/provider.go | 2 + .../testdata/kubelet-v1.23-aws-external.yaml | 1 - .../testdata/kubelet-v1.23-aws.yaml | 1 - .../testdata/kubelet-v1.23-nutanix.yaml | 1 - .../kubelet-v1.23-vsphere-mirrors.yaml | 1 - .../testdata/kubelet-v1.23-vsphere-proxy.yaml | 1 - .../testdata/kubelet-v1.23-vsphere.yaml | 1 - .../testdata/kubelet-v1.24-aws.yaml | 1 - .../testdata/kubelet-v1.25-aws.yaml | 1 - pkg/userdata/sles/provider.go | 2 + .../sles/testdata/dist-upgrade-on-boot.yaml | 5 -- .../kubelet-version-without-v-prefix.yaml | 5 -- .../sles/testdata/multiple-dns-servers.yaml | 5 -- .../sles/testdata/multiple-ssh-keys.yaml | 5 -- .../openstack-overwrite-cloud-config.yaml | 1 - pkg/userdata/sles/testdata/openstack.yaml | 1 - .../sles/testdata/version-1.23.13.yaml | 5 -- .../sles/testdata/version-1.24.7.yaml | 5 -- .../sles/testdata/version-1.25.3.yaml | 5 -- .../sles/testdata/vsphere-mirrors.yaml | 1 - pkg/userdata/sles/testdata/vsphere-proxy.yaml | 1 - pkg/userdata/sles/testdata/vsphere.yaml | 1 - pkg/userdata/ubuntu/provider.go | 2 + pkg/userdata/ubuntu/testdata/containerd.yaml | 5 -- .../testdata/digitalocean-dualstack.yaml | 1 - .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 5 -- pkg/userdata/ubuntu/testdata/docker.yaml | 5 -- .../kubelet-version-without-v-prefix.yaml | 5 -- .../ubuntu/testdata/multiple-dns-servers.yaml | 5 -- .../ubuntu/testdata/multiple-ssh-keys.yaml | 5 -- pkg/userdata/ubuntu/testdata/nutanix.yaml | 1 - .../ubuntu/testdata/openstack-dualstack.yaml | 1 - .../openstack-overwrite-cloud-config.yaml | 1 - pkg/userdata/ubuntu/testdata/openstack.yaml | 1 - .../ubuntu/testdata/version-1.23.13.yaml | 5 -- .../ubuntu/testdata/version-1.24.7.yaml | 5 -- .../ubuntu/testdata/version-1.25.3.yaml | 5 -- .../ubuntu/testdata/vsphere-mirrors.yaml | 1 - .../ubuntu/testdata/vsphere-proxy.yaml | 1 - pkg/userdata/ubuntu/testdata/vsphere.yaml | 1 - 70 files changed, 41 insertions(+), 149 deletions(-) diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 5c183c6b8..734479675 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -756,11 +756,6 @@ func (r *Reconciler) ensureInstanceExistsForMachine( } } - cloudConfig, kubeletCloudProviderName, err := prov.GetCloudConfig(machine.Spec) - if err != nil { - return nil, fmt.Errorf("failed to render cloud config: %w", err) - } - // grab kubelet featureGates from the annotations kubeletFeatureGates := common.GetKubeletFeatureGates(machine.GetAnnotations()) if len(kubeletFeatureGates) == 0 { @@ -778,6 +773,15 @@ func (r *Reconciler) ensureInstanceExistsForMachine( externalCloudProvider, _ = strconv.ParseBool(val) } + cloudConfig, kubeletCloudProviderName, err := prov.GetCloudConfig(machine.Spec) + if err != nil { + return nil, fmt.Errorf("failed to render cloud config: %w", err) + } + + if providerConfig.CloudProvider == providerconfigtypes.CloudProviderVsphere && externalCloudProvider { + cloudConfig = "" + } + registryCredentials, err := containerruntime.GetContainerdAuthConfig(ctx, r.client, r.nodeSettings.RegistryCredentialsSecretRef) if err != nil { return nil, fmt.Errorf("failed to get containerd auth config: %w", err) @@ -794,23 +798,6 @@ func (r *Reconciler) ensureInstanceExistsForMachine( crRuntime.ContainerLogMaxFiles = val } - req := plugin.UserDataRequest{ - MachineSpec: machine.Spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: string(providerConfig.CloudProvider), - ExternalCloudProvider: externalCloudProvider, - DNSIPs: r.nodeSettings.ClusterDNSIPs, - PauseImage: r.nodeSettings.PauseImage, - KubeletCloudProviderName: kubeletCloudProviderName, - KubeletFeatureGates: kubeletFeatureGates, - KubeletConfigs: kubeletConfigs, - NoProxy: r.nodeSettings.NoProxy, - HTTPProxy: r.nodeSettings.HTTPProxy, - ContainerRuntime: crRuntime, - NodePortRange: r.nodePortRange, - } - // Here we do stuff! var userdata string @@ -838,8 +825,25 @@ func (r *Reconciler) ensureInstanceExistsForMachine( return nil, fmt.Errorf(CloudInitNotReadyError, bootstrap.BootstrapCloudConfig, machine.Name) } - userdata = getOSMBootstrapUserdata(req.MachineSpec.Name, *bootstrapSecret) + userdata = getOSMBootstrapUserdata(machine.Spec.Name, *bootstrapSecret) } else { + req := plugin.UserDataRequest{ + MachineSpec: machine.Spec, + Kubeconfig: kubeconfig, + CloudConfig: cloudConfig, + CloudProviderName: string(providerConfig.CloudProvider), + ExternalCloudProvider: externalCloudProvider, + DNSIPs: r.nodeSettings.ClusterDNSIPs, + PauseImage: r.nodeSettings.PauseImage, + KubeletCloudProviderName: kubeletCloudProviderName, + KubeletFeatureGates: kubeletFeatureGates, + KubeletConfigs: kubeletConfigs, + NoProxy: r.nodeSettings.NoProxy, + HTTPProxy: r.nodeSettings.HTTPProxy, + ContainerRuntime: crRuntime, + NodePortRange: r.nodePortRange, + } + userdata, err = userdataPlugin.UserData(req) if err != nil { return nil, fmt.Errorf("failed get userdata: %w", err) diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index 10e5fb480..67fb2f115 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -256,10 +256,12 @@ write_files: content: | {{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} +{{- if ne (len .CloudConfig) 0 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | {{ .CloudConfig | indent 4 }} +{{- end }} - path: "/opt/bin/setup_net_env.sh" permissions: "0755" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml index a3590ffa4..41b8b3285 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml @@ -230,7 +230,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml index 34502bff9..c40cdffcd 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws.yaml @@ -230,7 +230,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 65c9163f2..cec128619 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -247,7 +247,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-proxy.yaml index b59b08e0f..0d8b9b1c9 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -247,7 +247,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere.yaml index 3e814c32e..623a1df54 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.23-vsphere.yaml @@ -238,7 +238,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml index c623222c1..eb2122fc8 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml @@ -232,7 +232,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml index 4e1dc46c2..a96eb9d8f 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml @@ -232,7 +232,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index 3943bf0a8..353e8ed68 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -274,10 +274,12 @@ write_files: content: | {{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} +{{- if ne (len .CloudConfig) 0 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | {{ .CloudConfig | indent 4 }} +{{- end }} - path: "/opt/bin/setup_net_env.sh" permissions: "0755" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml index c5189aac5..6a137b749 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml @@ -240,7 +240,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml index f94370545..093952f6f 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-aws.yaml @@ -240,7 +240,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml index 19a57bbc9..ace011233 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-nutanix.yaml @@ -248,7 +248,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml index dd1e83fc8..bfc70c0bc 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -257,7 +257,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml index 6a7352d08..45ff5f31c 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -257,7 +257,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml index 25051264e..5db328279 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.23-vsphere.yaml @@ -248,7 +248,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml index cf85e964d..c1406f191 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24-aws.yaml @@ -238,7 +238,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml index 1d156ff8d..2c9c6dc9c 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml @@ -238,7 +238,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index 7ffc944c3..6a22a14a6 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -388,12 +388,14 @@ storage: inline: | {{ .Kubeconfig | indent 10 }} +{{- if ne (len .CloudConfig) 0 }} - path: /etc/kubernetes/cloud-config filesystem: root mode: 0400 contents: inline: | {{ .CloudConfig | indent 10 }} +{{- end }} - path: /etc/kubernetes/pki/ca.crt filesystem: root diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 41016bc28..5eee08984 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -281,10 +281,12 @@ write_files: content: | {{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} +{{- if ne (len .CloudConfig) 0 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | {{ .CloudConfig | indent 4 }} +{{- end }} - path: "/opt/bin/setup_net_env.sh" permissions: "0755" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml index a79090572..6080a6997 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml @@ -247,7 +247,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml index 159364690..b9119d159 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml @@ -247,7 +247,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml index 60d49d610..a3e40693e 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -265,7 +265,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml index dd7d36833..b9a9d49f2 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -265,7 +265,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml index 1e4785fc6..7d2579248 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml @@ -256,7 +256,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml index 5cc284938..3d3488b37 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml @@ -245,7 +245,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml index 8e8a5918f..7f6a0a312 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml @@ -245,7 +245,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml index 8ee45c1f2..4c0bd1450 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml @@ -245,7 +245,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml index 7ae0c8f32..66b93cd24 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml @@ -254,7 +254,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index f48f536b3..ad57862cf 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -251,7 +251,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rockylinux/provider.go b/pkg/userdata/rockylinux/provider.go index 7efb05332..c19a5dcd1 100644 --- a/pkg/userdata/rockylinux/provider.go +++ b/pkg/userdata/rockylinux/provider.go @@ -278,10 +278,12 @@ write_files: content: | {{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} +{{- if ne (len .CloudConfig) 0 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | {{ .CloudConfig | indent 4 }} +{{- end }} - path: "/opt/bin/setup_net_env.sh" permissions: "0755" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml index de9e8b81a..3d718ee13 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml @@ -247,7 +247,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml index 871afd164..5c155c38a 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml @@ -247,7 +247,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml index 6280d8c5a..6018fe461 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml @@ -255,7 +255,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml index c77004f2b..3cfecc74c 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml @@ -264,7 +264,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml index 968eaf047..929a9aed7 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml @@ -264,7 +264,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml index 2900f5c3c..f0d5c8ea8 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml @@ -255,7 +255,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml index 7af96633d..f1a87ecc7 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml @@ -245,7 +245,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml index c532a9225..2c82621c0 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml @@ -245,7 +245,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/sles/provider.go b/pkg/userdata/sles/provider.go index 41fbf06a5..7e3a3c773 100644 --- a/pkg/userdata/sles/provider.go +++ b/pkg/userdata/sles/provider.go @@ -223,10 +223,12 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" +{{- if ne (len .CloudConfig) 0 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | {{ .CloudConfig | indent 4 }} +{{- end }} - path: "/opt/bin/setup_net_env.sh" permissions: "0755" diff --git a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml index d3c8cab0c..db5c5a074 100644 --- a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml @@ -201,11 +201,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml index d4a8e4b41..0c8d538fb 100644 --- a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml @@ -199,11 +199,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml index f6eae6a35..256c96780 100644 --- a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml @@ -199,11 +199,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml index 7e68ecab5..73f64c416 100644 --- a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml @@ -201,11 +201,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml index 6bdf72df4..e03af45db 100644 --- a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml @@ -200,7 +200,6 @@ write_files: content: | [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/sles/testdata/openstack.yaml b/pkg/userdata/sles/testdata/openstack.yaml index 11cc65653..fe64d675b 100644 --- a/pkg/userdata/sles/testdata/openstack.yaml +++ b/pkg/userdata/sles/testdata/openstack.yaml @@ -200,7 +200,6 @@ write_files: content: | [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/sles/testdata/version-1.23.13.yaml b/pkg/userdata/sles/testdata/version-1.23.13.yaml index d4a8e4b41..0c8d538fb 100644 --- a/pkg/userdata/sles/testdata/version-1.23.13.yaml +++ b/pkg/userdata/sles/testdata/version-1.23.13.yaml @@ -199,11 +199,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/sles/testdata/version-1.24.7.yaml b/pkg/userdata/sles/testdata/version-1.24.7.yaml index dffeefcae..4177de049 100644 --- a/pkg/userdata/sles/testdata/version-1.24.7.yaml +++ b/pkg/userdata/sles/testdata/version-1.24.7.yaml @@ -198,11 +198,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/sles/testdata/version-1.25.3.yaml b/pkg/userdata/sles/testdata/version-1.25.3.yaml index 30681b8d1..d301bdb07 100644 --- a/pkg/userdata/sles/testdata/version-1.25.3.yaml +++ b/pkg/userdata/sles/testdata/version-1.25.3.yaml @@ -198,11 +198,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml index 14f503d83..5d6625bc0 100644 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml @@ -211,7 +211,6 @@ write_files: content: | [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml index 0ea26f723..e24d417c6 100644 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/sles/testdata/vsphere-proxy.yaml @@ -211,7 +211,6 @@ write_files: content: | [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml index df58c104e..d2b24d668 100644 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ b/pkg/userdata/sles/testdata/vsphere.yaml @@ -201,7 +201,6 @@ write_files: content: | [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index 5a83a8a1b..2dfb8d3a7 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -270,10 +270,12 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" +{{- if ne (len .CloudConfig) 0 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | {{ .CloudConfig | indent 4 }} +{{- end }} - path: "/opt/bin/setup_net_env.sh" permissions: "0755" diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index fff144f5e..a9145f640 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -245,11 +245,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml index 83120b0c9..e9325a051 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml @@ -242,7 +242,6 @@ write_files: content: | [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 2857fb810..99474c110 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -245,11 +245,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml index d761c1a2e..eb31b4c9e 100644 --- a/pkg/userdata/ubuntu/testdata/docker.yaml +++ b/pkg/userdata/ubuntu/testdata/docker.yaml @@ -245,11 +245,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index fef7938b8..e9226242a 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -243,11 +243,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 5dd087238..0f68c6f19 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -243,11 +243,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index ed129f539..7b9f2a6e1 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -245,11 +245,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 344806614..bbbc89e30 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -245,7 +245,6 @@ write_files: content: | [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml index be1c64a23..f862a2f7c 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml @@ -242,7 +242,6 @@ write_files: content: | [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index 5e7ec0126..c2d25c5af 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -244,7 +244,6 @@ write_files: content: | [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index b087ed873..c9c4306b8 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -244,7 +244,6 @@ write_files: content: | [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.13.yaml b/pkg/userdata/ubuntu/testdata/version-1.23.13.yaml index fbe60e5d9..ae7b981d8 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.23.13.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.23.13.yaml @@ -241,11 +241,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.7.yaml b/pkg/userdata/ubuntu/testdata/version-1.24.7.yaml index bfc934f34..8dbed92ef 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.24.7.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.24.7.yaml @@ -240,11 +240,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/ubuntu/testdata/version-1.25.3.yaml b/pkg/userdata/ubuntu/testdata/version-1.25.3.yaml index 524eb9fa6..a2a512b45 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.25.3.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.25.3.yaml @@ -240,11 +240,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index 31c647d6d..81cb912ca 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -255,7 +255,6 @@ write_files: content: | [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 16127b363..27d0f8aeb 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -255,7 +255,6 @@ write_files: content: | [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 8d838337d..aeb8fb52b 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -245,7 +245,6 @@ write_files: content: | [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | From 662594df2538cb3cef0115a482780da240143b3c Mon Sep 17 00:00:00 2001 From: Moritz Bracht <682686+dermorz@users.noreply.github.com> Date: Wed, 16 Nov 2022 09:03:33 +0100 Subject: [PATCH 244/489] Fix e2e tests: Update CentOS image filters & vcenter datastores (#1482) * Update filter for CentOS Linux images Signed-off-by: Moritz Bracht * Change vcenter datastore to ceph Signed-off-by: Moritz Bracht Signed-off-by: Moritz Bracht --- pkg/cloudprovider/provider/aws/provider.go | 4 ++-- .../testdata/machinedeployment-vsphere-static-ip.yaml | 2 +- test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index e72cdcae4..a199954aa 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -104,12 +104,12 @@ var ( // Source: https://wiki.centos.org/Cloud/AWS providerconfigtypes.OperatingSystemCentOS: { awstypes.CPUArchitectureX86_64: { - description: "CentOS 7* x86_64", + description: "CentOS Linux 7* x86_64*", // The AWS marketplace ID from CentOS Community Platform Engineering (CPE) owner: "125523088429", }, awstypes.CPUArchitectureARM64: { - description: "CentOS 7* aarch64", + description: "CentOS Linux 7* aarch64*", // The AWS marketplace ID from CentOS Community Platform Engineering (CPE) owner: "125523088429", }, diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml index a5407d5b7..56ef49c9c 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml @@ -33,7 +33,7 @@ spec: folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically - datastore: alpha1 + datastore: ceph-vm cpus: 2 MemoryMB: 2048 allowInsecure: true diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml index 5f1f969d2..a7906ed6b 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml @@ -33,7 +33,7 @@ spec: folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically - datastore: alpha1 + datastore: ceph-vm cpus: 2 MemoryMB: 4096 diskSizeGB: << DISK_SIZE >> From 57950074e27ac969695287cb61ad722cdf3e6693 Mon Sep 17 00:00:00 2001 From: Helene Durand <90329641+hdurand0710@users.noreply.github.com> Date: Wed, 16 Nov 2022 12:02:51 +0100 Subject: [PATCH 245/489] KubeVirt fix pref wrong error check (#1480) Signed-off-by: Helene Durand Signed-off-by: Helene Durand --- pkg/cloudprovider/provider/kubevirt/provider.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index b84a0154a..483f13fcb 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -276,9 +276,6 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p // Instancetype and Preference config.Instancetype = rawConfig.VirtualMachine.Instancetype config.Preference = rawConfig.VirtualMachine.Preference - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "preference" field: %w`, err) - } dnsPolicyString, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.DNSPolicy) if err != nil { From 099b8e354ea90b841d6078e3ee9fdfdba2de5d34 Mon Sep 17 00:00:00 2001 From: Sankalp Rangare Date: Mon, 21 Nov 2022 09:14:30 +0100 Subject: [PATCH 246/489] add http source for kubevirt diskImage (#1470) Signed-off-by: Sankalp Rangare Signed-off-by: Sankalp Rangare --- .../provider/kubevirt/provider.go | 80 +++++++++++-------- .../provider/kubevirt/provider_test.go | 14 ++++ .../kubevirt/testdata/http-image-source.yaml | 72 +++++++++++++++++ .../kubevirt/testdata/pvc-image-source.yaml | 73 +++++++++++++++++ .../provider/kubevirt/types/types.go | 2 + 5 files changed, 208 insertions(+), 33 deletions(-) create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 483f13fcb..9f5358509 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -60,12 +60,18 @@ func init() { } } +type imageSource string + const ( // topologyKeyHostname defines the topology key for the node hostname. topologyKeyHostname = "kubernetes.io/hostname" // machineDeploymentLabelKey defines the label key used to contains as value the MachineDeployment name // which machine comes from. machineDeploymentLabelKey = "md" + // httpSource defines the http source type for VM Disk Image. + httpSource imageSource = "http" + // pvcSource defines the pvc source type for VM Disk Image. + pvcSource imageSource = "pvc" ) var supportedOS = map[providerconfigtypes.OperatingSystem]*struct{}{ @@ -93,7 +99,7 @@ type Config struct { CPUs string Memory string Namespace string - OsImage OSImage + OSImageSource *cdiv1beta1.DataVolumeSource StorageClassName string PVCSize resource.Quantity FlavorName string @@ -148,11 +154,6 @@ type SecondaryDisks struct { StorageClassName string } -type OSImage struct { - URL string - DataVolumeName string -} - type kubeVirtServer struct { vmi kubevirtv1.VirtualMachineInstance } @@ -247,15 +248,12 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf(`failed to get value of "memory" field: %w`, err) } config.Namespace = getNamespace() - osImage, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.OsImage) + + config.OSImageSource, err = p.parseOSImageSource(rawConfig.VirtualMachine.Template.PrimaryDisk, config.Namespace) if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "sourceURL" field: %w`, err) - } - if _, err = url.ParseRequestURI(osImage); err == nil { - config.OsImage.URL = osImage - } else { - config.OsImage.DataVolumeName = osImage + return nil, nil, fmt.Errorf(`failed to get value of "osImageSource" field: %w`, err) } + pvcSize, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.Size) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "pvcSize" field: %w`, err) @@ -374,6 +372,35 @@ func (p *provider) parseTopologySpreadConstraint(topologyConstraints []kubevirtt return parsedTopologyConstraints, nil } +func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, nameSpace string) (*cdiv1beta1.DataVolumeSource, error) { + osImage, err := p.configVarResolver.GetConfigVarStringValue(primaryDisk.OsImage) + if err != nil { + return nil, fmt.Errorf(`failed to get value of "primaryDisk.osImage" field: %w`, err) + } + osImageSource, err := p.configVarResolver.GetConfigVarStringValue(primaryDisk.Source) + if err != nil { + return nil, fmt.Errorf(`failed to get value of "primaryDisk.source" field: %w`, err) + } + switch imageSource(osImageSource) { + case httpSource: + return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage}}, nil + case pvcSource: + if namespaceAndName := strings.Split(osImage, "/"); len(namespaceAndName) >= 2 { + return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: namespaceAndName[1], Namespace: namespaceAndName[0]}}, nil + } + return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: nameSpace}}, nil + default: + // handle old API for backward compatibility. + if _, err = url.ParseRequestURI(osImage); err == nil { + return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage}}, nil + } + if namespaceAndName := strings.Split(osImage, "/"); len(namespaceAndName) >= 2 { + return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: namespaceAndName[1], Namespace: namespaceAndName[0]}}, nil + } + return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: nameSpace}}, nil + } +} + // getNamespace returns the namespace where the VM is created. // VM is created in a dedicated namespace // which is the namespace where the machine-controller pod is running. @@ -506,7 +533,11 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s if err == nil { labels["cpus"] = c.CPUs labels["memoryMIB"] = c.Memory - labels["osImage"] = c.OsImage.URL + if c.OSImageSource.HTTP != nil { + labels["osImage"] = c.OSImageSource.HTTP.URL + } else if c.OSImageSource.PVC != nil { + labels["osImage"] = c.OSImageSource.PVC.Name + } } return labels, err @@ -787,7 +818,6 @@ func getVMVolumes(config *Config, dataVolumeName string, userDataSecretName stri } func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1.DataVolumeTemplateSpec { - dataVolumeSource := getDataVolumeSource(config.OsImage) pvcRequest := corev1.ResourceList{corev1.ResourceStorage: config.PVCSize} dataVolumeTemplates := []kubevirtv1.DataVolumeTemplateSpec{ { @@ -804,7 +834,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. Requests: pvcRequest, }, }, - Source: dataVolumeSource, + Source: config.OSImageSource, }, }, } @@ -823,29 +853,13 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. Requests: corev1.ResourceList{corev1.ResourceStorage: sd.Size}, }, }, - Source: dataVolumeSource, + Source: config.OSImageSource, }, }) } return dataVolumeTemplates } -// getDataVolumeSource returns DataVolumeSource, HTTP or PVC. -func getDataVolumeSource(osImage OSImage) *cdiv1beta1.DataVolumeSource { - dataVolumeSource := &cdiv1beta1.DataVolumeSource{} - if osImage.URL != "" { - dataVolumeSource.HTTP = &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage.URL} - } else if osImage.DataVolumeName != "" { - if nameSpaceAndName := strings.Split(osImage.DataVolumeName, "/"); len(nameSpaceAndName) >= 2 { - dataVolumeSource.PVC = &cdiv1beta1.DataVolumeSourcePVC{ - Namespace: nameSpaceAndName[0], - Name: nameSpaceAndName[1], - } - } - } - return dataVolumeSource -} - func getAffinity(config *Config, matchKey, matchValue string) *corev1.Affinity { affinity := &corev1.Affinity{} diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index 3af0d314c..38e8933c4 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -103,6 +103,7 @@ type kubevirtProviderSpecConf struct { TopologySpreadConstraint bool Affinity bool SecondaryDisks bool + OsImageSource imageSource } func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { @@ -167,7 +168,12 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { "osImage": "http://x.y.z.t/ubuntu.img", {{- end }} "size": "10Gi", + {{- if .OsImageSource }} + "storageClassName": "longhorn", + "source": "{{ .OsImageSource }}" + {{- else }} "storageClassName": "longhorn" + {{- end }} } } } @@ -267,6 +273,14 @@ func TestNewVirtualMachine(t *testing.T) { name: "custom-local-disk", specConf: kubevirtProviderSpecConf{OsImageDV: "ns/dvname"}, }, + { + name: "http-image-source", + specConf: kubevirtProviderSpecConf{OsImageSource: httpSource}, + }, + { + name: "pvc-image-source", + specConf: kubevirtProviderSpecConf{OsImageSource: pvcSource, OsImageDV: "ns/dvname"}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml new file mode 100644 index 000000000..b488ce0c4 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml @@ -0,0 +1,72 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: http-image-source + md: md-name + name: http-image-source + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: http-image-source + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: http-image-source + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: http-image-source + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml new file mode 100644 index 000000000..809c87ab9 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml @@ -0,0 +1,73 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: pvc-image-source + md: md-name + name: pvc-image-source + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: pvc-image-source + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + pvc: + namespace: ns + name: dvname + running: true + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: pvc-image-source + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: pvc-image-source + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index 365171d08..a0c6d35a6 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -67,6 +67,8 @@ type Template struct { type PrimaryDisk struct { Disk OsImage providerconfigtypes.ConfigVarString `json:"osImage,omitempty"` + // Source describes the VM Disk Image source. + Source providerconfigtypes.ConfigVarString `json:"source,omitempty"` } // SecondaryDisks. From cdf5b1afd99e3297e2f05726f1d940bf4a99e5bd Mon Sep 17 00:00:00 2001 From: Mario Reggiori Date: Mon, 21 Nov 2022 11:29:01 +0100 Subject: [PATCH 247/489] Anexia Provider: Utilize `Creating` state instead of blocking `Create` call (#1483) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mario Schäfer Signed-off-by: Mario Schäfer --- .../provider/anexia/helper_test.go | 23 ------ pkg/cloudprovider/provider/anexia/instance.go | 5 ++ pkg/cloudprovider/provider/anexia/provider.go | 77 +++++++------------ .../provider/anexia/provider_test.go | 32 -------- 4 files changed, 33 insertions(+), 104 deletions(-) diff --git a/pkg/cloudprovider/provider/anexia/helper_test.go b/pkg/cloudprovider/provider/anexia/helper_test.go index 0bcea21f8..38c3a37ef 100644 --- a/pkg/cloudprovider/provider/anexia/helper_test.go +++ b/pkg/cloudprovider/provider/anexia/helper_test.go @@ -18,11 +18,9 @@ package anexia import ( "encoding/json" - "net/http" "testing" "github.com/gophercloud/gophercloud/testhelper" - "go.anx.io/go-anxcloud/pkg/vsphere/search" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" @@ -64,27 +62,6 @@ func getSpecsForValidationTest(t *testing.T, configCases []ConfigTestCase) []Val return testCases } -func createSearchHandler(t *testing.T, iterations int) http.HandlerFunc { - counter := 0 - return func(writer http.ResponseWriter, request *http.Request) { - test := request.URL.Query().Get("name") - testhelper.AssertEquals(t, "%-TestMachine", test) - testhelper.TestMethod(t, request, http.MethodGet) - if iterations == counter { - encoder := json.NewEncoder(writer) - testhelper.AssertNoErr(t, encoder.Encode(map[string]interface{}{ - "data": []search.VM{ - { - Name: "543053-TestMachine", - Identifier: TestIdentifier, - }, - }, - })) - } - counter++ - } -} - func newConfigVarString(str string) types.ConfigVarString { return types.ConfigVarString{ Value: str, diff --git a/pkg/cloudprovider/provider/anexia/instance.go b/pkg/cloudprovider/provider/anexia/instance.go index fa53467a1..cd67d80c5 100644 --- a/pkg/cloudprovider/provider/anexia/instance.go +++ b/pkg/cloudprovider/provider/anexia/instance.go @@ -28,6 +28,7 @@ import ( ) type anexiaInstance struct { + isCreating bool info *info.Info reservedAddresses []string } @@ -85,6 +86,10 @@ func (ai *anexiaInstance) Addresses() map[string]v1.NodeAddressType { } func (ai *anexiaInstance) Status() instance.Status { + if ai.isCreating { + return instance.StatusCreating + } + if ai.info != nil { if ai.info.Status == anxtypes.MachinePoweredOn { return instance.StatusRunning diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 841723724..b89763308 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -23,6 +23,8 @@ import ( "errors" "fmt" "net/http" + "strings" + "sync" "time" anxclient "go.anx.io/go-anxcloud/pkg/client" @@ -45,7 +47,6 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" k8stypes "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog" ) @@ -115,16 +116,6 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, retErr = anxtypes.NewMultiError(retErr, updateMachineStatus(machine, status, data.Update)) }() - // check whether machine is already provisioning - if isAlreadyProvisioning(ctx) && status.ProvisioningID == "" { - klog.Info("ongoing provisioning detected") - err := waitForVM(ctx, client) - if err != nil { - return nil, err - } - return p.Get(ctx, machine, data) - } - // provision machine err = provisionVM(ctx, client) if err != nil { @@ -133,33 +124,6 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return p.Get(ctx, machine, data) } -func waitForVM(ctx context.Context, client anxclient.Client) error { - reconcileContext := getReconcileContext(ctx) - api := vsphere.NewAPI(client) - var identifier string - err := wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) { - klog.V(2).Info("checking for VM with name ", reconcileContext.Machine.Name) - vms, err := api.Search().ByName(ctx, fmt.Sprintf("%%-%s", reconcileContext.Machine.Name)) - if err != nil { - return false, nil - } - if len(vms) < 1 { - return false, nil - } - if len(vms) > 1 { - return false, errors.New("too many VMs returned by search") - } - identifier = vms[0].Identifier - return true, nil - }) - if err != nil { - return err - } - - reconcileContext.Status.InstanceID = identifier - return updateMachineStatus(reconcileContext.Machine, *reconcileContext.Status, reconcileContext.ProviderData.Update) -} - func provisionVM(ctx context.Context, client anxclient.Client) error { reconcileContext := getReconcileContext(ctx) vmAPI := vsphere.NewAPI(client) @@ -229,15 +193,6 @@ func provisionVM(ctx context.Context, client anxclient.Client) error { klog.V(2).Info(fmt.Sprintf("Using provisionID from machine '%s' to await completion", reconcileContext.Machine.Name)) - instanceID, err := vmAPI.Provisioning().Progress().AwaitCompletion(ctx, status.ProvisioningID) - if err != nil { - klog.Errorf("failed to await machine completion '%s'", reconcileContext.Machine.Name) - // something went wrong remove provisioning ID, so we can start from scratch - status.ProvisioningID = "" - return newError(common.CreateMachineError, "instance provisioning failed: %v", err) - } - - status.InstanceID = instanceID meta.SetStatusCondition(&status.Conditions, v1.Condition{ Type: ProvisionedType, Status: v1.ConditionTrue, @@ -248,6 +203,8 @@ func provisionVM(ctx context.Context, client anxclient.Client) error { return updateMachineStatus(reconcileContext.Machine, *status, reconcileContext.ProviderData.Update) } +var _engsup3404mutex sync.Mutex + func getIPAddress(ctx context.Context, client anxclient.Client) (string, error) { reconcileContext := getReconcileContext(ctx) status := reconcileContext.Status @@ -258,6 +215,9 @@ func getIPAddress(ctx context.Context, client anxclient.Client) (string, error) return status.ReservedIP, nil } + _engsup3404mutex.Lock() + defer _engsup3404mutex.Unlock() + klog.Info(fmt.Sprintf("Creating a new IP for machine %q", reconcileContext.Machine.Name)) addrAPI := anxaddr.NewAPI(client) config := reconcileContext.Config @@ -447,7 +407,7 @@ func (p *provider) Validate(_ context.Context, machinespec clusterv1alpha1.Machi return nil } -func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, pd *cloudprovidertypes.ProviderData) (instance.Instance, error) { config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, "failed to retrieve config: %v", err) @@ -464,10 +424,29 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ return nil, newError(common.InvalidConfigurationMachineError, "failed to get machine status: %v", err) } - if status.InstanceID == "" { + if status.InstanceID == "" && status.ProvisioningID == "" { return nil, cloudprovidererrors.ErrInstanceNotFound } + if status.InstanceID == "" { + progress, err := vsphereAPI.Provisioning().Progress().Get(ctx, status.ProvisioningID) + if err != nil { + return nil, fmt.Errorf("failed to get provisioning progress: %w", err) + } + if len(progress.Errors) > 0 { + return nil, fmt.Errorf("vm provisioning had errors: %s", strings.Join(progress.Errors, ",")) + } + if progress.Progress < 100 || progress.VMIdentifier == "" { + return &anexiaInstance{isCreating: true}, nil + } + + status.InstanceID = progress.VMIdentifier + + if err := updateMachineStatus(machine, status, pd.Update); err != nil { + return nil, fmt.Errorf("failed updating machine status: %w", err) + } + } + instance := anexiaInstance{} if status.IPState == anxtypes.IPStateBound && status.ReservedIP != "" { diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index b26d29f30..08cec3257 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -50,38 +50,6 @@ func TestAnexiaProvider(t *testing.T) { server.Close() }) - t.Run("Test waiting for VM", func(t *testing.T) { - t.Parallel() - - waitUntilVMIsFound := 2 - testhelper.Mux.HandleFunc("/api/vsphere/v1/search/by_name.json", createSearchHandler(t, waitUntilVMIsFound)) - - providerStatus := anxtypes.ProviderStatus{} - ctx := createReconcileContext(context.Background(), reconcileContext{ - Machine: &v1alpha1.Machine{ - ObjectMeta: metav1.ObjectMeta{Name: "TestMachine"}, - }, - Status: &providerStatus, - UserData: "", - Config: resolvedConfig{}, - - ProviderData: &cloudprovidertypes.ProviderData{ - Update: func(m *clusterv1alpha1.Machine, mod ...cloudprovidertypes.MachineModifier) error { - return nil - }, - }, - }) - - err := waitForVM(ctx, client) - if err != nil { - t.Fatal("No error was expected", err) - } - - if providerStatus.InstanceID != TestIdentifier { - t.Error("Expected InstanceID to be set") - } - }) - t.Run("Test provision VM", func(t *testing.T) { t.Parallel() testhelper.Mux.HandleFunc("/api/ipam/v1/address/reserve/ip/count.json", func(writer http.ResponseWriter, request *http.Request) { From 40cb0e45bae0aeda90c1b7eedd66c05e331a2d8c Mon Sep 17 00:00:00 2001 From: Helene Durand <90329641+hdurand0710@users.noreply.github.com> Date: Tue, 22 Nov 2022 13:25:17 +0100 Subject: [PATCH 248/489] KubeVirt deprecate Flavor - no migration (#1491) Signed-off-by: Helene Durand Signed-off-by: Helene Durand --- .../provider/kubevirt/provider.go | 27 +------ .../provider/kubevirt/provider_test.go | 75 +------------------ .../provider/kubevirt/testdata/flavor.yaml | 67 ----------------- .../testdata/instancetype-flavor.yaml | 71 ------------------ .../provider/kubevirt/types/types.go | 1 + 5 files changed, 8 insertions(+), 233 deletions(-) delete mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/flavor.yaml delete mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/instancetype-flavor.yaml diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 9f5358509..df73b5aac 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -102,12 +102,9 @@ type Config struct { OSImageSource *cdiv1beta1.DataVolumeSource StorageClassName string PVCSize resource.Quantity - FlavorName string Instancetype *kubevirtv1.InstancetypeMatcher Preference *kubevirtv1.PreferenceMatcher SecondaryDisks []SecondaryDisks - PodAffinityPreset AffinityType - PodAntiAffinityPreset AffinityType NodeAffinityPreset NodeAffinityPreset TopologySpreadConstraints []corev1.TopologySpreadConstraint } @@ -265,11 +262,6 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "storageClassName" field: %w`, err) } - // Keep Flavor during migration. - config.FlavorName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Flavor.Name) - if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "flavor.name" field: %w`, err) - } // Instancetype and Preference config.Instancetype = rawConfig.VirtualMachine.Instancetype @@ -479,9 +471,9 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe if err != nil { return fmt.Errorf("failed to parse config: %w", err) } - // If instancetype is specified (or flavor until deprecation), skip CPU and Memory validation. + // If instancetype is specified, skip CPU and Memory validation. // Values will come from instancetype. - if c.Instancetype == nil && c.FlavorName == "" { + if c.Instancetype == nil { if _, err := parseResources(c.CPUs, c.Memory); err != nil { return err } @@ -606,25 +598,14 @@ func (p *provider) newVirtualMachine(ctx context.Context, c *Config, pc *provide labels[machineDeploymentLabelKey] = mdName } - // Priority to instancetype. - // if no instancetype and no flavor, resources are from config. - if c.Instancetype == nil && c.FlavorName == "" { + // if no instancetype, resources are from config. + if c.Instancetype == nil { requestsAndLimits, err := parseResources(c.CPUs, c.Memory) if err != nil { return nil, err } resourceRequirements.Requests = *requestsAndLimits resourceRequirements.Limits = *requestsAndLimits - } else if c.FlavorName != "" && c.Instancetype == nil { - // if flavor is specified, then take it from flavor (if instancetype is not set!). - // Add VMIPreset label if specified. - vmiPreset := kubevirtv1.VirtualMachineInstancePreset{} - if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: c.FlavorName}, &vmiPreset); err != nil { - return nil, err - } - for key, val := range vmiPreset.Spec.Selector.MatchLabels { - labels[key] = val - } } var ( diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index 38e8933c4..52dc0c5ab 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -32,7 +32,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -47,58 +46,18 @@ var ( vmDir = "testdata" fakeclient ctrlruntimeclient.WithWatch expectedVms map[string]*kubevirtv1.VirtualMachine - flavorName = "to-deprecate-flavor" ) func init() { - presets := []ctrlruntimeclient.Object{getPreset("77", "77Gi", flavorName)} - fakeclient = fakectrlruntimeclient.NewClientBuilder().WithObjects(presets...).Build() + fakeclient = fakectrlruntimeclient.NewClientBuilder().Build() objs := runtimeFromYaml(fakeclient, vmManifestsFS, vmDir) expectedVms = toVirtualMachines(objs) } -func getPreset(cpu, memory, presetName string) *kubevirtv1.VirtualMachineInstancePreset { - cpuQuantity, err := resource.ParseQuantity(cpu) - if err != nil { - return nil - } - memoryQuantity, err := resource.ParseQuantity(memory) - if err != nil { - return nil - } - resourceList := corev1.ResourceList{ - corev1.ResourceMemory: memoryQuantity, - corev1.ResourceCPU: cpuQuantity, - } - - return &kubevirtv1.VirtualMachineInstancePreset{ - TypeMeta: metav1.TypeMeta{ - Kind: kubevirtv1.VirtualMachineInstancePresetGroupVersionKind.Kind, - APIVersion: kubevirtv1.GroupVersion.String(), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: presetName, - Namespace: testNamespace, - }, - Spec: kubevirtv1.VirtualMachineInstancePresetSpec{ - Selector: metav1.LabelSelector{ - MatchLabels: map[string]string{"kubevirt.io/flavor": presetName}, - }, - Domain: &kubevirtv1.DomainSpec{ - Resources: kubevirtv1.ResourceRequirements{ - Requests: resourceList, - Limits: resourceList, - }, - }, - }, - } -} - type kubevirtProviderSpecConf struct { OsImageDV string // if OsImage from DV and not from http source Instancetype *kubevirtv1.InstancetypeMatcher Preference *kubevirtv1.PreferenceMatcher - Flavor string // to remove when Flavor is deprecated OperatingSystem string TopologySpreadConstraint bool Affinity bool @@ -146,11 +105,6 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { "kind": "{{ .Preference.Kind }}" }, {{- end }} - {{- if .Flavor }} - "flavor": { - "name": "{{ .Flavor }}" - }, - {{- end }} "template": { "cpus": "2", "memory": "2Gi", @@ -169,11 +123,9 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { {{- end }} "size": "10Gi", {{- if .OsImageSource }} - "storageClassName": "longhorn", - "source": "{{ .OsImageSource }}" - {{- else }} - "storageClassName": "longhorn" + "source": "{{ .OsImageSource }}", {{- end }} + "storageClassName": "longhorn" } } } @@ -236,27 +188,6 @@ func TestNewVirtualMachine(t *testing.T) { }, }, }, - { - name: "flavor", // to be deprecated when UI is switched to instancetype - specConf: kubevirtProviderSpecConf{ - Flavor: flavorName, - }, - }, - { - name: "instancetype-flavor", // to be deprecated when UI is switched to instancetype, instancetype wins - // no flavor labels - specConf: kubevirtProviderSpecConf{ - Flavor: flavorName, - Instancetype: &kubevirtv1.InstancetypeMatcher{ - Name: "standard-it", - Kind: "VirtualMachineInstancetype", - }, - Preference: &kubevirtv1.PreferenceMatcher{ - Name: "standard-pref", - Kind: "VirtualMachinePreference", - }, - }, - }, { name: "topologyspreadconstraints", specConf: kubevirtProviderSpecConf{TopologySpreadConstraint: true}, diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/flavor.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/flavor.yaml deleted file mode 100644 index d9e666f2a..000000000 --- a/pkg/cloudprovider/provider/kubevirt/testdata/flavor.yaml +++ /dev/null @@ -1,67 +0,0 @@ -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - annotations: - labels: - kubevirt.io/flavor: to-deprecate-flavor - kubevirt.io/vm: flavor - md: md-name - name: flavor - namespace: test-namespace -spec: - dataVolumeTemplates: - - metadata: - name: flavor - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: longhorn - source: - http: - url: http://x.y.z.t/ubuntu.img - running: true - template: - metadata: - creationTimestamp: null - labels: - kubevirt.io/flavor: to-deprecate-flavor - kubevirt.io/vm: flavor - md: md-name - spec: - affinity: {} - domain: - devices: - disks: - - disk: - bus: virtio - name: datavolumedisk - - disk: - bus: virtio - name: cloudinitdisk - interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default - bridge: {} - networks: - - name: default - pod: {} - terminationGracePeriodSeconds: 30 - topologyspreadconstraints: - - maxskew: 1 - topologykey: kubernetes.io/hostname - whenunsatisfiable: ScheduleAnyway - labelselector: - matchlabels: - md: md-name - volumes: - - dataVolume: - name: flavor - name: datavolumedisk - - cloudInitNoCloud: - secretRef: - name: udsn - name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-flavor.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-flavor.yaml deleted file mode 100644 index 2880ba431..000000000 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-flavor.yaml +++ /dev/null @@ -1,71 +0,0 @@ -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - annotations: - labels: - kubevirt.io/vm: instancetype-flavor - md: md-name - name: instancetype-flavor - namespace: test-namespace -spec: - dataVolumeTemplates: - - metadata: - name: instancetype-flavor - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: longhorn - source: - http: - url: http://x.y.z.t/ubuntu.img - running: true - instancetype: - kind: VirtualMachineInstancetype - name: standard-it - preference: - kind: VirtualMachinePreference - name: standard-pref - template: - metadata: - creationTimestamp: null - labels: - kubevirt.io/vm: instancetype-flavor - md: md-name - spec: - affinity: {} - domain: - devices: - disks: - - disk: - bus: virtio - name: datavolumedisk - - disk: - bus: virtio - name: cloudinitdisk - interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default - bridge: {} - networks: - - name: default - pod: {} - terminationGracePeriodSeconds: 30 - topologyspreadconstraints: - - maxskew: 1 - topologykey: kubernetes.io/hostname - whenunsatisfiable: ScheduleAnyway - labelselector: - matchlabels: - md: md-name - volumes: - - dataVolume: - name: instancetype-flavor - name: datavolumedisk - - cloudInitNoCloud: - secretRef: - name: udsn - name: cloudinitdisk diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index a0c6d35a6..72fb645ff 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -39,6 +39,7 @@ type Auth struct { // VirtualMachine. type VirtualMachine struct { + // Deprecated: use Instancetype/Preference instead. Flavor Flavor `json:"flavor,omitempty"` // Instancetype is optional. Instancetype *kubevirtv1.InstancetypeMatcher `json:"instancetype,omitempty"` From 68b11acab339ac83f075715bf94b02c3f87a5a3e Mon Sep 17 00:00:00 2001 From: Helene Durand <90329641+hdurand0710@users.noreply.github.com> Date: Wed, 23 Nov 2022 20:40:02 +0100 Subject: [PATCH 249/489] KubeVirt small cleanup: Flavor and PodAffinity (#1494) Signed-off-by: Helene Durand Signed-off-by: Helene Durand --- examples/kubevirt-machinedeployment.yaml | 8 -------- .../provisioning/testdata/machinedeployment-kubevirt.yaml | 8 -------- 2 files changed, 16 deletions(-) diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index 41fe29c57..86ada4d3d 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -38,9 +38,6 @@ spec: preference: name: "sockets-advantage" kind: "VirtualMachinePreference" # Allowed values: "VirtualMachinePreference"/"VirtualMachineClusterPreference" - # will be deprecated: in favor instancetype and preference - flavor: - name: "kubermatic-standard" template: cpus: "1" memory: "2048M" @@ -49,10 +46,6 @@ spec: size: "10Gi" storageClassName: kubermatic-fast affinity: - # Deprecated: Use topologySpreadConstraints instead. - podAffinityPreset: "" # Allowed values: "", "soft", "hard" - # Deprecated: Use topologySpreadConstraints instead. - podAntiAffinityPreset: "" # Allowed values: "", "soft", "hard" nodeAffinityPreset: type: "" # Allowed values: "", "soft", "hard" key: "foo" @@ -62,7 +55,6 @@ spec: - maxSkew: "1" topologyKey: "kubernetes.io/hostname" whenUnsatisfiable: "" # Allowed values: "DoNotSchedule", "ScheduleAnyway" - # Can also be `centos`, must align with he configured registryImage above operatingSystem: "ubuntu" operatingSystemSpec: diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index 184427fa8..9c3eaab9b 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -43,14 +43,6 @@ spec: dnsConfig: nameservers: - 8.8.8.8 - affinity: - podAffinityPreset: "" # Allowed values: "", "soft", "hard" - podAntiAffinityPreset: "" # Allowed values: "", "soft", "hard" - nodeAffinityPreset: - type: "" # Allowed values: "", "soft", "hard" - key: "foo" - values: - - bar operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false From 798bc637eaa0802187cd8688c086309722efa054 Mon Sep 17 00:00:00 2001 From: Moritz Bracht <682686+dermorz@users.noreply.github.com> Date: Wed, 23 Nov 2022 23:39:47 +0100 Subject: [PATCH 250/489] Add cluster labels to kubevirt vm (#1472) * Add cluster labels to KubeVirt VM Linter: Drop unnecessary else block Signed-off-by: Moritz Bracht Use MC namespace as cluster-name label Signed-off-by: Moritz Bracht * Upgrade kubevirt.io/api to v0.58.0 Signed-off-by: Moritz Bracht * Clean up trailing whitespace Signed-off-by: Moritz Bracht * Adjust test data to new cluster labels Signed-off-by: Moritz Bracht Fix typo Signed-off-by: Moritz Bracht revert rename of test-namespace Signed-off-by: Moritz Bracht * Add ClusterName to KubeVirt providerconfig Signed-off-by: Moritz Bracht * Add documentation of new clusterName field for kubevirt Signed-off-by: Moritz Bracht Signed-off-by: Moritz Bracht --- docs/kubevirt.md | 3 ++- examples/kubevirt-machinedeployment.yaml | 1 + go.mod | 2 +- go.sum | 4 ++-- pkg/cloudprovider/provider/kubevirt/provider.go | 10 ++++++++++ pkg/cloudprovider/provider/kubevirt/provider_test.go | 7 ++++--- .../provider/kubevirt/testdata/affinity.yaml | 4 ++++ .../provider/kubevirt/testdata/custom-local-disk.yaml | 4 ++++ .../provider/kubevirt/testdata/http-image-source.yaml | 4 ++++ .../testdata/instancetype-preference-custom.yaml | 4 ++++ .../testdata/instancetype-preference-standard.yaml | 4 ++++ .../provider/kubevirt/testdata/nominal-case.yaml | 4 ++++ .../provider/kubevirt/testdata/pvc-image-source.yaml | 4 ++++ .../provider/kubevirt/testdata/secondary-disks.yaml | 4 ++++ .../kubevirt/testdata/topologyspreadconstraints.yaml | 4 ++++ pkg/cloudprovider/provider/kubevirt/types/types.go | 9 +++++---- 16 files changed, 61 insertions(+), 11 deletions(-) diff --git a/docs/kubevirt.md b/docs/kubevirt.md index 118663546..2f86ba155 100644 --- a/docs/kubevirt.md +++ b/docs/kubevirt.md @@ -7,13 +7,14 @@ under `examples/cdi-operator.yaml` to provision storage. It is important to have information regarding which types of storage can be used please refer to [KubeVirt documentation](https://github.com/kubevirt/containerized-data-importer/blob/main/doc/basic_pv_pvc_dv.md). -Afterwards, you can use the provided `exampes/examples/kubevirt-machinedeployment.yaml` as base. There +Afterwards, you can use the provided `examples/kubevirt-machinedeployment.yaml` as base. There are some things you need to keep in mind: * The machine-controller will create `VMIs` that have the same name as the underlying `machine`. To avoid collisions, use one namespace per cluster that runs the `machine-controller` * Service CIDR range: The CIDR ranges of the cluster that runs Kubevirt and the cluster that hosts the machine-controller must not overlap, otherwise routing of services that run in the kubevirt cluster won't work anymore. This is especially important for the DNS ClusterIP. +* `clusterName` is used to [label VMs](https://github.com/kubevirt/cloud-provider-kubevirt#prerequisites) for LoadBalancer selection ## Serving Supported Images diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index 86ada4d3d..0f59d2e0a 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -26,6 +26,7 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "kubevirt" cloudProviderSpec: + clusterName: cluster-name auth: kubeconfig: # Can also be set via the env var 'KUBEVIRT_KUBECONFIG' on the machine-controller. diff --git a/go.mod b/go.mod index 2560f8da1..9172d3228 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( k8s.io/klog v1.0.0 k8s.io/kubelet v0.24.2 k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed - kubevirt.io/api v0.57.1 + kubevirt.io/api v0.58.0 kubevirt.io/containerized-data-importer-api v1.55.0 sigs.k8s.io/controller-runtime v0.12.1 sigs.k8s.io/yaml v1.3.0 diff --git a/go.sum b/go.sum index 34adc05c4..b9448c299 100644 --- a/go.sum +++ b/go.sum @@ -1678,8 +1678,8 @@ k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -kubevirt.io/api v0.57.1 h1:z6ImWKCQL2efFYqMWmxEsDNyt8c6mbWk7oCY6ZAa06U= -kubevirt.io/api v0.57.1/go.mod h1:U0CQlZR0JoJCaC+Va0wz4dMOtYDdVywJ98OT1KmOkzI= +kubevirt.io/api v0.58.0 h1:qeNeRtD6AIJ5WVJuRXajmmXtnrO5dYchy+hpCm6QwhE= +kubevirt.io/api v0.58.0/go.mod h1:U0CQlZR0JoJCaC+Va0wz4dMOtYDdVywJ98OT1KmOkzI= kubevirt.io/containerized-data-importer-api v1.55.0 h1:IQNc8PYVq1cTwKNPEJza5xSlcnXeYVNt76M5kZ8X7xo= kubevirt.io/containerized-data-importer-api v1.55.0/go.mod h1:92HiQEyzPoeMiCbgfG5Qe10JQVbtWMZOXucy56dKdGg= kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index df73b5aac..9f28e1f7d 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -93,6 +93,7 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes type Config struct { Kubeconfig string + ClusterName string RestConfig *rest.Config DNSConfig *corev1.PodDNSConfig DNSPolicy corev1.DNSPolicy @@ -231,6 +232,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } } + config.ClusterName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ClusterName) + if err != nil { + return nil, nil, fmt.Errorf(`failed to get value of "clusterName" field: %w`, err) + } + config.RestConfig, err = clientcmd.RESTConfigFromKubeConfig([]byte(config.Kubeconfig)) if err != nil { return nil, nil, fmt.Errorf("failed to decode kubeconfig: %w", err) @@ -608,6 +614,10 @@ func (p *provider) newVirtualMachine(ctx context.Context, c *Config, pc *provide resourceRequirements.Limits = *requestsAndLimits } + // Add cluster labels + labels["cluster.x-k8s.io/cluster-name"] = c.ClusterName + labels["cluster.x-k8s.io/role"] = "worker" + var ( dataVolumeName = machine.Name annotations map[string]string diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index 52dc0c5ab..261ac347e 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -70,6 +70,7 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { tmpl, err := template.New("test").Parse(`{ "cloudProvider": "kubevirt", "cloudProviderSpec": { + "clusterName": "cluster-name", "auth": { "kubeconfig": "eyJhcGlWZXJzaW9uIjoidjEiLCJjbHVzdGVycyI6W3siY2x1c3RlciI6eyJjZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YSI6IiIsInNlcnZlciI6Imh0dHBzOi8vOTUuMjE2LjIwLjE0Njo2NDQzIn0sIm5hbWUiOiJrdWJlcm5ldGVzIn1dLCJjb250ZXh0cyI6W3siY29udGV4dCI6eyJjbHVzdGVyIjoia3ViZXJuZXRlcyIsIm5hbWVzcGFjZSI6Imt1YmUtc3lzdGVtIiwidXNlciI6Imt1YmVybmV0ZXMtYWRtaW4ifSwibmFtZSI6Imt1YmVybmV0ZXMtYWRtaW5Aa3ViZXJuZXRlcyJ9XSwiY3VycmVudC1jb250ZXh0Ijoia3ViZXJuZXRlcy1hZG1pbkBrdWJlcm5ldGVzIiwia2luZCI6IkNvbmZpZyIsInByZWZlcmVuY2VzIjp7fSwidXNlcnMiOlt7Im5hbWUiOiJrdWJlcm5ldGVzLWFkbWluIiwidXNlciI6eyJjbGllbnQtY2VydGlmaWNhdGUtZGF0YSI6IiIsImNsaWVudC1rZXktZGF0YSI6IiJ9fV19" }, @@ -86,20 +87,20 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { "affinity": { "nodeAffinityPreset": { "type": "hard", - "key": "key1", + "key": "key1", "values": [ "foo1", "foo2" ] } }, {{- end }} "virtualMachine": { - {{- if .Instancetype }} + {{- if .Instancetype }} "instancetype": { "name": "{{ .Instancetype.Name }}", "kind": "{{ .Instancetype.Kind }}" }, {{- end }} - {{- if .Preference }} + {{- if .Preference }} "preference": { "name": "{{ .Preference.Name }}", "kind": "{{ .Preference.Kind }}" diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml index 4c1ffe470..fb2392076 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml @@ -3,6 +3,8 @@ kind: VirtualMachine metadata: annotations: labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker kubevirt.io/vm: affinity md: md-name name: affinity @@ -27,6 +29,8 @@ spec: template: metadata: labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker kubevirt.io/vm: affinity md: md-name spec: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml index a84d7c91a..5c1a645ac 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml @@ -3,6 +3,8 @@ kind: VirtualMachine metadata: annotations: labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker kubevirt.io/vm: custom-local-disk md: md-name name: custom-local-disk @@ -28,6 +30,8 @@ spec: metadata: creationTimestamp: null labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker kubevirt.io/vm: custom-local-disk md: md-name spec: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml index b488ce0c4..b350d2053 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml @@ -4,6 +4,8 @@ metadata: annotations: labels: kubevirt.io/vm: http-image-source + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker md: md-name name: http-image-source namespace: test-namespace @@ -28,6 +30,8 @@ spec: creationTimestamp: null labels: kubevirt.io/vm: http-image-source + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker md: md-name spec: affinity: {} diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml index 8d90a8747..d6e7abc67 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml @@ -3,6 +3,8 @@ kind: VirtualMachine metadata: annotations: labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker kubevirt.io/vm: instancetype-preference-custom md: md-name name: instancetype-preference-custom @@ -33,6 +35,8 @@ spec: template: metadata: labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker kubevirt.io/vm: instancetype-preference-custom md: md-name spec: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml index 709de1199..19457b9cd 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml @@ -3,6 +3,8 @@ kind: VirtualMachine metadata: annotations: labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker kubevirt.io/vm: instancetype-preference-standard md: md-name name: instancetype-preference-standard @@ -33,6 +35,8 @@ spec: metadata: creationTimestamp: null labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker kubevirt.io/vm: instancetype-preference-standard md: md-name spec: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml index bdb107a14..f49ddf69c 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml @@ -3,6 +3,8 @@ kind: VirtualMachine metadata: annotations: labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker kubevirt.io/vm: nominal-case md: md-name name: nominal-case @@ -27,6 +29,8 @@ spec: metadata: creationTimestamp: null labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker kubevirt.io/vm: nominal-case md: md-name spec: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml index 809c87ab9..b78a189cf 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml @@ -4,6 +4,8 @@ metadata: annotations: labels: kubevirt.io/vm: pvc-image-source + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker md: md-name name: pvc-image-source namespace: test-namespace @@ -29,6 +31,8 @@ spec: creationTimestamp: null labels: kubevirt.io/vm: pvc-image-source + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker md: md-name spec: affinity: {} diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml index b1137331a..ec2186480 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml @@ -3,6 +3,8 @@ kind: VirtualMachine metadata: annotations: labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker kubevirt.io/vm: secondary-disks md: md-name name: secondary-disks @@ -53,6 +55,8 @@ spec: metadata: creationTimestamp: null labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker kubevirt.io/vm: secondary-disks md: md-name spec: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml index 4f51eeb63..fdfa6fc57 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml @@ -3,6 +3,8 @@ kind: VirtualMachine metadata: annotations: labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker kubevirt.io/vm: topologyspreadconstraints md: md-name name: topologyspreadconstraints @@ -27,6 +29,8 @@ spec: metadata: creationTimestamp: null labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker kubevirt.io/vm: topologyspreadconstraints md: md-name spec: diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index 72fb645ff..427c4eae8 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -26,10 +26,11 @@ import ( ) type RawConfig struct { - Auth Auth `json:"auth,omitempty"` - VirtualMachine VirtualMachine `json:"virtualMachine,omitempty"` - Affinity Affinity `json:"affinity,omitempty"` - TopologySpreadConstraints []TopologySpreadConstraint `json:"topologySpreadConstraints"` + ClusterName providerconfigtypes.ConfigVarString `json:"clusterName"` + Auth Auth `json:"auth,omitempty"` + VirtualMachine VirtualMachine `json:"virtualMachine,omitempty"` + Affinity Affinity `json:"affinity,omitempty"` + TopologySpreadConstraints []TopologySpreadConstraint `json:"topologySpreadConstraints"` } // Auth. From 46f4fa63161f6bed01a39e4e5a5a21930887443c Mon Sep 17 00:00:00 2001 From: Moritz Bracht <682686+dermorz@users.noreply.github.com> Date: Thu, 24 Nov 2022 13:42:45 +0100 Subject: [PATCH 251/489] Set ownership for kubevirt provider to sig-virt (#1496) Signed-off-by: Moritz Bracht Signed-off-by: Moritz Bracht --- OWNERS_ALIASES | 6 ++++++ pkg/cloudprovider/provider/kubevirt/OWNERS | 13 +++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 pkg/cloudprovider/provider/kubevirt/OWNERS diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 1d4d78a34..d833f9f6a 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -14,6 +14,12 @@ aliases: - xmudrii - xrstf + sig-virtualization: + - dermorz + - hdurand0710 + - mfranczy + - sankalp-r + # Temporary SIG to oversee changes in userdata and cloudprovider sub-directories # This SIG is responsible for ensuring that OSM and machine-controller are in sync sig-osm: diff --git a/pkg/cloudprovider/provider/kubevirt/OWNERS b/pkg/cloudprovider/provider/kubevirt/OWNERS new file mode 100644 index 000000000..ac28b34e2 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/OWNERS @@ -0,0 +1,13 @@ +# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md + +approvers: + - sig-virtualization + +reviewers: + - sig-virtualization + +labels: + - sig/virtualization + +options: + no_parent_owners: true From afd191ed6a90a443ee5798d58006d2960925d88d Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 30 Nov 2022 14:17:05 +0500 Subject: [PATCH 252/489] Update trigger condition for cloud provider specific E2E jobs (#1499) Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- .prow/provider-aws.yaml | 8 ++++---- .prow/provider-azure.yaml | 2 +- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-vsphere.yaml | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 8d5906f80..f325d0cf0 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -75,7 +75,7 @@ presubmits: memory: 7Gi - name: pull-machine-controller-e2e-aws-arm - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -103,7 +103,7 @@ presubmits: memory: 7Gi - name: pull-machine-controller-e2e-aws-ebs-encryption-enabled - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -131,7 +131,7 @@ presubmits: memory: 7Gi - name: pull-machine-controller-e2e-aws-flatcar-containerd - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -159,7 +159,7 @@ presubmits: memory: 7Gi - name: pull-machine-controller-e2e-aws-spot-instance - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index d0fe23112..5377f5656 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -43,7 +43,7 @@ presubmits: memory: 7Gi - name: pull-machine-controller-e2e-azure-custom-image-reference - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/azure/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index cb6984b8a..23a7d89a8 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-digitalocean - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/digitalocean/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 2fe932093..d2cb8d2ca 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -15,7 +15,7 @@ presubmits: - name: pull-machine-controller-e2e-equinix-metal optional: true - run_if_changed: pkg\/cloudprovider\/provider\/equinixmetal\/.* + run_if_changed: "(pkg/cloudprovider/provider/equinixmetal/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index fe4aa3c4a..ec19ebd9a 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-gce - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/gce/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 54063d31f..950a799ca 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-hetzner - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/hetzner/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 43c8ab7f8..f12951aa4 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-kubevirt - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/kubevirt/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" max_concurrency: 1 diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 5076ab630..d112d4464 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-openstack - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -43,7 +43,7 @@ presubmits: memory: 7Gi - name: pull-machine-controller-e2e-openstack-project-auth - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 9ce2522f6..87c182bd8 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-vsphere - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/vsphere/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: From a53f05c95b7a0f521d913737a184620040768ed0 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 30 Nov 2022 15:26:07 +0500 Subject: [PATCH 253/489] Update OSM deployment (#1498) Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- examples/operating-system-manager.yaml | 1685 +++++++++++------------- 1 file changed, 805 insertions(+), 880 deletions(-) diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index c025fa24b..545070324 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.9.2 creationTimestamp: null name: operatingsystemconfigs.operatingsystemmanager.k8c.io spec: @@ -14,399 +14,358 @@ spec: listKind: OperatingSystemConfigList plural: operatingsystemconfigs shortNames: - - osc + - osc singular: operatingsystemconfig scope: Namespaced versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: OperatingSystemConfig is the object that represents the OperatingSystemConfig - properties: - apiVersion: - description: - "APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" - type: string - kind: - description: - "Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" - type: string - metadata: - type: object - spec: - description: - OperatingSystemConfigSpec represents the operating system - configuration spec. - properties: - bootstrapConfig: - description: - BootstrapConfig is used for initial configuration of - machine and to fetch the kubernetes secret that contains the provisioning - config. - properties: - files: - description: - Files is a list of files that should exist in the - instance - items: - description: - File is a file that should get written to the host's - file system. The content can either be inlined or referenced - from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. - properties: - inline: - description: - Inline is a struct that contains information - about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: - Encoding is the file's encoding (e.g. - base64). - type: string - required: - - data - type: object - type: object - path: - description: - Path is the path of the file system where the - file should get written to. - type: string - permissions: - default: 644 - description: - Permissions describes with which permissions - the file should get written to the file system. Should - be in decimal base and without any leading zeroes. - format: int32 - type: integer - required: - - content - - path - type: object - type: array - modules: - description: - CloudInitModules contains the supported cloud-init - modules + - name: v1alpha1 + schema: + openAPIV3Schema: + description: OperatingSystemConfig is the object that represents the OperatingSystemConfig + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OperatingSystemConfigSpec represents the operating system + configuration spec. + properties: + bootstrapConfig: + description: BootstrapConfig is used for initial configuration of + machine and to fetch the kubernetes secret that contains the provisioning + config. + properties: + files: + description: Files is a list of files that should exist in the + instance + items: + description: File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. properties: - bootcmd: - description: - BootCMD module runs arbitrary commands very early - in the boot process, only slightly after a boothook would - run. - items: - type: string - type: array - rh_subscription: - additionalProperties: - type: string - description: - RHSubscription registers a Red Hat system either - by username and password or activation and org - type: object - runcmd: - description: - RunCMD Run arbitrary commands at a rc.local like - level with output to the console. - items: - type: string - type: array - yum_repo_dir: - description: - "YumRepoDir the repo parts directory where individual - yum repo config files will be written. Default: /etc/yum.repos.d" - type: string - yum_repos: - additionalProperties: - additionalProperties: - type: string - type: object - description: - YumRepos adds yum repository configuration to - the system. - type: object - type: object - units: - description: - Units a list of the systemd unit files which will - run on the instance - items: - description: - Unit is a systemd unit used for the operating system - config. - properties: - content: - description: Content is the unit's content. - type: string - dropIns: - description: DropIns is a list of drop-ins for this unit. - items: - description: - DropIn is a drop-in configuration for a systemd - unit. + content: + description: Content describe the file's content. + properties: + inline: + description: Inline is a struct that contains information + about the inlined data. properties: - content: - description: Content is the content of the drop-in. + data: + description: Data is the file's data. type: string - name: - description: Name is the name of the drop-in. + encoding: + description: Encoding is the file's encoding (e.g. + base64). type: string required: - - content - - name + - data type: object - type: array - enable: - description: - Enable describes whether the unit is enabled - or not. - type: boolean - mask: - description: - Mask describes whether the unit is masked or - not. - type: boolean - name: - description: Name is the name of a unit. - type: string - required: - - name - type: object - type: array - userSSHKeys: - description: UserSSHKeys is a list of attached user ssh keys - items: - type: string - type: array - type: object - cloudProvider: - description: - CloudProvider represent the cloud provider that support - the given operating system version - properties: - name: - description: Name represents the name of the supported cloud provider - enum: - - aws - - azure - - digitalocean - - gce - - hetzner - - kubevirt - - linode - - nutanix - - openstack - - equinixmetal - - vsphere - - fake - - alibaba - - anexia - - scaleway - - baremetal - - external - - vmware-cloud-director - type: string - spec: - description: - Spec represents the os/image reference in the supported - cloud provider + type: object + path: + description: Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path type: object - x-kubernetes-preserve-unknown-fields: true - required: - - name - type: object - osName: - description: "OSType represent the operating system name e.g: ubuntu" - enum: - - flatcar - - rhel - - centos - - ubuntu - - sles - - amzn2 - - rockylinux - type: string - osVersion: - description: OSVersion the version of the operating system - type: string - provisioningConfig: - description: - ProvisioningConfig is used for provisioning the worker - node. - properties: - files: - description: - Files is a list of files that should exist in the - instance - items: - description: - File is a file that should get written to the host's - file system. The content can either be inlined or referenced - from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. - properties: - inline: - description: - Inline is a struct that contains information - about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: - Encoding is the file's encoding (e.g. - base64). - type: string - required: - - data - type: object - type: object - path: - description: - Path is the path of the file system where the - file should get written to. - type: string - permissions: - default: 644 - description: - Permissions describes with which permissions - the file should get written to the file system. Should - be in decimal base and without any leading zeroes. - format: int32 - type: integer - required: - - content - - path + type: array + modules: + description: CloudInitModules contains the supported cloud-init + modules + properties: + bootcmd: + description: BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string + type: array + rh_subscription: + additionalProperties: + type: string + description: RHSubscription registers a Red Hat system either + by username and password or activation and org type: object - type: array - modules: - description: - CloudInitModules contains the supported cloud-init - modules - properties: - bootcmd: - description: - BootCMD module runs arbitrary commands very early - in the boot process, only slightly after a boothook would - run. - items: - type: string - type: array - rh_subscription: + runcmd: + description: RunCMD Run arbitrary commands at a rc.local like + level with output to the console. + items: + type: string + type: array + yum_repo_dir: + description: 'YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d' + type: string + yum_repos: + additionalProperties: additionalProperties: type: string - description: - RHSubscription registers a Red Hat system either - by username and password or activation and org type: object - runcmd: - description: - RunCMD Run arbitrary commands at a rc.local like - level with output to the console. + description: YumRepos adds yum repository configuration to + the system. + type: object + type: object + units: + description: Units a list of the systemd unit files which will + run on the instance + items: + description: Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. items: - type: string + description: DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object type: array - yum_repo_dir: - description: - "YumRepoDir the repo parts directory where individual - yum repo config files will be written. Default: /etc/yum.repos.d" + enable: + description: Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. type: string - yum_repos: - additionalProperties: - additionalProperties: - type: string - type: object - description: - YumRepos adds yum repository configuration to - the system. - type: object + required: + - name type: object - units: - description: - Units a list of the systemd unit files which will - run on the instance - items: - description: - Unit is a systemd unit used for the operating system - config. - properties: - content: - description: Content is the unit's content. - type: string - dropIns: - description: DropIns is a list of drop-ins for this unit. - items: - description: - DropIn is a drop-in configuration for a systemd - unit. + type: array + userSSHKeys: + description: UserSSHKeys is a list of attached user ssh keys + items: + type: string + type: array + type: object + cloudProvider: + description: CloudProvider represent the cloud provider that support + the given operating system version + properties: + name: + description: Name represents the name of the supported cloud provider + enum: + - aws + - azure + - digitalocean + - gce + - hetzner + - kubevirt + - linode + - nutanix + - openstack + - equinixmetal + - vsphere + - fake + - alibaba + - anexia + - scaleway + - baremetal + - external + - vmware-cloud-director + type: string + spec: + description: Spec represents the os/image reference in the supported + cloud provider + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - name + type: object + osName: + description: 'OSType represent the operating system name e.g: ubuntu' + enum: + - flatcar + - rhel + - centos + - ubuntu + - sles + - amzn2 + - rockylinux + type: string + osVersion: + description: OSVersion the version of the operating system + type: string + provisioningConfig: + description: ProvisioningConfig is used for provisioning the worker + node. + properties: + files: + description: Files is a list of files that should exist in the + instance + items: + description: File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: Inline is a struct that contains information + about the inlined data. properties: - content: - description: Content is the content of the drop-in. + data: + description: Data is the file's data. type: string - name: - description: Name is the name of the drop-in. + encoding: + description: Encoding is the file's encoding (e.g. + base64). type: string required: - - content - - name + - data type: object - type: array - enable: - description: - Enable describes whether the unit is enabled - or not. - type: boolean - mask: - description: - Mask describes whether the unit is masked or - not. - type: boolean - name: - description: Name is the name of a unit. - type: string - required: - - name + type: object + path: + description: Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + modules: + description: CloudInitModules contains the supported cloud-init + modules + properties: + bootcmd: + description: BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string + type: array + rh_subscription: + additionalProperties: + type: string + description: RHSubscription registers a Red Hat system either + by username and password or activation and org type: object - type: array - userSSHKeys: - description: UserSSHKeys is a list of attached user ssh keys - items: + runcmd: + description: RunCMD Run arbitrary commands at a rc.local like + level with output to the console. + items: + type: string + type: array + yum_repo_dir: + description: 'YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d' type: string - type: array - type: object - required: - - bootstrapConfig - - cloudProvider - - osName - - osVersion - - provisioningConfig - type: object - required: - - spec - type: object - served: true - storage: true + yum_repos: + additionalProperties: + additionalProperties: + type: string + type: object + description: YumRepos adds yum repository configuration to + the system. + type: object + type: object + units: + description: Units a list of the systemd unit files which will + run on the instance + items: + description: Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. + items: + description: DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object + type: array + enable: + description: Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. + type: string + required: + - name + type: object + type: array + userSSHKeys: + description: UserSSHKeys is a list of attached user ssh keys + items: + type: string + type: array + type: object + required: + - bootstrapConfig + - cloudProvider + - osName + - osVersion + - provisioningConfig + type: object + required: + - spec + type: object + served: true + storage: true --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.9.2 creationTimestamp: null name: operatingsystemprofiles.operatingsystemmanager.k8c.io spec: @@ -416,557 +375,496 @@ spec: listKind: OperatingSystemProfileList plural: operatingsystemprofiles shortNames: - - osp + - osp singular: operatingsystemprofile scope: Namespaced versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: OperatingSystemProfile is the object that represents the OperatingSystemProfile - properties: - apiVersion: - description: - "APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" - type: string - kind: - description: - "Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" - type: string - metadata: - type: object - spec: - description: - OperatingSystemProfileSpec represents the operating system - configuration spec. - properties: - bootstrapConfig: - description: - BootstrapConfig is used for initial configuration of - machine and to fetch the kubernetes secret that contains the provisioning - config. - properties: - files: - description: - Files is a list of files that should exist in the - instance - items: - description: - File is a file that should get written to the host's - file system. The content can either be inlined or referenced - from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. - properties: - inline: - description: - Inline is a struct that contains information - about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: - Encoding is the file's encoding (e.g. - base64). - type: string - required: - - data - type: object - type: object - path: - description: - Path is the path of the file system where the - file should get written to. + - name: v1alpha1 + schema: + openAPIV3Schema: + description: OperatingSystemProfile is the object that represents the OperatingSystemProfile + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OperatingSystemProfileSpec represents the operating system + configuration spec. + properties: + bootstrapConfig: + description: BootstrapConfig is used for initial configuration of + machine and to fetch the kubernetes secret that contains the provisioning + config. + properties: + files: + description: Files is a list of files that should exist in the + instance + items: + description: File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: Inline is a struct that contains information + about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: Encoding is the file's encoding (e.g. + base64). + type: string + required: + - data + type: object + type: object + path: + description: Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + modules: + description: CloudInitModules field contains the optional cloud-init + modules which are supported by OSM + properties: + bootcmd: + description: BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string + type: array + rh_subscription: + additionalProperties: + type: string + description: RHSubscription registers a Red Hat system either + by username and password or activation and org + type: object + runcmd: + description: RunCMD Run arbitrary commands at a rc.local like + level with output to the console. + items: + type: string + type: array + yum_repo_dir: + description: 'YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d' + type: string + yum_repos: + additionalProperties: + additionalProperties: type: string - permissions: - default: 644 - description: - Permissions describes with which permissions - the file should get written to the file system. Should - be in decimal base and without any leading zeroes. - format: int32 - type: integer - required: - - content - - path + type: object + description: YumRepos adds yum repository configuration to + the system. type: object - type: array - modules: - description: - CloudInitModules field contains the optional cloud-init - modules which are supported by OSM + type: object + supportedContainerRuntimes: + description: SupportedContainerRuntimes represents the container + runtimes supported by the given OS + items: + description: ContainerRuntimeSpec aggregates information about + a specific container runtime properties: - bootcmd: - description: - BootCMD module runs arbitrary commands very early - in the boot process, only slightly after a boothook would - run. + files: + description: Files to add to the main files list when the + containerRuntime is selected items: - type: string + description: File is a file that should get written to + the host's file system. The content can either be inlined + or referenced from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: Inline is a struct that contains + information about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: Encoding is the file's encoding + (e.g. base64). + type: string + required: + - data + type: object + type: object + path: + description: Path is the path of the file system where + the file should get written to. + type: string + permissions: + default: 644 + description: Permissions describes with which permissions + the file should get written to the file system. + Should be in decimal base and without any leading + zeroes. + format: int32 + type: integer + required: + - content + - path + type: object type: array - rh_subscription: + name: + description: Name of the Container runtime + enum: + - docker + - containerd + type: string + templates: additionalProperties: type: string - description: - RHSubscription registers a Red Hat system either - by username and password or activation and org + description: Templates to add to the available templates + when the containerRuntime is selected type: object - runcmd: - description: - RunCMD Run arbitrary commands at a rc.local like - level with output to the console. + required: + - files + - name + type: object + type: array + templates: + additionalProperties: + type: string + description: Templates to be included in units and files + type: object + units: + description: Units a list of the systemd unit files which will + run on the instance + items: + description: Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. items: - type: string + description: DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object type: array - yum_repo_dir: - description: - "YumRepoDir the repo parts directory where individual - yum repo config files will be written. Default: /etc/yum.repos.d" + enable: + description: Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. type: string - yum_repos: - additionalProperties: - additionalProperties: - type: string - type: object - description: - YumRepos adds yum repository configuration to - the system. - type: object + required: + - name type: object - supportedContainerRuntimes: - description: - SupportedContainerRuntimes represents the container - runtimes supported by the given OS - items: - description: - ContainerRuntimeSpec aggregates information about - a specific container runtime - properties: - files: - description: - Files to add to the main files list when the - containerRuntime is selected - items: - description: - File is a file that should get written to - the host's file system. The content can either be inlined - or referenced from a secret in the same namespace. + type: array + type: object + osName: + description: 'OSType represent the operating system name e.g: ubuntu' + enum: + - flatcar + - rhel + - centos + - ubuntu + - sles + - amzn2 + - rockylinux + type: string + osVersion: + description: OSVersion the version of the operating system + type: string + provisioningConfig: + description: ProvisioningConfig is used for provisioning the worker + node. + properties: + files: + description: Files is a list of files that should exist in the + instance + items: + description: File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: Inline is a struct that contains information + about the inlined data. properties: - content: - description: Content describe the file's content. - properties: - inline: - description: - Inline is a struct that contains - information about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: - Encoding is the file's encoding - (e.g. base64). - type: string - required: - - data - type: object - type: object - path: - description: - Path is the path of the file system where - the file should get written to. + data: + description: Data is the file's data. + type: string + encoding: + description: Encoding is the file's encoding (e.g. + base64). type: string - permissions: - default: 644 - description: - Permissions describes with which permissions - the file should get written to the file system. - Should be in decimal base and without any leading - zeroes. - format: int32 - type: integer required: - - content - - path + - data type: object - type: array - name: - description: Name of the Container runtime - enum: - - docker - - containerd - type: string - templates: - additionalProperties: - type: string - description: - Templates to add to the available templates - when the containerRuntime is selected - type: object - required: - - files - - name + type: object + path: + description: Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + modules: + description: CloudInitModules field contains the optional cloud-init + modules which are supported by OSM + properties: + bootcmd: + description: BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string + type: array + rh_subscription: + additionalProperties: + type: string + description: RHSubscription registers a Red Hat system either + by username and password or activation and org type: object - type: array - templates: - additionalProperties: + runcmd: + description: RunCMD Run arbitrary commands at a rc.local like + level with output to the console. + items: + type: string + type: array + yum_repo_dir: + description: 'YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d' type: string - description: Templates to be included in units and files - type: object - units: - description: - Units a list of the systemd unit files which will - run on the instance - items: - description: - Unit is a systemd unit used for the operating system - config. - properties: - content: - description: Content is the unit's content. - type: string - dropIns: - description: DropIns is a list of drop-ins for this unit. - items: - description: - DropIn is a drop-in configuration for a systemd - unit. - properties: - content: - description: Content is the content of the drop-in. - type: string - name: - description: Name is the name of the drop-in. - type: string - required: - - content - - name - type: object - type: array - enable: - description: - Enable describes whether the unit is enabled - or not. - type: boolean - mask: - description: - Mask describes whether the unit is masked or - not. - type: boolean - name: - description: Name is the name of a unit. + yum_repos: + additionalProperties: + additionalProperties: type: string - required: - - name + type: object + description: YumRepos adds yum repository configuration to + the system. type: object - type: array - type: object - osName: - description: "OSType represent the operating system name e.g: ubuntu" - enum: - - flatcar - - rhel - - centos - - ubuntu - - sles - - amzn2 - - rockylinux - type: string - osVersion: - description: OSVersion the version of the operating system - type: string - provisioningConfig: - description: - ProvisioningConfig is used for provisioning the worker - node. - properties: - files: - description: - Files is a list of files that should exist in the - instance - items: - description: - File is a file that should get written to the host's - file system. The content can either be inlined or referenced - from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. + type: object + supportedContainerRuntimes: + description: SupportedContainerRuntimes represents the container + runtimes supported by the given OS + items: + description: ContainerRuntimeSpec aggregates information about + a specific container runtime + properties: + files: + description: Files to add to the main files list when the + containerRuntime is selected + items: + description: File is a file that should get written to + the host's file system. The content can either be inlined + or referenced from a secret in the same namespace. properties: - inline: - description: - Inline is a struct that contains information - about the inlined data. + content: + description: Content describe the file's content. properties: - data: - description: Data is the file's data. - type: string - encoding: - description: - Encoding is the file's encoding (e.g. - base64). - type: string - required: - - data + inline: + description: Inline is a struct that contains + information about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: Encoding is the file's encoding + (e.g. base64). + type: string + required: + - data + type: object type: object + path: + description: Path is the path of the file system where + the file should get written to. + type: string + permissions: + default: 644 + description: Permissions describes with which permissions + the file should get written to the file system. + Should be in decimal base and without any leading + zeroes. + format: int32 + type: integer + required: + - content + - path type: object - path: - description: - Path is the path of the file system where the - file should get written to. - type: string - permissions: - default: 644 - description: - Permissions describes with which permissions - the file should get written to the file system. Should - be in decimal base and without any leading zeroes. - format: int32 - type: integer - required: - - content - - path - type: object - type: array - modules: - description: - CloudInitModules field contains the optional cloud-init - modules which are supported by OSM - properties: - bootcmd: - description: - BootCMD module runs arbitrary commands very early - in the boot process, only slightly after a boothook would - run. - items: - type: string type: array - rh_subscription: + name: + description: Name of the Container runtime + enum: + - docker + - containerd + type: string + templates: additionalProperties: type: string - description: - RHSubscription registers a Red Hat system either - by username and password or activation and org + description: Templates to add to the available templates + when the containerRuntime is selected type: object - runcmd: - description: - RunCMD Run arbitrary commands at a rc.local like - level with output to the console. + required: + - files + - name + type: object + type: array + templates: + additionalProperties: + type: string + description: Templates to be included in units and files + type: object + units: + description: Units a list of the systemd unit files which will + run on the instance + items: + description: Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. items: - type: string + description: DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object type: array - yum_repo_dir: - description: - "YumRepoDir the repo parts directory where individual - yum repo config files will be written. Default: /etc/yum.repos.d" + enable: + description: Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. type: string - yum_repos: - additionalProperties: - additionalProperties: - type: string - type: object - description: - YumRepos adds yum repository configuration to - the system. - type: object + required: + - name type: object - supportedContainerRuntimes: - description: - SupportedContainerRuntimes represents the container - runtimes supported by the given OS - items: - description: - ContainerRuntimeSpec aggregates information about - a specific container runtime - properties: - files: - description: - Files to add to the main files list when the - containerRuntime is selected - items: - description: - File is a file that should get written to - the host's file system. The content can either be inlined - or referenced from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. - properties: - inline: - description: - Inline is a struct that contains - information about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: - Encoding is the file's encoding - (e.g. base64). - type: string - required: - - data - type: object - type: object - path: - description: - Path is the path of the file system where - the file should get written to. - type: string - permissions: - default: 644 - description: - Permissions describes with which permissions - the file should get written to the file system. - Should be in decimal base and without any leading - zeroes. - format: int32 - type: integer - required: - - content - - path - type: object - type: array - name: - description: Name of the Container runtime - enum: - - docker - - containerd - type: string - templates: - additionalProperties: - type: string - description: - Templates to add to the available templates - when the containerRuntime is selected - type: object - required: - - files - - name - type: object - type: array - templates: - additionalProperties: - type: string - description: Templates to be included in units and files + type: array + type: object + supportedCloudProviders: + description: SupportedCloudProviders represent the cloud providers + that support the given operating system version + items: + description: CloudProviderSpec contains the os/image reference for + a specific supported cloud provider + properties: + name: + description: Name represents the name of the supported cloud + provider + enum: + - aws + - azure + - digitalocean + - gce + - hetzner + - kubevirt + - linode + - nutanix + - openstack + - equinixmetal + - vsphere + - fake + - alibaba + - anexia + - scaleway + - baremetal + - external + - vmware-cloud-director + type: string + spec: + description: Spec represents the os/image reference in the supported + cloud provider type: object - units: - description: - Units a list of the systemd unit files which will - run on the instance - items: - description: - Unit is a systemd unit used for the operating system - config. - properties: - content: - description: Content is the unit's content. - type: string - dropIns: - description: DropIns is a list of drop-ins for this unit. - items: - description: - DropIn is a drop-in configuration for a systemd - unit. - properties: - content: - description: Content is the content of the drop-in. - type: string - name: - description: Name is the name of the drop-in. - type: string - required: - - content - - name - type: object - type: array - enable: - description: - Enable describes whether the unit is enabled - or not. - type: boolean - mask: - description: - Mask describes whether the unit is masked or - not. - type: boolean - name: - description: Name is the name of a unit. - type: string - required: - - name - type: object - type: array + x-kubernetes-preserve-unknown-fields: true + required: + - name type: object - supportedCloudProviders: - description: - SupportedCloudProviders represent the cloud providers - that support the given operating system version - items: - description: - CloudProviderSpec contains the os/image reference for - a specific supported cloud provider - properties: - name: - description: - Name represents the name of the supported cloud - provider - enum: - - aws - - azure - - digitalocean - - gce - - hetzner - - kubevirt - - linode - - nutanix - - openstack - - equinixmetal - - vsphere - - fake - - alibaba - - anexia - - scaleway - - baremetal - - external - - vmware-cloud-director - type: string - spec: - description: - Spec represents the os/image reference in the supported - cloud provider - type: object - x-kubernetes-preserve-unknown-fields: true - required: - - name - type: object - type: array - version: - description: Version is the version of the operating System Profile - pattern: v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$ - type: string - required: - - bootstrapConfig - - osName - - osVersion - - provisioningConfig - - supportedCloudProviders - - version - type: object - required: - - spec - type: object - served: true - storage: true + type: array + version: + description: Version is the version of the operating System Profile + pattern: v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$ + type: string + required: + - bootstrapConfig + - osName + - osVersion + - provisioningConfig + - supportedCloudProviders + - version + type: object + required: + - spec + type: object + served: true + storage: true --- apiVersion: cert-manager.io/v1 kind: Issuer @@ -1077,7 +975,7 @@ spec: serviceAccountName: operating-system-manager-webhook containers: # TODO: Update this to a semver tag before release. - - image: quay.io/kubermatic/operating-system-manager:ea2250874bbcea46ad956cbba8972be7ffdf9ce2 + - image: quay.io/kubermatic/operating-system-manager:192412d78cbfb9d826fbc7cf7e077cfe7629d6ba imagePullPolicy: IfNotPresent name: webhook command: @@ -1119,6 +1017,33 @@ spec: secretName: webhook-server-cert --- apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: operatingsystemmanager.k8c.io + annotations: + cert-manager.io/inject-ca-from: kube-system/operating-system-manager-serving-cert +webhooks: + - name: machinedeployments.cluster.k8s.io + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: ["v1", "v1beta1"] + rules: + - apiGroups: + - "cluster.k8s.io" + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - machinedeployments + clientConfig: + service: + namespace: kube-system + name: operating-system-manager-webhook + path: /mutate-v1alpha1-machinedeployment +--- +apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: operatingsystemmanager.k8c.io @@ -1379,7 +1304,7 @@ spec: serviceAccountName: operating-system-manager containers: # TODO: Update this to a semver tag before release. - - image: quay.io/kubermatic/operating-system-manager:ea2250874bbcea46ad956cbba8972be7ffdf9ce2 + - image: quay.io/kubermatic/operating-system-manager:192412d78cbfb9d826fbc7cf7e077cfe7629d6ba imagePullPolicy: IfNotPresent name: operating-system-manager command: From 08403974f6011af0ce00e9b9ce5a2d297f4cb4cc Mon Sep 17 00:00:00 2001 From: Pratik Deoghare Date: Thu, 1 Dec 2022 13:25:19 +0100 Subject: [PATCH 254/489] Support for IPv6+IPv4 IP Family (#1493) * add IPv6+IPv4 ip family type * adjust code for IPv6+IPv4 * rebase --- pkg/cloudprovider/provider/aws/provider.go | 6 +- .../provider/azure/create_delete_resources.go | 2 +- pkg/cloudprovider/provider/azure/provider.go | 16 +- .../provider/digitalocean/provider.go | 8 +- pkg/cloudprovider/provider/gce/provider.go | 6 +- pkg/cloudprovider/provider/gce/service.go | 4 +- pkg/cloudprovider/util/net.go | 21 +- pkg/providerconfig/types/types.go | 2 +- pkg/userdata/helper/helper.go | 15 +- pkg/userdata/helper/kubelet.go | 2 +- pkg/userdata/ubuntu/provider_test.go | 60 ++- .../digitalocean-dualstack-IPv6+IPv4.yaml | 456 ++++++++++++++++++ .../openstack-dualstack-IPv6+IPv4.yaml | 456 ++++++++++++++++++ 13 files changed, 1021 insertions(+), 33 deletions(-) create mode 100644 pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml create mode 100644 pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index a199954aa..9d1472868 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -595,9 +595,9 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe } switch f := pc.Network.GetIPFamily(); f { - case util.Unspecified, util.IPv4: + case util.IPFamilyUnspecified, util.IPFamilyIPv4: // noop - case util.IPv6, util.DualStack: + case util.IPFamilyIPv6, util.IPFamilyIPv4IPv6, util.IPFamilyIPv6IPv4: if len(vpc.Ipv6CidrBlockAssociationSet) == 0 { return fmt.Errorf("vpc %s does not have IPv6 CIDR block", pointer.StringDeref(vpc.VpcId, "")) } @@ -792,7 +792,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, }, } - if pc.Network.GetIPFamily() == util.IPv6 || pc.Network.GetIPFamily() == util.DualStack { + if pc.Network.GetIPFamily().HasIPv6() { instanceRequest.NetworkInterfaces[0].Ipv6AddressCount = aws.Int32(1) } diff --git a/pkg/cloudprovider/provider/azure/create_delete_resources.go b/pkg/cloudprovider/provider/azure/create_delete_resources.go index 68931e11a..b751736a1 100644 --- a/pkg/cloudprovider/provider/azure/create_delete_resources.go +++ b/pkg/cloudprovider/provider/azure/create_delete_resources.go @@ -349,7 +349,7 @@ func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineU }, }) - if ipFamily == util.DualStack { + if ipFamily.IsDualstack() { *ifSpec.InterfacePropertiesFormat.IPConfigurations = append(*ifSpec.InterfacePropertiesFormat.IPConfigurations, network.InterfaceIPConfiguration{ Name: to.StringPtr("ip-config-2"), InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index f30e8ad5b..e5f5838ff 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -461,7 +461,7 @@ func getNICIPAddresses(ctx context.Context, c *config, ipFamily util.IPFamily, i ipAddresses[ip] = v1.NodeExternalIP } - if ipFamily == util.DualStack || ipFamily == util.IPv6 { + if ipFamily.HasIPv6() { publicIP6s, err := getIPAddressStrings(ctx, c, publicIPv6Name(ifaceName)) if err != nil { return nil, fmt.Errorf("failed to retrieve IP string for IP %q: %w", name, err) @@ -598,7 +598,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, if config.PublicIPSKU != nil { sku = *config.PublicIPSKU - } else if ipFamily == util.DualStack { + } else if ipFamily.IsDualstack() { // 1. Cannot specify basic sku PublicIp for an IPv6 network interface ipConfiguration. // 2. Different basic sku and standard sku public Ip resources in availability set is not allowed. // 1 & 2 means we have to use standard sku in dual-stack configuration. @@ -623,7 +623,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, fmt.Errorf("failed to create public IP: %w", err) } - if ipFamily == util.DualStack { + if ipFamily.IsDualstack() { publicIPv6, err = createOrUpdatePublicIPAddress(ctx, publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodStatic, machine.UID, config) if err != nil { return nil, fmt.Errorf("failed to create public IP: %w", err) @@ -1050,11 +1050,11 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe } switch f := providerConfig.Network.GetIPFamily(); f { - case util.Unspecified, util.IPv4: + case util.IPFamilyUnspecified, util.IPFamilyIPv4: //noop - case util.IPv6: + case util.IPFamilyIPv6: return fmt.Errorf(util.ErrIPv6OnlyUnsupported) - case util.DualStack: + case util.IPFamilyIPv4IPv6, util.IPFamilyIPv6IPv4: // validate default: return fmt.Errorf(util.ErrUnknownNetworkFamily, f) @@ -1072,7 +1072,7 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return fmt.Errorf("unknown public IP address SKU: %s", *c.PublicIPSKU) } - if providerConfig.Network.GetIPFamily() == util.DualStack && *c.PublicIPSKU == network.PublicIPAddressSkuNameBasic { + if providerConfig.Network.GetIPFamily().IsDualstack() && *c.PublicIPSKU == network.PublicIPAddressSkuNameBasic { return fmt.Errorf("cannot use %s public IP address SKU with dualstack", network.PublicIPAddressSkuNameBasic) } } @@ -1157,7 +1157,7 @@ func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Mach } if kuberneteshelper.HasFinalizer(machine, finalizerNIC) { - _, err = createOrUpdateNetworkInterface(ctx, ifaceName(machine), newUID, config, publicIP, publicIPv6, util.Unspecified, config.EnableAcceleratedNetworking) + _, err = createOrUpdateNetworkInterface(ctx, ifaceName(machine), newUID, config, publicIP, publicIPv6, util.IPFamilyUnspecified, config.EnableAcceleratedNetworking) if err != nil { return fmt.Errorf("failed to update UID on main network interface: %w", err) } diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index 6f4a9a6c4..63b5d65ae 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -190,11 +190,11 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe } switch f := pc.Network.GetIPFamily(); f { - case util.Unspecified, util.IPv4: + case util.IPFamilyUnspecified, util.IPFamilyIPv4: // noop - case util.IPv6: + case util.IPFamilyIPv6: return fmt.Errorf(util.ErrIPv6OnlyUnsupported) - case util.DualStack: + case util.IPFamilyIPv4IPv6, util.IPFamilyIPv6IPv4: // noop default: return fmt.Errorf(util.ErrUnknownNetworkFamily, f) @@ -310,7 +310,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, Name: machine.Spec.Name, Region: c.Region, Size: c.Size, - IPv6: c.IPv6 || pc.Network.GetIPFamily() == util.DualStack, + IPv6: c.IPv6 || pc.Network.GetIPFamily().IsDualstack(), PrivateNetworking: c.PrivateNetworking, Backups: c.Backups, Monitoring: c.Monitoring, diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 28e7cc3f4..72f13602a 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -117,11 +117,11 @@ func (p *Provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) } switch cfg.providerConfig.Network.GetIPFamily() { - case util.Unspecified, util.IPv4: + case util.IPFamilyUnspecified, util.IPFamilyIPv4: // noop - case util.IPv6: + case util.IPFamilyIPv6: return newError(common.InvalidConfigurationMachineError, util.ErrIPv6OnlyUnsupported) - case util.DualStack: + case util.IPFamilyIPv4IPv6, util.IPFamilyIPv6IPv4: default: return newError(common.InvalidConfigurationMachineError, util.ErrUnknownNetworkFamily, cfg.providerConfig.Network.GetIPFamily()) } diff --git a/pkg/cloudprovider/provider/gce/service.go b/pkg/cloudprovider/provider/gce/service.go index 5b15873f2..d58a9268d 100644 --- a/pkg/cloudprovider/provider/gce/service.go +++ b/pkg/cloudprovider/provider/gce/service.go @@ -28,8 +28,6 @@ import ( "google.golang.org/api/compute/v1" "google.golang.org/api/option" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog" ) @@ -93,7 +91,7 @@ func (svc *service) networkInterfaces(cfg *config) ([]*compute.NetworkInterface, // if assigning public IP addresses is enabled. if cfg.assignPublicIPAddress { // GCP doesn't support IPv6 only stack - if cfg.providerConfig.Network.GetIPFamily() == util.DualStack { + if cfg.providerConfig.Network.GetIPFamily().IsDualstack() { ifc.StackType = "IPV4_IPV6" ifc.Ipv6AccessConfigs = []*compute.AccessConfig{ diff --git a/pkg/cloudprovider/util/net.go b/pkg/cloudprovider/util/net.go index f682ca05c..bdb1fb2a7 100644 --- a/pkg/cloudprovider/util/net.go +++ b/pkg/cloudprovider/util/net.go @@ -64,12 +64,25 @@ func GenerateRandMAC() (net.HardwareAddr, error) { type IPFamily string const ( - Unspecified IPFamily = "" // interpreted as IPv4 - IPv4 IPFamily = "IPv4" - IPv6 IPFamily = "IPv6" - DualStack IPFamily = "IPv4+IPv6" + IPFamilyUnspecified IPFamily = "" // interpreted as IPv4 + IPFamilyIPv4 IPFamily = "IPv4" // IPv4 only + IPFamilyIPv6 IPFamily = "IPv6" // IPv6 only + IPFamilyIPv4IPv6 IPFamily = "IPv4+IPv6" // dualstack with IPv4 as primary + IPFamilyIPv6IPv4 IPFamily = "IPv6+IPv4" // dualstack with IPv6 as primary ) +func (f IPFamily) HasIPv6() bool { + return f == IPFamilyIPv6 || f == IPFamilyIPv4IPv6 || f == IPFamilyIPv6IPv4 +} + +func (f IPFamily) HasIPv4() bool { + return f == IPFamilyUnspecified || f == IPFamilyIPv4 || f == IPFamilyIPv4IPv6 || f == IPFamilyIPv6IPv4 +} + +func (f IPFamily) IsDualstack() bool { + return f == IPFamilyIPv4IPv6 || f == IPFamilyIPv6IPv4 +} + // IsLinkLocal checks if given ip address is link local.. func IsLinkLocal(ipAddr string) bool { addr := net.ParseIP(ipAddr) diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index b4e12ad73..ac4ed465e 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -129,7 +129,7 @@ func (n *NetworkConfig) IsStaticIPConfig() bool { func (n *NetworkConfig) GetIPFamily() util.IPFamily { if n == nil { - return util.Unspecified + return util.IPFamilyUnspecified } return n.IPFamily } diff --git a/pkg/userdata/helper/helper.go b/pkg/userdata/helper/helper.go index 86163d982..59c8af94f 100644 --- a/pkg/userdata/helper/helper.go +++ b/pkg/userdata/helper/helper.go @@ -153,11 +153,11 @@ func SetupNodeIPEnvScript(ipFamily util.IPFamily) string { var defaultIfcIP string switch ipFamily { - case util.IPv4: + case util.IPFamilyIPv4: defaultIfcIP = defaultIfcIPv4 - case util.IPv6: + case util.IPFamilyIPv6: defaultIfcIP = `DEFAULT_IFC_IP=$(ip -o -6 route get 1:: | grep -oP "src \K\S+")` - case util.DualStack: + case util.IPFamilyIPv4IPv6: defaultIfcIP = `DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") if [ -z "${DEFAULT_IFC_IP6}" ] @@ -166,6 +166,15 @@ then exit 1 fi DEFAULT_IFC_IP=$DEFAULT_IFC_IP,$DEFAULT_IFC_IP6` + case util.IPFamilyIPv6IPv4: + defaultIfcIP = `DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") +DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") +if [ -z "${DEFAULT_IFC_IP6}" ] +then + echodate "Failed to get IPv6 address for the default route interface" + exit 1 +fi +DEFAULT_IFC_IP=$DEFAULT_IFC_IP6,$DEFAULT_IFC_IP` default: defaultIfcIP = defaultIfcIPv4 } diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index 2a3ad59d5..5259fc4f0 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -138,7 +138,7 @@ var kubeletTLSCipherSuites = []string{ func withNodeIPFlag(ipFamily util.IPFamily, cloudProvider string, external bool) bool { // If external or in-tree CCM is in use we don't need to set --node-ip // as the cloud provider will know what IPs to return. - if ipFamily == util.DualStack { + if ipFamily.IsDualstack() { if external || cloudProvider != "" { return false } diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 404381e3a..c5701f4db 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -303,7 +303,7 @@ func TestUserDataGeneration(t *testing.T) { CloudProvider: "openstack", SSHPublicKeys: []string{"ssh-rsa AAABBB"}, Network: &providerconfigtypes.NetworkConfig{ - IPFamily: util.DualStack, + IPFamily: util.IPFamilyIPv4IPv6, }, }, spec: clusterv1alpha1.MachineSpec{ @@ -332,7 +332,63 @@ func TestUserDataGeneration(t *testing.T) { CloudProvider: "digitalocean", SSHPublicKeys: []string{"ssh-rsa AAABBB"}, Network: &providerconfigtypes.NetworkConfig{ - IPFamily: util.DualStack, + IPFamily: util.IPFamilyIPv4IPv6, + }, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: defaultVersion, + }, + }, + ccProvider: &fakeCloudConfigProvider{ + config: "{digitalocean-config:true}", + err: nil, + }, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, + kubernetesCACert: "CACert", + osConfig: &Config{ + DistUpgradeOnBoot: false, + }, + }, + { + name: "openstack-dualstack-IPv6+IPv4", + providerSpec: &providerconfigtypes.Config{ + CloudProvider: "openstack", + SSHPublicKeys: []string{"ssh-rsa AAABBB"}, + Network: &providerconfigtypes.NetworkConfig{ + IPFamily: util.IPFamilyIPv6IPv4, + }, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: defaultVersion, + }, + }, + ccProvider: &fakeCloudConfigProvider{ + name: "openstack", + config: "{openstack-config:true}", + err: nil, + }, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, + kubernetesCACert: "CACert", + osConfig: &Config{ + DistUpgradeOnBoot: false, + }, + externalCloudProvider: true, + }, + { + name: "digitalocean-dualstack-IPv6+IPv4", + providerSpec: &providerconfigtypes.Config{ + CloudProvider: "digitalocean", + SSHPublicKeys: []string{"ssh-rsa AAABBB"}, + Network: &providerconfigtypes.NetworkConfig{ + IPFamily: util.IPFamilyIPv6IPv4, }, }, spec: clusterv1alpha1.MachineSpec{ diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml new file mode 100644 index 000000000..3de8da793 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml @@ -0,0 +1,456 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {digitalocean-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") + if [ -z "${DEFAULT_IFC_IP6}" ] + then + echodate "Failed to get IPv6 address for the default route interface" + exit 1 + fi + DEFAULT_IFC_IP=$DEFAULT_IFC_IP6,$DEFAULT_IFC_IP + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + - 10.10.10.11 + - 10.10.10.12 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml new file mode 100644 index 000000000..1f82ce5d3 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml @@ -0,0 +1,456 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=docker.service + Requires=docker.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=docker \ + --container-runtime-endpoint=unix:///var/run/dockershim.sock \ + --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ + --feature-gates=DynamicKubeletConfig=true \ + --network-plugin=cni \ + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {openstack-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") + if [ -z "${DEFAULT_IFC_IP6}" ] + then + echodate "Failed to get IPv6 address for the default route interface" + exit 1 + fi + DEFAULT_IFC_IP=$DEFAULT_IFC_IP6,$DEFAULT_IFC_IP + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/docker/daemon.json + permissions: "0644" + content: | + {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + - 10.10.10.11 + - 10.10.10.12 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service From e611f0924345758fece668039c4dd8adf05babe1 Mon Sep 17 00:00:00 2001 From: Moritz Bracht <682686+dermorz@users.noreply.github.com> Date: Fri, 2 Dec 2022 11:43:13 +0100 Subject: [PATCH 255/489] Set EvictionStrategy to external on KubeVirt VMs (#1504) * Set EvictionStrategy to external on kubevirt VMI Signed-off-by: Moritz Bracht * Add remark about EvictionStratey in kubevirt docs Signed-off-by: Moritz Bracht * Adjust testdata Signed-off-by: Moritz Bracht * Improve kubevirt eviction strategy docs Signed-off-by: Moritz Bracht Signed-off-by: Moritz Bracht --- docs/kubevirt.md | 1 + pkg/cloudprovider/provider/kubevirt/provider.go | 3 +++ pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml | 1 + .../provider/kubevirt/testdata/custom-local-disk.yaml | 1 + .../provider/kubevirt/testdata/http-image-source.yaml | 1 + .../kubevirt/testdata/instancetype-preference-custom.yaml | 1 + .../kubevirt/testdata/instancetype-preference-standard.yaml | 1 + pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml | 1 + .../provider/kubevirt/testdata/pvc-image-source.yaml | 1 + .../provider/kubevirt/testdata/secondary-disks.yaml | 3 ++- .../provider/kubevirt/testdata/topologyspreadconstraints.yaml | 1 + 11 files changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/kubevirt.md b/docs/kubevirt.md index 2f86ba155..819d84140 100644 --- a/docs/kubevirt.md +++ b/docs/kubevirt.md @@ -12,6 +12,7 @@ are some things you need to keep in mind: * The machine-controller will create `VMIs` that have the same name as the underlying `machine`. To avoid collisions, use one namespace per cluster that runs the `machine-controller` +* EvictionStratey of `VMIs` is set to external, so VMI eviction needs to handled properly by a custom external controller or manual action * Service CIDR range: The CIDR ranges of the cluster that runs Kubevirt and the cluster that hosts the machine-controller must not overlap, otherwise routing of services that run in the kubevirt cluster won't work anymore. This is especially important for the DNS ClusterIP. * `clusterName` is used to [label VMs](https://github.com/kubevirt/cloud-provider-kubevirt#prerequisites) for LoadBalancer selection diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 9f28e1f7d..e85bc2658 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -597,6 +597,8 @@ func (p *provider) newVirtualMachine(ctx context.Context, c *Config, pc *provide // The secret has an ownerRef on the VMI so garbace collection will take care of cleaning up. terminationGracePeriodSeconds := int64(30) + evictionStrategy := kubevirtv1.EvictionStrategyExternal + resourceRequirements := kubevirtv1.ResourceRequirements{} labels := map[string]string{"kubevirt.io/vm": machine.Name} //Add a common label to all VirtualMachines spawned by the same MachineDeployment (= MachineDeployment name). @@ -652,6 +654,7 @@ func (p *provider) newVirtualMachine(ctx context.Context, c *Config, pc *provide Labels: labels, }, Spec: kubevirtv1.VirtualMachineInstanceSpec{ + EvictionStrategy: &evictionStrategy, Networks: []kubevirtv1.Network{ *kubevirtv1.DefaultPodNetwork(), }, diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml index fb2392076..309e30d7b 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml @@ -83,3 +83,4 @@ spec: secretRef: name: udsn name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml index 5c1a645ac..e10d07c62 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml @@ -75,3 +75,4 @@ spec: secretRef: name: udsn name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml index b350d2053..3bf7731e7 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml @@ -74,3 +74,4 @@ spec: secretRef: name: udsn name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml index d6e7abc67..060171c4b 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml @@ -73,3 +73,4 @@ spec: secretRef: name: udsn name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml index 19457b9cd..b56229f69 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml @@ -73,3 +73,4 @@ spec: secretRef: name: udsn name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml index f49ddf69c..5dddb4b52 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml @@ -74,3 +74,4 @@ spec: secretRef: name: udsn name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml index b78a189cf..2cafabbd7 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml @@ -75,3 +75,4 @@ spec: secretRef: name: udsn name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml index ec2186480..576172c93 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml @@ -110,5 +110,6 @@ spec: name: secondary-disks-secondarydisk0 name: secondary-disks-secondarydisk0 - dataVolume: - name: secondary-disks-secondarydisk1 + name: secondary-disks-secondarydisk1 name: secondary-disks-secondarydisk1 + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml index fdfa6fc57..a36f6b84d 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml @@ -80,3 +80,4 @@ spec: secretRef: name: udsn name: cloudinitdisk + evictionStrategy: External From 0a94443b41516398b5bd6a129589dea599f3e7b8 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 2 Dec 2022 19:24:12 +0500 Subject: [PATCH 256/489] Update Go dependencies and DCO (#1500) * Update GO dependencies and DCO Signed-off-by: Waleed Malik * Update DCO Signed-off-by: Waleed Malik * Fix license and module proxy issue Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- .wwhrd.yml | 1 + CONTRIBUTING.md | 41 +- DCO | 2 - go.mod | 184 ++-- go.sum | 837 ++++++------------ pkg/cloudprovider/provider/aws/provider.go | 8 +- pkg/cloudprovider/provider/azure/provider.go | 36 +- .../provider/kubevirt/provider.go | 6 +- .../provider/openstack/provider_test.go | 4 +- .../openstack/types/cloudconfig_test.go | 4 +- .../provider/vsphere/provider_test.go | 16 +- pkg/userdata/helper/kubelet.go | 6 +- pkg/userdata/sles/provider_test.go | 8 +- 13 files changed, 466 insertions(+), 687 deletions(-) diff --git a/.wwhrd.yml b/.wwhrd.yml index 239a0dc31..497ede2eb 100644 --- a/.wwhrd.yml +++ b/.wwhrd.yml @@ -32,3 +32,4 @@ exceptions: - github.com/embik/nutanix-client-go/internal/utils # MPL-2.0 - github.com/ajeddeloh/go-json # Since it's a fork, https://github.com/golang/go/blob/master/LICENSE - github.com/hashicorp/go-version # MPL-2.0 + - github.com/hashicorp/go-cleanhttp # MPL-2.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d52315994..322c21134 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,11 +28,50 @@ Note that we're requiring all commits in a PR to be signed-off. If you already c git rebase --signoff origin/main ``` -By doing this you state that you can certify the following (from https://developercertificate.org/): +By doing this you state that you can certify the following (from ): + +``` +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. + +``` ## Email and Chat The Machine Controller project currently uses the general Kubermatic email list and Slack channel: + - Email: [loodse-dev](https://groups.google.com/forum/#!forum/loodse-dev) - Slack: #kubermatic channel on [Kubermatic Slack]((http://slack.kubermatic.io/)) diff --git a/DCO b/DCO index 716561d5d..49b8cb054 100644 --- a/DCO +++ b/DCO @@ -2,8 +2,6 @@ Developer Certificate of Origin Version 1.1 Copyright (C) 2004, 2006 The Linux Foundation and its contributors. -660 York Street, Suite 102, -San Francisco, CA 94110 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. diff --git a/go.mod b/go.mod index 9172d3228..c83267815 100644 --- a/go.mod +++ b/go.mod @@ -3,168 +3,180 @@ module github.com/kubermatic/machine-controller go 1.19 require ( - cloud.google.com/go/logging v1.4.2 - cloud.google.com/go/monitoring v1.5.0 + cloud.google.com/go/logging v1.5.0 + cloud.google.com/go/monitoring v1.9.0 github.com/Azure/azure-sdk-for-go v65.0.0+incompatible github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 github.com/Azure/go-autorest/autorest/to v0.4.0 - github.com/BurntSushi/toml v1.1.0 - github.com/Masterminds/semver/v3 v3.1.1 + github.com/BurntSushi/toml v1.2.1 + github.com/Masterminds/semver/v3 v3.2.0 github.com/Masterminds/sprig/v3 v3.2.2 - github.com/aliyun/alibaba-cloud-sdk-go v1.61.1645 - github.com/aws/aws-sdk-go-v2 v1.16.12 - github.com/aws/aws-sdk-go-v2/config v1.17.3 - github.com/aws/aws-sdk-go-v2/credentials v1.12.16 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.54.2 - github.com/aws/aws-sdk-go-v2/service/sts v1.16.15 - github.com/aws/smithy-go v1.13.0 + github.com/aliyun/alibaba-cloud-sdk-go v1.62.47 + github.com/aws/aws-sdk-go-v2 v1.17.1 + github.com/aws/aws-sdk-go-v2/config v1.18.3 + github.com/aws/aws-sdk-go-v2/credentials v1.13.3 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.73.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 + github.com/aws/smithy-go v1.13.4 github.com/coreos/container-linux-config-transpiler v0.9.0 github.com/davecgh/go-spew v1.1.1 - github.com/digitalocean/godo v1.81.0 + github.com/digitalocean/godo v1.91.1 github.com/ghodss/yaml v1.0.0 github.com/go-test/deep v1.0.8 github.com/google/uuid v1.3.0 - github.com/gophercloud/gophercloud v0.25.0 - github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 - github.com/hetznercloud/hcloud-go v1.34.0 - github.com/linode/linodego v1.8.0 - github.com/nutanix-cloud-native/prism-go-client v0.3.0 - github.com/packethost/packngo v0.25.0 + github.com/gophercloud/gophercloud v1.0.0 + github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb + github.com/hetznercloud/hcloud-go v1.37.0 + github.com/linode/linodego v1.9.3 + github.com/nutanix-cloud-native/prism-go-client v0.3.4 + github.com/packethost/packngo v0.29.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 - github.com/prometheus/client_golang v1.12.2 - github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9 + github.com/prometheus/client_golang v1.14.0 + github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10 github.com/sethvargo/go-password v0.2.0 - github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda - github.com/vmware/go-vcloud-director/v2 v2.15.0 - github.com/vmware/govmomi v0.28.0 - go.anx.io/go-anxcloud v0.4.4 - golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d - golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb + github.com/tinkerbell/tink v0.8.0 + github.com/vmware/go-vcloud-director/v2 v2.17.0 + github.com/vmware/govmomi v0.29.0 + go.anx.io/go-anxcloud v0.4.6 + golang.org/x/crypto v0.3.0 + golang.org/x/oauth2 v0.2.0 gomodules.xyz/jsonpatch/v2 v2.2.0 - google.golang.org/api v0.74.0 - google.golang.org/grpc v1.45.0 + google.golang.org/api v0.103.0 + google.golang.org/grpc v1.51.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.25.0 - k8s.io/apiextensions-apiserver v0.24.2 - k8s.io/apimachinery v0.25.0 + k8s.io/api v0.25.4 + k8s.io/apiextensions-apiserver v0.25.4 + k8s.io/apimachinery v0.25.4 k8s.io/client-go v12.0.0+incompatible k8s.io/klog v1.0.0 - k8s.io/kubelet v0.24.2 - k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed + k8s.io/kubelet v0.25.4 + k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 kubevirt.io/api v0.58.0 - kubevirt.io/containerized-data-importer-api v1.55.0 - sigs.k8s.io/controller-runtime v0.12.1 + kubevirt.io/containerized-data-importer-api v1.55.1 + sigs.k8s.io/controller-runtime v0.13.1 sigs.k8s.io/yaml v1.3.0 ) require ( - cloud.google.com/go v0.100.2 // indirect - cloud.google.com/go/compute v1.5.0 // indirect + cloud.google.com/go v0.107.0 // indirect + cloud.google.com/go/compute v1.12.1 // indirect + cloud.google.com/go/compute/metadata v0.2.1 // indirect + cloud.google.com/go/longrunning v0.3.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.27 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect - github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect + github.com/Azure/go-autorest/autorest v0.11.28 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.21 // indirect + github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/PaesslerAG/gval v1.0.0 // indirect + github.com/PaesslerAG/gval v1.2.1 // indirect github.com/PaesslerAG/jsonpath v0.1.1 // indirect github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect - github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.20 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.13 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.19 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.1 // indirect + github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/ignition v0.35.0 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/docker/distribution v2.7.1+incompatible // indirect - github.com/emicklei/go-restful/v3 v3.8.0 // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect + github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/swag v0.21.1 // indirect - github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-resty/resty/v2 v2.7.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.2.0 // indirect + github.com/golang-jwt/jwt/v4 v4.4.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/gnostic v0.6.9 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/googleapis/gax-go/v2 v2.3.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/hashicorp/go-version v1.2.0 // indirect - github.com/huandu/xstrings v1.3.2 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/huandu/xstrings v1.3.3 // indirect github.com/imdario/mergo v0.3.13 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kr/pretty v0.3.0 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/opencontainers/go-digest v1.0.0-rc1 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect - github.com/peterhellberg/link v1.1.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.35.0 // indirect - github.com/prometheus/procfs v0.7.3 // indirect - github.com/rogpeppe/go-internal v1.8.0 // indirect + github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect + github.com/packethost/pkg v0.0.0-20211110202003-387414657e83 // indirect + github.com/peterhellberg/link v1.2.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/cobra v1.6.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect - go.opencensus.io v0.23.0 // indirect - go.uber.org/atomic v1.9.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.4 // indirect + go.opentelemetry.io/otel v1.11.1 // indirect + go.opentelemetry.io/otel/trace v1.11.1 // indirect + go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - go.uber.org/zap v1.21.0 // indirect + go.uber.org/zap v1.23.0 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect - golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect - golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect + golang.org/x/net v0.2.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.2.0 // indirect + golang.org/x/term v0.2.0 // indirect + golang.org/x/text v0.4.0 // indirect + golang.org/x/time v0.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9 // indirect - google.golang.org/protobuf v1.28.0 // indirect + google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.66.4 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/component-base v0.24.2 // indirect - k8s.io/klog/v2 v2.70.1 // indirect - k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect - kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect - sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect + k8s.io/component-base v0.25.4 // indirect + k8s.io/klog/v2 v2.80.1 // indirect + k8s.io/kube-openapi v0.0.0-20221123214604-86e75ddd809a // indirect + kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) -replace k8s.io/client-go => k8s.io/client-go v0.25.0 +replace k8s.io/client-go => k8s.io/client-go v0.25.4 diff --git a/go.sum b/go.sum index b9448c299..c1bd5351f 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,11 @@ cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+Y cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2 h1:t9Iw5QH5v4XtlEQaCtUY7x6sCABps8sW0acw7e2WQ6Y= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= +cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww= +cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -36,15 +39,23 @@ cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4g cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0 h1:b1zWmYuuHz7gO9kDcM/EpHGr06UgsYNRpNJzI2kFiLM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0= +cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= +cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48= +cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/logging v1.4.2 h1:Mu2Q75VBDQlW1HlBMjTX4X84UFR73G1TiLlRYc/b7tA= -cloud.google.com/go/logging v1.4.2/go.mod h1:jco9QZSx8HiVVqLJReq7z7bVdj0P1Jb9PDFs63T+axo= -cloud.google.com/go/monitoring v1.5.0 h1:ZltYv8e69fJVga7RTthUBGdx4+Pwz6GRF1V3zylERl4= -cloud.google.com/go/monitoring v1.5.0/go.mod h1:/o9y8NYX5j91JjD/JvGLYbi86kL11OjyJXq2XziLJu4= +cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= +cloud.google.com/go/logging v1.5.0 h1:DcR52smaYLgeK9KPzJlBJyyBYqW/EGKiuRRl8boL1s4= +cloud.google.com/go/logging v1.5.0/go.mod h1:c/57U/aLdzSFuBtvbtFduG1Ii54uSm95HOBnp58P7/U= +cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= +cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= +cloud.google.com/go/monitoring v1.9.0 h1:O2A5HsrhvRMzD3OMUimPXF46vOzwc9vh6oGCGf9i/ws= +cloud.google.com/go/monitoring v1.9.0/go.mod h1:/FsTS0gkEFUc4cgB16s6jYDnyjzRBkRJNRzBn5Zx+wA= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -54,25 +65,27 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/99designs/gqlgen v0.15.1 h1:48bRXecwlCNTa/n2bMSp2rQsXNxwZ54QHbiULNf78ec= github.com/99designs/gqlgen v0.15.1/go.mod h1:nbeSjFkqphIqpZsYe1ULVz0yfH8hjpJdJIQoX/e0G2I= github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= -github.com/Azure/go-autorest/autorest v0.11.27 h1:F3R3q42aWytozkV8ihzcgMO4OA4cuqr3bNlsEuF6//A= github.com/Azure/go-autorest/autorest v0.11.27/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U= +github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= +github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= -github.com/Azure/go-autorest/autorest/adal v0.9.20 h1:gJ3E98kMpFB1MFqQCvA1yFab8vthOeD4VlFRQULxahg= github.com/Azure/go-autorest/autorest/adal v0.9.20/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= +github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 h1:P6bYXFoao05z5uhOQzbC3Qd8JqF3jUoocoTeIxkp2cA= github.com/Azure/go-autorest/autorest/azure/auth v0.5.11/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 h1:0W/yGmFdTIT77fvdlGZ0LMISoLHFJ7Tx4U0yeB+uFs4= github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= @@ -87,33 +100,26 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= -github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= -github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PaesslerAG/gval v1.0.0 h1:GEKnRwkWDdf9dOmKcNrar9EA1bz1z9DqPIO1+iLzhd8= github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= +github.com/PaesslerAG/gval v1.2.1 h1:Ggwtej1xCyt1994VuDCSjycybIDo3duDCDghK/xc/A0= +github.com/PaesslerAG/gval v1.2.1/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac= github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8= github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk= github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/agnivade/levenshtein v1.1.0 h1:n6qGwyHG61v3ABce1rPVZklEYRT8NFpCMrpZdBUbYGM= github.com/agnivade/levenshtein v1.1.0/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= @@ -121,7 +127,6 @@ github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 h1:4SPQljF/GJ8Q+ github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd h1:NlKlOv3aVJ5ODMC0JWPvddw05KENkL3cZttIuu8kJRo= github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd/go.mod h1:idhzw68Q7v4j+rQ2AGyq3OlZW2Jij9mdmGA4/Sk6J0E= -github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -129,74 +134,53 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/aliyun/alibaba-cloud-sdk-go v1.61.1645 h1:IEL/Da0Dtg9j/36UnzyxD84n0eDj0JIoTKTKobN2eks= -github.com/aliyun/alibaba-cloud-sdk-go v1.61.1645/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.47 h1:xMEN30mWG8D0yfa3imurqQQ02tjPtOVvLRknv/R+pls= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.47/go.mod h1:Api2AkmMgGaSUAhmk76oaFObkoeCPc/bKAqcyplPODs= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195 h1:c4mLfegoDw6OhSJXTd2jUEQgZUQuJWtocudb97Qn9EM= -github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.42.23/go.mod h1:gyRszuZ/icHmHAVE4gc/r+cfCmhA1AD+vqfWbgI+eHs= -github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/aws/aws-sdk-go-v2 v1.16.12 h1:wbMYa2PlFysFx2GLIQojr6FJV5+OWCM/BwyHXARxETA= -github.com/aws/aws-sdk-go-v2 v1.16.12/go.mod h1:C+Ym0ag2LIghJbXhfXZ0YEEp49rBWowxKzJLUoob0ts= -github.com/aws/aws-sdk-go-v2/config v1.17.3 h1:s1As/fiVMmM3CObC4GcSaSbkhm88S6a5qn8St3wgal0= -github.com/aws/aws-sdk-go-v2/config v1.17.3/go.mod h1:tRGUOfk9Rrf6UCJm5qDlL9AizSsgvteuKX4qajAV3pU= -github.com/aws/aws-sdk-go-v2/credentials v1.12.16 h1:HXczS88Pg36j8dq0KSjtHBPFs8gdRyBSS1hueeG/rxA= -github.com/aws/aws-sdk-go-v2/credentials v1.12.16/go.mod h1:eLJ+j1lwQdHJ0c56tRoDWcgss1e/laVmvW2AaOicuAw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.13 h1:+uferi8SUDZtMloCDt24Zenyy/i71C/ua5mjUCpbpN0= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.13/go.mod h1:y0eXmsNBFIVjUE8ZBjES8myOHlMsXDz7qGT93+MVdjk= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19 h1:gC5mudiFrWGhzcdoWj1iCGUfrzCpQG0MQIQf0CXFFQQ= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.19/go.mod h1:llxE6bwUZhuCas0K7qGiu5OgMis3N7kdWtFSxoHmJ7E= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13 h1:qezY57na06d6kSE7uuB0N7XEflu914AXx/hg2L8Ykcw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.13/go.mod h1:lB12mkZqCSo5PsdBFLNqc2M/OOYgNAy8UtaktyuWvE8= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.20 h1:GvszACAU8GSV3+Tant5GutW6smY8WavrP8ZuRS9Ku4Q= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.20/go.mod h1:bfTcsThj5a9P5pIGRy0QudJ8k4+issxXX+O6Djnd5Cs= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.54.2 h1:rlqJWpugIyaw7UROyETCgao75pyiq2pkETFSg8oq+fU= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.54.2/go.mod h1:d1gxyomADOqOm0m9lGsr1m61ubU7lUyWhxkEeJBgPF4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.13 h1:ObfthqDyhe7rMAOa7pqft6974VHIk8BAJB7kYdoIfTA= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.13/go.mod h1:V390DK4MQxLpDdXxFqizyz8KUxuWImkW/xzgXMz0yyk= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.19 h1:WdCwfJmu23XiIDeZwclSyAorQe916M3LeHd53xqBjfA= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.19/go.mod h1:ytmEi5+qwcSNcV2pVA8PIb1DnKT/0Bu/K4nfJHwoM6c= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.1 h1:p48IfndYbRk3iDsoQAmVXdCKEM5+7Y50JAPikjwk8gI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.1/go.mod h1:NY+G+8PW0ISyJ7/6t5mgOe6qpJiwZa9Jix05WPscJjg= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.15 h1:ApuR2BK9vf5/XXsImHBBsYJ6aUhmUhBHnZMPyhJo1jQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.15/go.mod h1:Y+BUV19q3OmQVqNUlbZ40zVi3NM6Biuxwkx/qdSD/CY= -github.com/aws/smithy-go v1.13.0 h1:YfyEmSJLo7fAv8FbuDK4R8F9aAmi9DZ88Zb/KJJmUl0= -github.com/aws/smithy-go v1.13.0/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/aws/aws-sdk-go-v2 v1.17.1 h1:02c72fDJr87N8RAC2s3Qu0YuvMRZKNZJ9F+lAehCazk= +github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= +github.com/aws/aws-sdk-go-v2/config v1.18.3 h1:3kfBKcX3votFX84dm00U8RGA1sCCh3eRMOGzg5dCWfU= +github.com/aws/aws-sdk-go-v2/config v1.18.3/go.mod h1:BYdrbeCse3ZnOD5+2/VE/nATOK8fEUpBtmPMdKSyhMU= +github.com/aws/aws-sdk-go-v2/credentials v1.13.3 h1:ur+FHdp4NbVIv/49bUjBW+FE7e57HOo03ELodttmagk= +github.com/aws/aws-sdk-go-v2/credentials v1.13.3/go.mod h1:/rOMmqYBcFfNbRPU0iN9IgGqD5+V2yp3iWNmIlz0wI4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 h1:E3PXZSI3F2bzyj6XxUXdTIfvp425HHhwKsFvmzBwHgs= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19/go.mod h1:VihW95zQpeKQWVPGkwT+2+WJNQV8UXFfMTWdU6VErL8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 h1:nBO/RFxeq/IS5G9Of+ZrgucRciie2qpLy++3UGZ+q2E= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 h1:oRHDrwCTVT8ZXi4sr9Ld+EXk7N/KGssOr2ygNeojEhw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 h1:Mza+vlnZr+fPKFKRq/lKGVvM6B/8ZZmNdEopOwSQLms= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26/go.mod h1:Y2OJ+P+MC1u1VKnavT+PshiEuGPyh/7DqxoDNij4/bg= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.73.0 h1:3AXOhjvPxEMWw5RItV47NRLuzqwlLly5GbS5aB3sXh4= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.73.0/go.mod h1:zul71QqzR4D1a90/5FloZiAnZ1CtuIjVH7R9MP997+A= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 h1:GE25AWCdNUPh9AOJzI9KIJnja7IwUc1WyUqz/JTyJ/I= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19/go.mod h1:02CP6iuYP+IVnBX5HULVdSAku/85eHB2Y9EsFhrkEwU= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 h1:GFZitO48N/7EsFDt8fMa5iYdmWqkUDDB3Eje6z3kbG0= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.25/go.mod h1:IARHuzTXmj1C0KS35vboR0FeJ89OkEy1M9mWbK2ifCI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 h1:jcw6kKZrtNfBPJkaHrscDOZoe5gvi9wjudnxvozYFJo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8/go.mod h1:er2JHN+kBY6FcMfcBBKNGCT3CarImmdFzishsqBmSRI= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 h1:60SJ4lhvn///8ygCzYy2l53bFW/Q15bVfyjyAWo6zuw= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.5/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4= +github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk= +github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bnkamalesh/webgo/v4 v4.1.11/go.mod h1:taIAonQTzao8G5rnB22WgKmQuIOWHpQ0n/YLAidBXlM= github.com/bnkamalesh/webgo/v6 v6.2.2/go.mod h1:2Y+dEdTp1xC/ra+3PAVZV6hh4sCI+iPK7mcHt+t9bfM= -github.com/briandowns/spinner v1.8.0/go.mod h1://Zf9tMcxfRUA36V23M6YGEAv+kECGfvpnLTnb8n4XQ= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= @@ -204,7 +188,6 @@ github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -213,40 +196,20 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/container-linux-config-transpiler v0.9.0 h1:UBGpT8qWqzi48hNLrzMAgAUNJsR0LW8Gk5/dR/caI8U= github.com/coreos/container-linux-config-transpiler v0.9.0/go.mod h1:SlcxXZQ2c42knj8pezMiQsM1f+ADxFMjGetuMKR/YSQ= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/ignition v0.35.0 h1:UFodoYq1mOPrbEjtxIsZbThcDyQwAI1owczRDqWmKkQ= github.com/coreos/ignition v0.35.0/go.mod h1:WJQapxzEn9DE0ryxsGvm8QnBajm/XsS/PkrDqSpz+bA= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creasty/defaults v1.5.2 h1:/VfB6uxpyp6h0fr7SPp7n8WJBoV8jfxQXPCnkVSjyls= github.com/creasty/defaults v1.5.2/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY= github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= @@ -259,38 +222,23 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= -github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= -github.com/digitalocean/godo v1.81.0 h1:sjb3fOfPfSlUQUK22E87BcI8Zx2qtnF7VUCCO4UK3C8= -github.com/digitalocean/godo v1.81.0/go.mod h1:BPCqvwbjbGqxuUnIKB4EvS/AX7IDnNmt5fwvIkWo+ew= +github.com/digitalocean/godo v1.91.1 h1:1o30VOCu1aC6488qBd0SkQiBeAZ35RSTvLwCA1pQMhc= +github.com/digitalocean/godo v1.91.1/go.mod h1:NRpFznZFvhHjBoqZAaOD3khVzsJ3EibzKqFL4R60dmA= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= -github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= +github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -299,32 +247,25 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= -github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= -github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs= github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg= github.com/go-chi/cors v1.2.0/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= @@ -335,7 +276,6 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -344,16 +284,15 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= -github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -362,11 +301,11 @@ github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= @@ -374,46 +313,26 @@ github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= -github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-playground/validator/v10 v10.10.1 h1:uA0+amWMiglNZKZ9FJRKUAe9U3RX91eVn1JYXMWt7ig= github.com/go-playground/validator/v10 v10.10.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= -github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 h1:JVrqSeQfdhYRFk24TvhTZWU0q8lfCojxZQFi3Ou7+uY= -github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= +github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w= -github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs= -github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= -github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kETFqjFoFlOc= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-json v0.9.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godror/godror v0.13.3/go.mod h1:2ouUT4kdhUBk7TAkHWD4SN0CdI0pgEQbo8FVHhbSKWg= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -446,14 +365,11 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/cel-go v0.10.1/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= -github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= @@ -470,8 +386,9 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -504,93 +421,63 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= +github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= +github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0 h1:nRJtk3y8Fm770D42QV6T90ZnvFZyk7agSo3Q+Z9p3WI= github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gophercloud/gophercloud v0.25.0 h1:C3Oae7y0fUVQGSsBrb3zliAjdX+riCSEh4lNMejFNI4= -github.com/gophercloud/gophercloud v0.25.0/go.mod h1:Q8fZtyi5zZxPS/j9aj3sSxtvj41AdQMDwyo1myduD5c= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= +github.com/gophercloud/gophercloud v1.0.0 h1:9nTGx0jizmHxDobe4mck89FyQHVyA3CaXLIUSGJjP9k= +github.com/gophercloud/gophercloud v1.0.0/go.mod h1:Q8fZtyi5zZxPS/j9aj3sSxtvj41AdQMDwyo1myduD5c= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.15.2/go.mod h1:vO11I9oWA+KsxmfFQPhLnnIb1VDE24M+pdxZFiuZcA8= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 h1:GT4RsKmHh1uZyhmTkWJTDALRjSHYQp6FRKrotf0zhAs= -github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= -github.com/hetznercloud/hcloud-go v1.34.0 h1:yCmlDl+S9LDDuk0PkStn7XT/DAlBquE5WS4BEnDE5Xc= -github.com/hetznercloud/hcloud-go v1.34.0/go.mod h1:ztUc4lPyGRKJDJ6i8evK4kwAlYO0aZkVAMoZwX9nSjQ= +github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb h1:tsEKRC3PU9rMw18w/uAptoijhgG4EvlA5kfJPtwrMDk= +github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= +github.com/hetznercloud/hcloud-go v1.37.0 h1:Uwu7OKfZvar86LfJuzItStoO1AL7DVDCqWzRGzrvdEw= +github.com/hetznercloud/hcloud-go v1.37.0/go.mod h1:mepQwR6va27S3UQthaEPGS86jtzSY9xWL1e9dyxXpgA= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= -github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= +github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -598,33 +485,28 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/keploy/go-sdk v0.4.3 h1:dCsmfANlZH94It+JKWx8/JEEC6dn8W7KIRRKRZwCPZQ= github.com/keploy/go-sdk v0.4.3/go.mod h1:tn62gQ8a/AD7mY51DvQfhudiBPTlD+w3XtXemDcbON4= -github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kevinmbeaulieu/eq-go v1.0.0/go.mod h1:G3S8ajA56gKBZm4UB9AOyoOS37JO3roToPzKNM8dtdM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= @@ -639,60 +521,37 @@ github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++ github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= github.com/lestrrat-go/jwx v1.2.20/go.mod h1:tLE1XszaFgd7zaS5wHe4NxA+XVhu7xgdRvDpNyi3kNM= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.1-0.20191011153232-f91d3411e481/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/linode/linodego v1.8.0 h1:7B2UaWu6C48tZZZrtINWRElAcwzk4TLnL9USjKf3xm0= -github.com/linode/linodego v1.8.0/go.mod h1:heqhl91D8QTPVm2k9qZHP78zzbOdTFLXE9NJc3bcc50= +github.com/linode/linodego v1.9.3 h1:+lxNZw4avRxhCqGjwfPgQ2PvMT+vOL0OMsTdzixR7hQ= +github.com/linode/linodego v1.9.3/go.mod h1:h6AuFR/JpqwwM/vkj7s8KV3iGN8/jxn+zc437F8SZ8w= github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matryer/moq v0.2.3/go.mod h1:9RtPYjTnH1bSBIkpvtHkFN7nbWAnO7oRpdJkEIn6UtE= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-oci8 v0.0.7/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.5/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -701,37 +560,20 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nutanix-cloud-native/prism-go-client v0.3.0 h1:4N6L8qLpEl7Y4jKmhGQNk+fMVYLc9FZCINApfuhrA+4= -github.com/nutanix-cloud-native/prism-go-client v0.3.0/go.mod h1:mwZsRrdiXVDtz8G1+Z79wbVHIJ41LB44xRN13/HlGPM= +github.com/nutanix-cloud-native/prism-go-client v0.3.4 h1:bHY3VPrHHYnbRtkpGaKK+2ZmvUjNVRC55CYZbXIfnOk= +github.com/nutanix-cloud-native/prism-go-client v0.3.4/go.mod h1:tTIH02E6o6AWSShr98QChoxuZl+jBhkXFixom9+fd1Y= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= @@ -739,125 +581,88 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.2.0 h1:3ZNA3L1c5FYDFTTxbFeVGGD8jYvjYauHD30YgLxVsNI= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.20.2 h1:8uQq0zMgLEfa0vRrrBgaJF2gyW9Da9BmfGV+OyUzfkY= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 h1:+ShYlGoPriGahTTFTjQ0RtNXW0srxDodk2STdc238Rk= github.com/openshift/api v0.0.0-20211217221424-8779abfbd571/go.mod h1:F/eU6jgr6Q2VhMu1mSpMmygxAELd7+BUxs3NHZ25jV4= github.com/openshift/build-machinery-go v0.0.0-20211213093930-7e33a7eb4ce3/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/packethost/packngo v0.25.0 h1:ujGXL3lVqTiaQoX2/Go74lQAlYfTeop7jBNy5w99w2A= -github.com/packethost/packngo v0.25.0/go.mod h1:/UHguFdPs6Lf6FOkkSEPnRY5tgS0fsVM+Zv/bvBrmt0= -github.com/packethost/pkg v0.0.0-20200903155310-0433e0605550/go.mod h1:GSv7cTtIjns4yc0pyajaM1RE/KE4djJONoblFIRDrxA= -github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= +github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= +github.com/packethost/packngo v0.29.0 h1:gRIhciVZQ/zLNrIdIdbOUyB/Tw5IgoaXyhP4bvE+D2s= +github.com/packethost/packngo v0.29.0/go.mod h1:/UHguFdPs6Lf6FOkkSEPnRY5tgS0fsVM+Zv/bvBrmt0= +github.com/packethost/pkg v0.0.0-20211110202003-387414657e83 h1:uhBvTY/Hnm7rLz7gPkA83JU4EQf4A2YZUBry6+Gyn9g= +github.com/packethost/pkg v0.0.0-20211110202003-387414657e83/go.mod h1:iF7Mj6XXQ6O+bCfrBCrsJrIGxG7ptrZwb0bW91+wzm8= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/peterhellberg/link v1.1.0 h1:s2+RH8EGuI/mI4QwrWGSYQCRz7uNgip9BaM04HKu5kc= -github.com/peterhellberg/link v1.1.0/go.mod h1:gtSlOT4jmkY8P47hbTc8PTgiDDWpdPbFYl75keYyBB8= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/peterhellberg/link v1.2.0 h1:UA5pg3Gp/E0F2WdX7GERiNrPQrM1K6CVJUUWfHa4t6c= +github.com/peterhellberg/link v1.2.0/go.mod h1:gYfAh+oJgQu2SrZHg5hROVRQe1ICoK0/HHJTcE0edxc= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= -github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.31.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.35.0 h1:Eyr+Pw2VymWejHqCugNaQXkAi6KayVNxaHeu6khmFBE= -github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= -github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rollbar/rollbar-go v1.4.2/go.mod h1:kLQ9gP3WCRGrvJmF0ueO3wK9xWocej8GRX98D8sa39w= +github.com/rollbar/rollbar-go/errors v0.0.0-20210929193720-32947096267e/go.mod h1:Ie0xEc1Cyj+T4XMO8s0Vf7pMfvSAAy1sb4AYc8aJsao= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9 h1:0roa6gXKgyta64uqh52AQG3wzZXH21unn+ltzQSXML0= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10 h1:wsfMs0iv+MJiViM37qh5VEKISi3/ZUq2nNKNdqmumAs= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= +github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI= @@ -867,66 +672,43 @@ github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5g github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.0.1-0.20200713175500-884edc58ad08/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/stormcat24/protodep v0.0.0-20200505140716-b02c9ba62816/go.mod h1:mBd5PI4uI6NkqJpCyiWiYzWyTFs4QRDss/JTMC2b4kc= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/testcontainers/testcontainers-go v0.9.0/go.mod h1:b22BFXhRbg4PJmeMVWh6ftqjyZHgiIl3w274e9r3C2E= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda h1:uAHwUH+06gowZMVLqQXm7jN1y3Sl+CDJHThNiKyLHus= -github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda/go.mod h1:s4k7CORR0OMWd4cYwBqNBFPSJZhnSQxeKdDtMa/aspk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tinkerbell/lint-install v0.0.0-20211012174934-5ee5ab01db76/go.mod h1:0h2KsALaQLNkoVeV+G+HjBWWCnp0COFYhJdRd5WCQPM= +github.com/tinkerbell/tink v0.8.0 h1:qgl/rglpO5Rvq6UKZd29O6X9mDgZZYgf841+Y0IYWak= +github.com/tinkerbell/tink v0.8.0/go.mod h1:bfAkSH7J/QQYIyqZRR6IQp8w78aac6l8Z2Lws5uXz6A= +github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= +github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= +github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.35.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= @@ -937,19 +719,16 @@ github.com/vektah/gqlparser/v2 v2.2.0 h1:bAc3slekAAJW6sZTi07aGq0OrfaCjj4jxARAaC7 github.com/vektah/gqlparser/v2 v2.2.0/go.mod h1:i3mQIGIrbK2PD1RrCeMTlVbkF2FJ6WkU1KJlJlC+3F4= github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= -github.com/vmware/go-vcloud-director/v2 v2.15.0 h1:idQ9NsHLr2dOSLBC8KIdBMq7XOvPiWmfxgWNaf580mk= -github.com/vmware/go-vcloud-director/v2 v2.15.0/go.mod h1:2BS1yw61VN34WI0/nUYoInFvBc3Zcuf84d4ESiAAl68= -github.com/vmware/govmomi v0.28.0 h1:VgeQ/Rvz79U9G8QIKLdgpsN9AndHJL+5iMJLgYIrBGI= -github.com/vmware/govmomi v0.28.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= -github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= +github.com/vmware/go-vcloud-director/v2 v2.17.0 h1:msrrtEKD7H/e3cNPaXlCkZf3TMzSSyH306EXettv0c8= +github.com/vmware/go-vcloud-director/v2 v2.17.0/go.mod h1:KjnB8t5l1bRrc+jLKDJbx0vZLRzz2RPzNQ7xzg7yI3o= +github.com/vmware/govmomi v0.29.0 h1:SHJQ7DUc4fltFZv16znJNGHR1/XhiDK5iKxm2OqwkuU= +github.com/vmware/govmomi v0.29.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -958,97 +737,63 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= -go.anx.io/go-anxcloud v0.4.4 h1:lnsF2H0xad7qbhxHl4wnExKwkaLvOP500SS/V5HnyxU= -go.anx.io/go-anxcloud v0.4.4/go.mod h1:rzQ48vxTWBgS62zNvaJlVfqZfySBBhNcY++rR+MVrPI= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= -go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q= -go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= -go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= -go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.anx.io/go-anxcloud v0.4.6 h1:aHf5i3UCpt+wmuP07saGGWiqS5eqQbdw0SJVpae8Hqs= +go.anx.io/go-anxcloud v0.4.6/go.mod h1:OSSNrMPa/zV9Bdpx2IdN8iIYm3fJOATR/cvwH6R3OFc= go.keploy.io/server v0.1.8 h1:b50vAt1+WKMscYVP5Bm8gx/iSaR7mpHox8VpaxjrQ88= go.keploy.io/server v0.1.8/go.mod h1:ZqhwTZOBb+dzx5t30Wt6eUGI6kO5QizvPg6coNPtbow= -go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.8.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.mongodb.org/mongo-driver v1.8.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= -go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= -go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= -go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.4 h1:PRXhsszxTt5bbPriTjmaweWUsAnJYeWBhUMLRetUgBU= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.4/go.mod h1:05eWWy6ZWzmpeImD3UowLTB3VjDMU1yxQ+ENuVWDM3c= +go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= +go.opentelemetry.io/otel v1.11.1 h1:4WLLAmcfkmDk2ukNXJyq3/kiz/3UzCaYq6PskJsaou4= +go.opentelemetry.io/otel v1.11.1/go.mod h1:1nNhXBbWSD0nsL38H6btgnFN2k4i0sNLHNNMZMSbUGE= +go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= +go.opentelemetry.io/otel/trace v1.11.1 h1:ofxdnzsNrGBYXbP7t7zpUK281+go5rF7dvdIZXF8gdQ= +go.opentelemetry.io/otel/trace v1.11.1/go.mod h1:f/Q9G7vzk5u91PhbmKbg1Qn0rzH1LJ4vbPHFGkTPtOk= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= +go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -1057,8 +802,10 @@ golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1096,15 +843,12 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1115,9 +859,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1136,7 +878,6 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -1150,13 +891,18 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1168,7 +914,6 @@ golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= @@ -1176,8 +921,10 @@ golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb h1:8tDJ3aechhddbdPAxpycgXHJRMLpk/Ab+aa4OgdN5/g= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.2.0 h1:GtQkldQ9m7yvzCL1V+LrYow3Khe0eJH0w7RbX/VbaIU= +golang.org/x/oauth2 v0.2.0/go.mod h1:Cwn6afJ8jrQwYMxQDTpISoXmXW9I6qF6vDeuuoX3Ibs= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1188,41 +935,33 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1243,7 +982,6 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1256,10 +994,8 @@ golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1273,6 +1009,7 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211015200801-69063c4bb744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1284,14 +1021,20 @@ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= +golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= -golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1300,46 +1043,34 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180810170437-e96c4e24768d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE= +golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1347,7 +1078,6 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1375,6 +1105,7 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= @@ -1385,16 +1116,18 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1416,7 +1149,6 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4ylg3I= google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= @@ -1430,10 +1162,14 @@ google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tD google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0 h1:ExR2D+5TYIrMphWgs5JCgwRhEDlPDXXrLwHHMgPHTXE= google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= +google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= +google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= +google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= @@ -1446,8 +1182,6 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190708153700-3bdd9d9f5532/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -1473,11 +1207,10 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201026171402-d4b8fe4fd877/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201102152239-715cce707fb0/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1486,10 +1219,9 @@ google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= @@ -1505,6 +1237,7 @@ google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= @@ -1518,18 +1251,22 @@ google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2 google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9 h1:XGQ6tc+EnM35IAazg4y6AHmUg4oK8NXsXaILte1vRlk= google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= +google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -1539,7 +1276,6 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= @@ -1553,11 +1289,17 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/grpc/examples v0.0.0-20210728214646-ad0a2a847cdf/go.mod h1:bF8wuZSAZTcbF7ZPKrDI/qY52toTP/yxLpRRY4Eu9Js= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1571,8 +1313,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1582,30 +1325,20 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= -gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= -gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= -gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= -gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1621,11 +1354,6 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v0.0.0-20181223230014-1083505acf35/go.mod h1:R//lfYlUuTOTfblYI3lGoAAAebUdzjvbmQsuB7Ykd90= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1636,71 +1364,72 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= -k8s.io/api v0.25.0 h1:H+Q4ma2U/ww0iGB78ijZx6DRByPz6/733jIuFpX70e0= -k8s.io/api v0.25.0/go.mod h1:ttceV1GyV1i1rnmvzT3BST08N6nGt+dudGrquzVQWPk= -k8s.io/apiextensions-apiserver v0.24.2 h1:/4NEQHKlEz1MlaK/wHT5KMKC9UKYz6NZz6JE6ov4G6k= -k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZfJiGOW6mm2hLQ= +k8s.io/api v0.25.4 h1:3YO8J4RtmG7elEgaWMb4HgmpS2CfY1QlaOz9nwB+ZSs= +k8s.io/api v0.25.4/go.mod h1:IG2+RzyPQLllQxnhzD8KQNEu4c4YvyDTpSMztf4A0OQ= +k8s.io/apiextensions-apiserver v0.25.4 h1:7hu9pF+xikxQuQZ7/30z/qxIPZc2J1lFElPtr7f+B6U= +k8s.io/apiextensions-apiserver v0.25.4/go.mod h1:bkSGki5YBoZWdn5pWtNIdGvDrrsRWlmnvl9a+tAw5vQ= k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apimachinery v0.25.0 h1:MlP0r6+3XbkUG2itd6vp3oxbtdQLQI94fD5gCS+gnoU= -k8s.io/apimachinery v0.25.0/go.mod h1:qMx9eAk0sZQGsXGu86fab8tZdffHbwUfsvzqKn4mfB0= -k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI= -k8s.io/client-go v0.25.0 h1:CVWIaCETLMBNiTUta3d5nzRbXvY5Hy9Dpl+VvREpu5E= -k8s.io/client-go v0.25.0/go.mod h1:lxykvypVfKilxhTklov0wz1FoaUZ8X4EwbhS6rpRfN8= +k8s.io/apimachinery v0.25.4 h1:CtXsuaitMESSu339tfhVXhQrPET+EiWnIY1rcurKnAc= +k8s.io/apimachinery v0.25.4/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= +k8s.io/client-go v0.25.4 h1:3RNRDffAkNU56M/a7gUfXaEzdhZlYhoW8dgViGy5fn8= +k8s.io/client-go v0.25.4/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw= k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/code-generator v0.24.2/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= -k8s.io/component-base v0.24.2 h1:kwpQdoSfbcH+8MPN4tALtajLDfSfYxBDYlXobNWI6OU= -k8s.io/component-base v0.24.2/go.mod h1:ucHwW76dajvQ9B7+zecZAP3BVqvrHoOxm8olHEg0nmM= +k8s.io/component-base v0.25.4 h1:n1bjg9Yt+G1C0WnIDJmg2fo6wbEU1UGMRiQSjmj7hNQ= +k8s.io/component-base v0.25.4/go.mod h1:nnZJU8OP13PJEm6/p5V2ztgX2oyteIaAGKGMYb2L2cY= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 h1:MQ8BAZPZlWk3S9K4a9NCkIFQtZShWqoha7snGixVgEA= k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= -k8s.io/kubelet v0.24.2 h1:VAvULig8RiylCtyxudgHV7nhKsLnNIrdVBCRD4bXQ3Y= -k8s.io/kubelet v0.24.2/go.mod h1:Xm9DkWQjwOs+uGOUIIGIPMvvmenvj0lDVOErvIKOOt0= +k8s.io/kube-openapi v0.0.0-20221123214604-86e75ddd809a h1:UR2YSPKAb8j3uL2yK8V+t2ElG4RoBxhJTxa5gg0ZtSo= +k8s.io/kube-openapi v0.0.0-20221123214604-86e75ddd809a/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/kubelet v0.25.4 h1:24MmTTQGBHr08UkMYFC/RaLjuiMREM53HfRgJKWRquI= +k8s.io/kubelet v0.25.4/go.mod h1:dWAxzvWR7B6LrSgE+6H6Dc7bOzNOzm+O+W6zLic9daA= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v0.58.0 h1:qeNeRtD6AIJ5WVJuRXajmmXtnrO5dYchy+hpCm6QwhE= kubevirt.io/api v0.58.0/go.mod h1:U0CQlZR0JoJCaC+Va0wz4dMOtYDdVywJ98OT1KmOkzI= -kubevirt.io/containerized-data-importer-api v1.55.0 h1:IQNc8PYVq1cTwKNPEJza5xSlcnXeYVNt76M5kZ8X7xo= -kubevirt.io/containerized-data-importer-api v1.55.0/go.mod h1:92HiQEyzPoeMiCbgfG5Qe10JQVbtWMZOXucy56dKdGg= -kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= -kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= +kubevirt.io/containerized-data-importer-api v1.55.1 h1:2WJdHrbN7pOTX1KkXKME94PG8i0Shd0DK0/3jP07d/E= +kubevirt.io/containerized-data-importer-api v1.55.1/go.mod h1:92HiQEyzPoeMiCbgfG5Qe10JQVbtWMZOXucy56dKdGg= +kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 h1:fZYvD3/Vnitfkx6IJxjLAk8ugnZQ7CXVYcRfkSKmuZY= +kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= +mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= -sigs.k8s.io/controller-runtime v0.12.1 h1:4BJY01xe9zKQti8oRjj/NeHKRXthf1YkYJAgLONFFoI= -sigs.k8s.io/controller-runtime v0.12.1/go.mod h1:BKhxlA4l7FPK4AQcsuL4X6vZeWnKDXez/vp1Y8dxTU0= +sigs.k8s.io/controller-runtime v0.13.1 h1:tUsRCSJVM1QQOOeViGeX3GMT3dQF1eePPw6sEE3xSlg= +sigs.k8s.io/controller-runtime v0.13.1/go.mod h1:Zbz+el8Yg31jubvAEyglRZGdLAjplZl+PgtYNI6WNTI= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 9d1472868..425be29a7 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -469,19 +469,19 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if err != nil { return nil, nil, nil, err } - c.SpotMaxPrice = pointer.StringPtr(maxPrice) + c.SpotMaxPrice = pointer.String(maxPrice) persistentRequest, _, err := p.configVarResolver.GetConfigVarBoolValue(rawConfig.SpotInstanceConfig.PersistentRequest) if err != nil { return nil, nil, nil, err } - c.SpotPersistentRequest = pointer.BoolPtr(persistentRequest) + c.SpotPersistentRequest = pointer.Bool(persistentRequest) interruptionBehavior, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.SpotInstanceConfig.InterruptionBehavior) if err != nil { return nil, nil, nil, err } - c.SpotInterruptionBehavior = pointer.StringPtr(interruptionBehavior) + c.SpotInterruptionBehavior = pointer.String(interruptionBehavior) } assumeRoleARN, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AssumeRoleARN, "AWS_ASSUME_ROLE_ARN") if err != nil { @@ -761,7 +761,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, DeleteOnTermination: aws.Bool(true), VolumeType: config.DiskType, Iops: config.DiskIops, - Encrypted: pointer.BoolPtr(config.EBSVolumeEncrypted), + Encrypted: pointer.Bool(config.EBSVolumeEncrypted), }, }, }, diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index e5f5838ff..dba1c5bed 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -175,19 +175,19 @@ var imageReferences = map[providerconfigtypes.OperatingSystem]compute.ImageRefer var osPlans = map[providerconfigtypes.OperatingSystem]*compute.Plan{ providerconfigtypes.OperatingSystemFlatcar: { - Name: pointer.StringPtr("stable"), - Publisher: pointer.StringPtr("kinvolk"), - Product: pointer.StringPtr("flatcar-container-linux"), + Name: pointer.String("stable"), + Publisher: pointer.String("kinvolk"), + Product: pointer.String("flatcar-container-linux"), }, providerconfigtypes.OperatingSystemRHEL: { - Name: pointer.StringPtr("rhel-lvm85"), - Publisher: pointer.StringPtr("redhat"), - Product: pointer.StringPtr("rhel-byos"), + Name: pointer.String("rhel-lvm85"), + Publisher: pointer.String("redhat"), + Product: pointer.String("rhel-byos"), }, providerconfigtypes.OperatingSystemRockyLinux: { - Name: pointer.StringPtr("rocky-linux-8-5"), - Publisher: pointer.StringPtr("procomputers"), - Product: pointer.StringPtr("rocky-linux-8-5"), + Name: pointer.String("rocky-linux-8-5"), + Publisher: pointer.String("procomputers"), + Product: pointer.String("rocky-linux-8-5"), }, } @@ -361,18 +361,18 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p if rawCfg.ImagePlan != nil && rawCfg.ImagePlan.Name != "" { c.ImagePlan = &compute.Plan{ - Name: pointer.StringPtr(rawCfg.ImagePlan.Name), - Publisher: pointer.StringPtr(rawCfg.ImagePlan.Publisher), - Product: pointer.StringPtr(rawCfg.ImagePlan.Product), + Name: pointer.String(rawCfg.ImagePlan.Name), + Publisher: pointer.String(rawCfg.ImagePlan.Publisher), + Product: pointer.String(rawCfg.ImagePlan.Product), } } if rawCfg.ImageReference != nil { c.ImageReference = &compute.ImageReference{ - Publisher: pointer.StringPtr(rawCfg.ImageReference.Publisher), - Offer: pointer.StringPtr(rawCfg.ImageReference.Offer), - Sku: pointer.StringPtr(rawCfg.ImageReference.Sku), - Version: pointer.StringPtr(rawCfg.ImageReference.Version), + Publisher: pointer.String(rawCfg.ImageReference.Publisher), + Offer: pointer.String(rawCfg.ImageReference.Offer), + Sku: pointer.String(rawCfg.ImageReference.Sku), + Version: pointer.String(rawCfg.ImageReference.Version), } } @@ -543,7 +543,7 @@ func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) } if config.OSDiskSize != 0 { sp.OsDisk = &compute.OSDisk{ - DiskSizeGB: pointer.Int32Ptr(config.OSDiskSize), + DiskSizeGB: pointer.Int32(config.OSDiskSize), CreateOption: compute.DiskCreateOptionTypesFromImage, } @@ -559,7 +559,7 @@ func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) { // this should be in range 0-63 and should be unique per datadisk, since we have only one datadisk, this should be fine Lun: new(int32), - DiskSizeGB: pointer.Int32Ptr(config.DataDiskSize), + DiskSizeGB: pointer.Int32(config.DataDiskSize), CreateOption: compute.DiskCreateOptionTypesEmpty, }, } diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index e85bc2658..9850f3603 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -645,7 +645,7 @@ func (p *provider) newVirtualMachine(ctx context.Context, c *Config, pc *provide Labels: labels, }, Spec: kubevirtv1.VirtualMachineSpec{ - Running: utilpointer.BoolPtr(true), + Running: utilpointer.Bool(true), Instancetype: c.Instancetype, Preference: c.Preference, Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ @@ -820,7 +820,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. }, Spec: cdiv1beta1.DataVolumeSpec{ PVC: &corev1.PersistentVolumeClaimSpec{ - StorageClassName: utilpointer.StringPtr(config.StorageClassName), + StorageClassName: utilpointer.String(config.StorageClassName), AccessModes: []corev1.PersistentVolumeAccessMode{ "ReadWriteOnce", }, @@ -839,7 +839,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. }, Spec: cdiv1beta1.DataVolumeSpec{ PVC: &corev1.PersistentVolumeClaimSpec{ - StorageClassName: utilpointer.StringPtr(sd.StorageClassName), + StorageClassName: utilpointer.String(sd.StorageClassName), AccessModes: []corev1.PersistentVolumeAccessMode{ "ReadWriteOnce", }, diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index 672aa7635..5b4583daa 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -240,13 +240,13 @@ func TestCreateServer(t *testing.T) { }, { name: "Custom disk size", - specConf: openstackProviderSpecConf{RootDiskSizeGB: pointer.Int32Ptr(10)}, + specConf: openstackProviderSpecConf{RootDiskSizeGB: pointer.Int32(10)}, userdata: "fake-userdata", wantServerReq: expectedBlockDeviceBootRequest, }, { name: "Custom disk type", - specConf: openstackProviderSpecConf{RootDiskSizeGB: pointer.Int32Ptr(10), RootDiskVolumeType: "ssd"}, + specConf: openstackProviderSpecConf{RootDiskSizeGB: pointer.Int32(10), RootDiskVolumeType: "ssd"}, userdata: "fake-userdata", wantServerReq: expectedBlockDeviceBootVolumeTypeRequest, }, diff --git a/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go b/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go index 8414bacc2..86d505036 100644 --- a/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go +++ b/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go @@ -78,7 +78,7 @@ func TestCloudConfigToString(t *testing.T) { }, LoadBalancer: LoadBalancerOpts{ ManageSecurityGroups: true, - UseOctavia: pointer.BoolPtr(true), + UseOctavia: pointer.Bool(true), }, Version: "1.10.0", }, @@ -102,7 +102,7 @@ func TestCloudConfigToString(t *testing.T) { }, LoadBalancer: LoadBalancerOpts{ ManageSecurityGroups: true, - UseOctavia: pointer.BoolPtr(false), + UseOctavia: pointer.Bool(false), }, Version: "1.10.0", }, diff --git a/pkg/cloudprovider/provider/vsphere/provider_test.go b/pkg/cloudprovider/provider/vsphere/provider_test.go index 481d030e7..3c28e66bf 100644 --- a/pkg/cloudprovider/provider/vsphere/provider_test.go +++ b/pkg/cloudprovider/provider/vsphere/provider_test.go @@ -92,7 +92,7 @@ func TestValidate(t *testing.T) { { name: "Valid Datastore", args: vsphereProviderSpecConf{ - Datastore: pointer.StringPtr("LocalDS_0"), + Datastore: pointer.String("LocalDS_0"), }, getConfigErr: nil, wantErr: false, @@ -100,8 +100,8 @@ func TestValidate(t *testing.T) { { name: "Valid Datastore end empty DatastoreCluster", args: vsphereProviderSpecConf{ - Datastore: pointer.StringPtr("LocalDS_0"), - DatastoreCluster: pointer.StringPtr(""), + Datastore: pointer.String("LocalDS_0"), + DatastoreCluster: pointer.String(""), }, getConfigErr: nil, wantErr: false, @@ -109,7 +109,7 @@ func TestValidate(t *testing.T) { { name: "Valid DatastoreCluster", args: vsphereProviderSpecConf{ - DatastoreCluster: pointer.StringPtr("DC0_POD0"), + DatastoreCluster: pointer.String("DC0_POD0"), }, getConfigErr: nil, wantErr: false, @@ -117,7 +117,7 @@ func TestValidate(t *testing.T) { { name: "Invalid Datastore", args: vsphereProviderSpecConf{ - Datastore: pointer.StringPtr("LocalDS_10"), + Datastore: pointer.String("LocalDS_10"), }, getConfigErr: nil, wantErr: true, @@ -125,7 +125,7 @@ func TestValidate(t *testing.T) { { name: "Invalid DatastoreCluster", args: vsphereProviderSpecConf{ - Datastore: pointer.StringPtr("DC0_POD10"), + Datastore: pointer.String("DC0_POD10"), }, getConfigErr: nil, wantErr: true, @@ -133,8 +133,8 @@ func TestValidate(t *testing.T) { { name: "Both Datastore and DatastoreCluster specified", args: vsphereProviderSpecConf{ - Datastore: pointer.StringPtr("DC0_POD10"), - DatastoreCluster: pointer.StringPtr("DC0_POD0"), + Datastore: pointer.String("DC0_POD10"), + DatastoreCluster: pointer.String("DC0_POD0"), }, getConfigErr: nil, wantErr: true, diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index 5259fc4f0..da364eb4a 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -217,10 +217,10 @@ func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGate ClientCAFile: "/etc/kubernetes/pki/ca.crt", }, Webhook: kubeletv1b1.KubeletWebhookAuthentication{ - Enabled: pointer.BoolPtr(true), + Enabled: pointer.Bool(true), }, Anonymous: kubeletv1b1.KubeletAnonymousAuthentication{ - Enabled: pointer.BoolPtr(false), + Enabled: pointer.Bool(false), }, }, Authorization: kubeletv1b1.KubeletAuthorization{ @@ -292,7 +292,7 @@ func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGate // Instead of breaking the workflow, just print a warning and skip the configuration klog.Warningf("Skipping invalid ContainerLogMaxSize value %v for Kubelet configuration", containerLogMaxFiles) } else { - cfg.ContainerLogMaxFiles = pointer.Int32Ptr(int32(maxFiles)) + cfg.ContainerLogMaxFiles = pointer.Int32(int32(maxFiles)) } } diff --git a/pkg/userdata/sles/provider_test.go b/pkg/userdata/sles/provider_test.go index cdfc37553..e4ff9bd38 100644 --- a/pkg/userdata/sles/provider_test.go +++ b/pkg/userdata/sles/provider_test.go @@ -275,7 +275,7 @@ func TestUserDataGeneration(t *testing.T) { providerSpec: &providerconfigtypes.Config{ CloudProvider: "openstack", SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: pointer.StringPtr("custom\ncloud\nconfig"), + OverwriteCloudConfig: pointer.String("custom\ncloud\nconfig"), }, spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -301,7 +301,7 @@ func TestUserDataGeneration(t *testing.T) { providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: pointer.StringPtr("custom\ncloud\nconfig"), + OverwriteCloudConfig: pointer.String("custom\ncloud\nconfig"), }, spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -327,7 +327,7 @@ func TestUserDataGeneration(t *testing.T) { providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: pointer.StringPtr("custom\ncloud\nconfig"), + OverwriteCloudConfig: pointer.String("custom\ncloud\nconfig"), }, spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{ @@ -357,7 +357,7 @@ func TestUserDataGeneration(t *testing.T) { providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: pointer.StringPtr("custom\ncloud\nconfig"), + OverwriteCloudConfig: pointer.String("custom\ncloud\nconfig"), }, spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{ From c140292fc46622c16e2ce9dab2c2411c11bf4e4e Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Mon, 5 Dec 2022 11:44:15 +0100 Subject: [PATCH 257/489] disable flatcar tests for vsphere (#1503) Signed-off-by: Moath Qasim Signed-off-by: Moath Qasim --- test/e2e/provisioning/all_e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 3716f473f..ca4ccb550 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -871,7 +871,7 @@ func getVSphereTestParams(t *testing.T) []string { func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() - selector := Not(OsSelector("sles", "amzn2", "rockylinux")) + selector := Not(OsSelector("sles", "amzn2", "rockylinux", "flatcar")) params := getVSphereTestParams(t) runScenarios(t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) From 3907af1ab47b026a08997abdf9e442cf1ca970c3 Mon Sep 17 00:00:00 2001 From: Matthias Osthues Date: Tue, 6 Dec 2022 11:54:45 +0100 Subject: [PATCH 258/489] Make CloudConfigToString a struct receiver function called String (#1505) * Make CloudConfigToString a struct receiver function called String and deprecate static function * make deprecation notice more clear on what function to use * fix godot --- .../provider/vsphere/provider.go | 2 +- .../provider/vsphere/types/cloudconfig.go | 20 +++++++++++++++++++ .../vsphere/types/cloudconfig_test.go | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index a16f58b62..7151f15ed 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -591,7 +591,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri }, } - s, err := vspheretypes.CloudConfigToString(cc) + s, err := cc.String() if err != nil { return "", "", fmt.Errorf("failed to convert the cloud-config to string: %w", err) } diff --git a/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go b/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go index 9e9f997bf..a20bcda8b 100644 --- a/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go +++ b/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go @@ -105,6 +105,26 @@ type CloudConfig struct { VirtualCenter map[string]*VirtualCenterConfig } +// String converts CloudConfig into its formatted string representation. +func (c *CloudConfig) String() (string, error) { + funcMap := sprig.TxtFuncMap() + funcMap["iniEscape"] = ini.Escape + + tpl, err := template.New("cloud-config").Funcs(funcMap).Parse(cloudConfigTpl) + if err != nil { + return "", fmt.Errorf("failed to parse the cloud config template: %w", err) + } + + buf := &bytes.Buffer{} + if err := tpl.Execute(buf, c); err != nil { + return "", fmt.Errorf("failed to execute cloud config template: %w", err) + } + + return buf.String(), nil +} + +// CloudConfigToString converts CloudConfig into its formatted string representation. +// Deprecated: use struct receiver function String() instead. func CloudConfigToString(c *CloudConfig) (string, error) { funcMap := sprig.TxtFuncMap() funcMap["iniEscape"] = ini.Escape diff --git a/pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go b/pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go index 399f31231..0f69bf4db 100644 --- a/pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go +++ b/pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go @@ -121,7 +121,7 @@ func TestCloudConfigToString(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - s, err := CloudConfigToString(test.config) + s, err := test.config.String() if err != nil { t.Fatal(err) } From d20b3d2e33cf484eb92c34418bb7e447e6ec823c Mon Sep 17 00:00:00 2001 From: Helene Durand <90329641+hdurand0710@users.noreply.github.com> Date: Tue, 6 Dec 2022 18:56:45 +0100 Subject: [PATCH 259/489] [WIP] Switch KubeVirt CI to DC (#1508) Signed-off-by: Helene Durand Signed-off-by: Helene Durand --- .prow/e2e-features.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index dd0540094..2d0f78adc 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -26,7 +26,7 @@ presubmits: preset-hetzner: "true" preset-openstack: "true" preset-vsphere: "true" - preset-kubevirt: "true" + preset-kubevirt-dc: "true" preset-alibaba: "true" preset-goproxy: "true" preset-kind-volume-mounts: "true" diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index f12951aa4..550c49201 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -19,7 +19,7 @@ presubmits: clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" max_concurrency: 1 labels: - preset-kubevirt: "true" + preset-kubevirt-dc: "true" preset-hetzner: "true" preset-e2e-ssh: "true" preset-rhel: "true" diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index 9c3eaab9b..6be2db2e7 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -38,7 +38,7 @@ spec: primaryDisk: osImage: http://image-repo.kube-system.svc.cluster.local/images/<< KUBEVIRT_OS_IMAGE >>.img size: "25Gi" - storageClassName: longhorn + storageClassName: csi-rbd dnsPolicy: "None" dnsConfig: nameservers: From 990dd416634e7d5aaf898e8289ae08a48ef8e7f8 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 8 Dec 2022 17:32:07 +0500 Subject: [PATCH 260/489] Upgrade to Go 1.19.4 (#1514) Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 20 ++++++++++---------- .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 6 +++--- .prow/verify.yaml | 8 ++++---- Dockerfile | 2 +- Makefile | 2 +- examples/operating-system-manager.yaml | 4 ++-- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- 23 files changed, 45 insertions(+), 45 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 2d0f78adc..a9a4a8461 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -91,7 +91,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -118,7 +118,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 2622cf92f..4145a8407 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index a9d76b58d..ac6dec341 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index dbf5a22bc..daaec94b8 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index f325d0cf0..2578a5a18 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -57,7 +57,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -88,7 +88,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -116,7 +116,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -144,7 +144,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -173,7 +173,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -201,7 +201,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -229,7 +229,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -257,7 +257,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -285,7 +285,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 5377f5656..06b592cd9 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -57,7 +57,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -87,7 +87,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 23a7d89a8..717a87f1c 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index d2cb8d2ca..6a0bcb985 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index ec19ebd9a..8434634f7 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 950a799ca..4ee6ddce3 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 550c49201..e04e83839 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 2a7b6b0a0..8d8edac3c 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 1e5832317..f669a460f 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index d112d4464..87774f7f0 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -57,7 +57,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index ff2c6a48a..a238fdd70 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 0a6511dc7..a256bda14 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 87c182bd8..452d24c01 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -57,7 +57,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -86,7 +86,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 8584ef439..6aad15648 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.19.3 + - image: golang:1.19.4 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.19.3 + - image: golang:1.19.4 command: - make args: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.19.3 + - image: golang:1.19.4 command: - make args: diff --git a/Dockerfile b/Dockerfile index 157690265..380cbde28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.19.3 +ARG GO_VERSION=1.19.4 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index 80b57afed..4118c4697 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.19.3 +GO_VERSION ?= 1.19.4 GOOS ?= $(shell go env GOOS) diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index 545070324..211412a85 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -975,7 +975,7 @@ spec: serviceAccountName: operating-system-manager-webhook containers: # TODO: Update this to a semver tag before release. - - image: quay.io/kubermatic/operating-system-manager:192412d78cbfb9d826fbc7cf7e077cfe7629d6ba + - image: quay.io/kubermatic/operating-system-manager:4f7c5a6873538e922afd70bff41850795657f313 imagePullPolicy: IfNotPresent name: webhook command: @@ -1304,7 +1304,7 @@ spec: serviceAccountName: operating-system-manager containers: # TODO: Update this to a semver tag before release. - - image: quay.io/kubermatic/operating-system-manager:192412d78cbfb9d826fbc7cf7e077cfe7629d6ba + - image: quay.io/kubermatic/operating-system-manager:4f7c5a6873538e922afd70bff41850795657f313 imagePullPolicy: IfNotPresent name: operating-system-manager command: diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 9422adf73..b17ff3596 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.19.3 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.19.4 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 470109588..7d71c7113 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-4 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 containerize ./hack/verify-licenses.sh go mod vendor From d40a14dc25a8d3f08771883eb7a1db95e6e8ad78 Mon Sep 17 00:00:00 2001 From: Helene Durand <90329641+hdurand0710@users.noreply.github.com> Date: Thu, 8 Dec 2022 14:29:05 +0100 Subject: [PATCH 261/489] KubeVirt cleanup switch CI to DC (#1515) Signed-off-by: Helene Durand Signed-off-by: Helene Durand --- .prow/e2e-features.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index a9a4a8461..4515e5b39 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -26,7 +26,7 @@ presubmits: preset-hetzner: "true" preset-openstack: "true" preset-vsphere: "true" - preset-kubevirt-dc: "true" + preset-kubevirt: "true" preset-alibaba: "true" preset-goproxy: "true" preset-kind-volume-mounts: "true" diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index e04e83839..504d72599 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -19,7 +19,7 @@ presubmits: clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" max_concurrency: 1 labels: - preset-kubevirt-dc: "true" + preset-kubevirt: "true" preset-hetzner: "true" preset-e2e-ssh: "true" preset-rhel: "true" From e7e4ba8eb535086374108ed235c2e33bab7f4292 Mon Sep 17 00:00:00 2001 From: Sankalp Rangare Date: Fri, 16 Dec 2022 10:20:07 +0100 Subject: [PATCH 262/489] Add validation for image-cloning and custom-images for kubevirt (#1517) Signed-off-by: Sankalp Rangare Signed-off-by: Sankalp Rangare --- .../provider/kubevirt/provider.go | 115 +++++++++++++++- .../provider/kubevirt/provider_test.go | 123 ++++++++++++++++++ 2 files changed, 231 insertions(+), 7 deletions(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 9850f3603..92415b521 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -58,6 +58,9 @@ func init() { if err := kubevirtv1.AddToScheme(scheme.Scheme); err != nil { klog.Fatalf("failed to add kubevirtv1 to scheme: %v", err) } + if err := cdiv1beta1.AddToScheme(scheme.Scheme); err != nil { + klog.Fatalf("failed to add cdiv1beta1 to scheme: %v", err) + } } type imageSource string @@ -72,15 +75,24 @@ const ( httpSource imageSource = "http" // pvcSource defines the pvc source type for VM Disk Image. pvcSource imageSource = "pvc" + // kubeVirtImagesNamespace namespace contains globally available custom images and cached standard images. + kubeVirtImagesNamespace = "kubevirt-images" + dataVolumeStandardImageAnnotation = "kubevirt-initialization.k8c.io/standard-image" + osAnnotationForCustomDisk = "cdi.kubevirt.io/os-type" ) -var supportedOS = map[providerconfigtypes.OperatingSystem]*struct{}{ - providerconfigtypes.OperatingSystemCentOS: nil, - providerconfigtypes.OperatingSystemUbuntu: nil, - providerconfigtypes.OperatingSystemRHEL: nil, - providerconfigtypes.OperatingSystemFlatcar: nil, - providerconfigtypes.OperatingSystemRockyLinux: nil, -} +var ( + supportedOS = map[providerconfigtypes.OperatingSystem]*struct{}{ + providerconfigtypes.OperatingSystemCentOS: nil, + providerconfigtypes.OperatingSystemUbuntu: nil, + providerconfigtypes.OperatingSystemRHEL: nil, + providerconfigtypes.OperatingSystemFlatcar: nil, + providerconfigtypes.OperatingSystemRockyLinux: nil, + } + errInvalidOsImage = fmt.Errorf("invalid primaryDisk.osImage") + errCustomImage = fmt.Errorf("custom-image cloning not allowed") + errStandardImage = fmt.Errorf("standard-image cloning not allowed") +) type provider struct { configVarResolver *providerconfig.ConfigVarResolver @@ -108,6 +120,8 @@ type Config struct { SecondaryDisks []SecondaryDisks NodeAffinityPreset NodeAffinityPreset TopologySpreadConstraints []corev1.TopologySpreadConstraint + AllowPVCClone bool + AllowCustomImages bool } type AffinityType string @@ -317,6 +331,16 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf(`failed to parse "topologySpreadConstraints" field: %w`, err) } + config.AllowPVCClone, err = isImageCloningAllowed() + if err != nil { + return nil, nil, fmt.Errorf(`failed to parse "KUBEVIRT_ALLOW_PVC_CLONE" environment variable: %w`, err) + } + + config.AllowCustomImages, err = isCustomImageAllowed() + if err != nil { + return nil, nil, fmt.Errorf(`failed to parse "KUBEVIRT_ALLOW_CUSTOM_IMAGES" environment variable: %w`, err) + } + return &config, pconfig, nil } @@ -412,6 +436,34 @@ func getNamespace() string { return ns } +// isImageCloningAllowed returns whether image-cloning is allowed or not. +// Default value is `true`. +func isImageCloningAllowed() (bool, error) { + value := os.Getenv("KUBEVIRT_ALLOW_PVC_CLONE") + if value == "" { + return true, nil + } + isImageCloningEnabled, err := strconv.ParseBool(value) + if err != nil { + return false, err + } + return isImageCloningEnabled, nil +} + +// isCustomImageAllowed returns whether custom-image for cloning is allowed or not. +// Default value is `true`. +func isCustomImageAllowed() (bool, error) { + value := os.Getenv("KUBEVIRT_ALLOW_CUSTOM_IMAGES") + if value == "" { + return true, nil + } + isCustomImagesEnabled, err := strconv.ParseBool(value) + if err != nil { + return false, err + } + return isCustomImagesEnabled, nil +} + func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { @@ -503,6 +555,9 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return fmt.Errorf("failed to request VirtualMachineInstances: %w", err) } + if c.OSImageSource.PVC != nil { + return validateOsImage(ctx, c, sigClient) + } return nil } @@ -920,3 +975,49 @@ func getTopologySpreadConstraints(config *Config, matchLabels map[string]string) }, } } + +// validateOsImage with PVC as source. +func validateOsImage(ctx context.Context, c *Config, sigClient client.Client) error { + switch c.OSImageSource.PVC.Namespace { + case c.Namespace: + if !c.AllowCustomImages { + return errCustomImage + } + + case kubeVirtImagesNamespace: + existingDiskList := cdiv1beta1.DataVolumeList{} + listOption := client.ListOptions{ + Namespace: kubeVirtImagesNamespace, + } + if err := sigClient.List(ctx, &existingDiskList, &listOption); client.IgnoreNotFound(err) != nil { + return fmt.Errorf("failed to request DataVolumeList: %w", err) + } + return validateKubeVirtImages(c.OSImageSource.PVC.Name, existingDiskList, c) + + default: + return errInvalidOsImage + } + return nil +} + +// validateKubeVirtImages from kubeVirtImagesNamespace. +func validateKubeVirtImages(sourcePVC string, existingDiskList cdiv1beta1.DataVolumeList, config *Config) error { + for _, existingDV := range existingDiskList.Items { + if sourcePVC == existingDV.Name { + if existingDV.Annotations[dataVolumeStandardImageAnnotation] == "true" { + if !config.AllowPVCClone { + return errStandardImage + } + return nil + } + if existingDV.Annotations[osAnnotationForCustomDisk] != "" { + if !config.AllowCustomImages { + return errCustomImage + } + return nil + } + break + } + } + return errInvalidOsImage +} diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index 261ac347e..ec2721df0 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -20,12 +20,14 @@ import ( "bytes" "context" "embed" + "errors" "html/template" "path" "reflect" "testing" kubevirtv1 "kubevirt.io/api/core/v1" + cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" cloudprovidertesting "github.com/kubermatic/machine-controller/pkg/cloudprovider/testing" "github.com/kubermatic/machine-controller/pkg/providerconfig" @@ -36,6 +38,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/client-go/kubernetes/scheme" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -320,3 +323,123 @@ func TestTopologySpreadConstraint(t *testing.T) { }) } } + +func TestValidateOsImage(t *testing.T) { + testClient := fakectrlruntimeclient. + NewClientBuilder(). + WithScheme(scheme.Scheme). + WithObjects(&cdiv1beta1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "standardDV", + Namespace: kubeVirtImagesNamespace, + Annotations: map[string]string{dataVolumeStandardImageAnnotation: "true"}}, + }, + &cdiv1beta1.DataVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "customDVByAdmin", + Namespace: kubeVirtImagesNamespace, + Annotations: map[string]string{osAnnotationForCustomDisk: "ubuntu"}}, + }, + ).Build() + + tests := []struct { + desc string + config Config + expectedErr error + }{ + { + desc: "valid osImage with cloned standard DataVolume as pvc source, cloning enabled", + config: Config{ + OSImageSource: &cdiv1beta1.DataVolumeSource{ + PVC: &cdiv1beta1.DataVolumeSourcePVC{ + Name: "standardDV", + Namespace: kubeVirtImagesNamespace, + }, + }, + AllowPVCClone: true, + }, + expectedErr: nil, + }, + { + desc: "valid osImage with cloned standard DataVolume as pvc source, cloning disabled", + config: Config{ + OSImageSource: &cdiv1beta1.DataVolumeSource{ + PVC: &cdiv1beta1.DataVolumeSourcePVC{ + Name: "standardDV", + Namespace: kubeVirtImagesNamespace, + }, + }, + AllowPVCClone: false, + }, + expectedErr: errStandardImage, + }, + { + desc: "valid osImage with custom-image-by-admin as pvc source, custom-images enabled", + config: Config{ + OSImageSource: &cdiv1beta1.DataVolumeSource{ + PVC: &cdiv1beta1.DataVolumeSourcePVC{ + Name: "customDVByAdmin", + Namespace: kubeVirtImagesNamespace, + }, + }, + AllowCustomImages: true, + }, + expectedErr: nil, + }, + { + desc: "valid osImage with custom-image-by-admin as pvc source, custom-images disabled", + config: Config{ + OSImageSource: &cdiv1beta1.DataVolumeSource{ + PVC: &cdiv1beta1.DataVolumeSourcePVC{ + Name: "customDVByAdmin", + Namespace: kubeVirtImagesNamespace, + }, + }, + AllowCustomImages: false, + }, + expectedErr: errCustomImage, + }, + { + desc: "valid osImage with custom-image-by-user as pvc source, custom-images disabled", + config: Config{ + Namespace: "cluster-test", + OSImageSource: &cdiv1beta1.DataVolumeSource{ + PVC: &cdiv1beta1.DataVolumeSourcePVC{ + Name: "customDVByUser", + Namespace: "cluster-test", + }, + }, + AllowCustomImages: false, + }, + expectedErr: errCustomImage, + }, + { + desc: "invalid osImage with non-existent pvc source, cloning enabled", + config: Config{ + OSImageSource: &cdiv1beta1.DataVolumeSource{ + PVC: &cdiv1beta1.DataVolumeSourcePVC{ + Name: "non-existent-DV", + Namespace: kubeVirtImagesNamespace, + }, + }, + AllowPVCClone: true, + }, + expectedErr: errInvalidOsImage, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + actualErr := validateOsImage(context.Background(), &test.config, testClient) + if test.expectedErr != nil { + if !errors.Is(actualErr, test.expectedErr) { + t.Errorf("expected error: %q, got: %q", test.expectedErr, actualErr) + } + } else { + if actualErr != nil { + t.Errorf("expected success, but got error: %q", actualErr) + } + } + }) + } +} From f391c61dad30fdf799a090a177b99a27d8c7ac59 Mon Sep 17 00:00:00 2001 From: Alex Stockinger Date: Wed, 21 Dec 2022 11:14:09 +0100 Subject: [PATCH 263/489] Allow manually configuring containerd version via CLI flag (#1518) * Allow manually configuring containerd version via CLI flag Signed-off-by: Alex Stockinger * Also configure docker's containerd version via CLI flag Signed-off-by: Alex Stockinger Signed-off-by: Alex Stockinger --- cmd/machine-controller/main.go | 3 +++ pkg/containerruntime/config.go | 2 ++ pkg/containerruntime/containerd.go | 10 +++++----- pkg/containerruntime/containerruntime.go | 9 +++++++++ pkg/containerruntime/docker.go | 15 ++++++++++----- 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 171327cc5..2a1b0a1a3 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -84,6 +84,7 @@ var ( podCIDR string nodePortRange string nodeRegistryCredentialsSecret string + nodeContainerdVersion string nodeContainerdRegistryMirrors = containerruntime.RegistryMirrorsFlags{} overrideBootstrapKubeletAPIServer string ) @@ -170,6 +171,7 @@ func main() { flag.StringVar(&nodePauseImage, "node-pause-image", "", "Image for the pause container including tag. If not set, the kubelet default will be used: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/") flag.String("node-kubelet-repository", "quay.io/kubermatic/kubelet", "[NO-OP] Repository for the kubelet container. Has no effects.") flag.StringVar(&nodeContainerRuntime, "node-container-runtime", "docker", "container-runtime to deploy") + flag.StringVar(&nodeContainerdVersion, "node-containerd-version", "", "version of containerd to deploy") flag.Var(&nodeContainerdRegistryMirrors, "node-containerd-registry-mirrors", "Configure registry mirrors endpoints. Can be used multiple times to specify multiple mirrors") flag.StringVar(&caBundleFile, "ca-bundle", "", "path to a file containing all PEM-encoded CA certificates (will be used instead of the host's certificates if set)") flag.BoolVar(&nodeCSRApprover, "node-csr-approver", true, "Enable NodeCSRApprover controller to automatically approve node serving certificate requests") @@ -240,6 +242,7 @@ func main() { containerRuntimeOpts := containerruntime.Opts{ ContainerRuntime: nodeContainerRuntime, + ContainerdVersion: nodeContainerdVersion, ContainerdRegistryMirrors: nodeContainerdRegistryMirrors, InsecureRegistries: nodeInsecureRegistries, PauseImage: nodePauseImage, diff --git a/pkg/containerruntime/config.go b/pkg/containerruntime/config.go index 90bce0306..f93a54d9f 100644 --- a/pkg/containerruntime/config.go +++ b/pkg/containerruntime/config.go @@ -31,6 +31,7 @@ import ( type Opts struct { ContainerRuntime string + ContainerdVersion string InsecureRegistries string RegistryMirrors string RegistryCredentialsSecret string @@ -92,6 +93,7 @@ func BuildConfig(opts Opts) (Config, error) { withInsecureRegistries(insecureRegistries), withRegistryMirrors(opts.ContainerdRegistryMirrors), withSandboxImage(opts.PauseImage), + withContainerdVersion(opts.ContainerdVersion), ), nil } diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index 6337e04ad..69631ad68 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -27,8 +27,8 @@ import ( ) const ( - LegacyContainerdVersion = "1.4" - DefaultContainerdVersion = "1.6" + LegacyContainerdVersion = "1.4*" + DefaultContainerdVersion = "1.6*" ) type Containerd struct { @@ -123,7 +123,7 @@ runtime-endpoint: unix:///run/containerd/containerd.sock EOF yum install -y \ - containerd-{{ .ContainerdVersion }}* \ + containerd-{{ .ContainerdVersion }} \ yum-plugin-versionlock yum versionlock add containerd @@ -151,7 +151,7 @@ Restart=always EnvironmentFile=-/etc/environment EOF -yum install -y containerd.io-{{ .ContainerdVersion }}* yum-plugin-versionlock +yum install -y containerd.io-{{ .ContainerdVersion }} yum-plugin-versionlock yum versionlock add containerd.io systemctl daemon-reload @@ -175,7 +175,7 @@ Restart=always EnvironmentFile=-/etc/environment EOF -apt-get install -y --allow-downgrades containerd.io={{ .ContainerdVersion }}* +apt-get install -y --allow-downgrades containerd.io={{ .ContainerdVersion }} apt-mark hold containerd.io systemctl daemon-reload diff --git a/pkg/containerruntime/containerruntime.go b/pkg/containerruntime/containerruntime.go index 95eb55076..b7fe47979 100644 --- a/pkg/containerruntime/containerruntime.go +++ b/pkg/containerruntime/containerruntime.go @@ -57,6 +57,12 @@ func withSandboxImage(image string) Opt { } } +func withContainerdVersion(version string) Opt { + return func(cfg *Config) { + cfg.ContainerdVersion = version + } +} + func get(containerRuntimeName string, opts ...Opt) Config { cfg := Config{} @@ -88,6 +94,7 @@ type Config struct { SandboxImage string `json:",omitempty"` ContainerLogMaxFiles string `json:",omitempty"` ContainerLogMaxSize string `json:",omitempty"` + ContainerdVersion string `json:",omitempty"` } // AuthConfig is a COPY of github.com/containerd/containerd/pkg/cri/config.AuthConfig. @@ -123,6 +130,7 @@ func (cfg Config) Engine(kubeletVersion *semver.Version) Engine { containerLogMaxFiles: cfg.ContainerLogMaxFiles, containerLogMaxSize: cfg.ContainerLogMaxSize, registryCredentials: cfg.RegistryCredentials, + containerdVersion: cfg.ContainerdVersion, } containerd := &Containerd{ @@ -130,6 +138,7 @@ func (cfg Config) Engine(kubeletVersion *semver.Version) Engine { registryMirrors: cfg.RegistryMirrors, sandboxImage: cfg.SandboxImage, registryCredentials: cfg.RegistryCredentials, + version: cfg.ContainerdVersion, } moreThan124, _ := semver.NewConstraint(">= 1.24") diff --git a/pkg/containerruntime/docker.go b/pkg/containerruntime/docker.go index 173ce3f6c..398368a54 100644 --- a/pkg/containerruntime/docker.go +++ b/pkg/containerruntime/docker.go @@ -27,8 +27,8 @@ import ( ) const ( - LegacyDockerContainerdVersion = "1.4" - DefaultDockerContainerdVersion = "1.6" + LegacyDockerContainerdVersion = "1.4*" + DefaultDockerContainerdVersion = "1.6*" DefaultDockerVersion = "20.10" LegacyDockerVersion = "19.03" ) @@ -39,6 +39,7 @@ type Docker struct { containerLogMaxFiles string containerLogMaxSize string registryCredentials map[string]AuthConfig + containerdVersion string } type DockerCfgJSON struct { @@ -88,6 +89,10 @@ func (eng *Docker) ScriptFor(os types.OperatingSystem) (string, error) { ContainerdVersion: DefaultDockerContainerdVersion, } + if eng.containerdVersion != "" { + args.ContainerdVersion = eng.containerdVersion + } + switch os { case types.OperatingSystemAmazonLinux2: args.ContainerdVersion = LegacyDockerContainerdVersion @@ -126,7 +131,7 @@ EOF yum install -y \ {{- if .ContainerdVersion }} - containerd-{{ .ContainerdVersion }}* \ + containerd-{{ .ContainerdVersion }} \ {{- end }} docker-{{ .DockerVersion }}* \ yum-plugin-versionlock @@ -152,7 +157,7 @@ EOF yum install -y \ {{- if .ContainerdVersion }} docker-ce-cli-{{ .DockerVersion }}* \ - containerd.io-{{ .ContainerdVersion }}* \ + containerd.io-{{ .ContainerdVersion }} \ {{- end }} docker-ce-{{ .DockerVersion }}* \ yum-plugin-versionlock @@ -178,7 +183,7 @@ EOF apt-get install --allow-downgrades -y \ {{- if .ContainerdVersion }} - containerd.io={{ .ContainerdVersion }}* \ + containerd.io={{ .ContainerdVersion }} \ docker-ce-cli=5:{{ .DockerVersion }}* \ {{- end }} docker-ce=5:{{ .DockerVersion }}* From ce155009eb669559872d0c02817e7d1492a8b207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Tue, 27 Dec 2022 11:08:14 +0100 Subject: [PATCH 264/489] Update Kubernetes deps to v1.26.0 and controller-runtime to v0.14.1 (#1519) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić Signed-off-by: Marko Mudrinić --- go.mod | 27 ++++--- go.sum | 79 +++++++++++-------- .../machinedeployments_validation.go | 2 +- pkg/apis/cluster/v1alpha1/machineset_types.go | 2 +- 4 files changed, 60 insertions(+), 50 deletions(-) diff --git a/go.mod b/go.mod index c83267815..4dc281bf2 100644 --- a/go.mod +++ b/go.mod @@ -48,16 +48,16 @@ require ( google.golang.org/grpc v1.51.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.25.4 - k8s.io/apiextensions-apiserver v0.25.4 - k8s.io/apimachinery v0.25.4 + k8s.io/api v0.26.0 + k8s.io/apiextensions-apiserver v0.26.0 + k8s.io/apimachinery v0.26.0 k8s.io/client-go v12.0.0+incompatible k8s.io/klog v1.0.0 - k8s.io/kubelet v0.25.4 + k8s.io/kubelet v0.26.0 k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 kubevirt.io/api v0.58.0 kubevirt.io/containerized-data-importer-api v1.55.1 - sigs.k8s.io/controller-runtime v0.13.1 + sigs.k8s.io/controller-runtime v0.14.1 sigs.k8s.io/yaml v1.3.0 ) @@ -89,6 +89,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect @@ -155,14 +156,14 @@ require ( go.opentelemetry.io/otel/trace v1.11.1 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect - go.uber.org/zap v1.23.0 // indirect + go.uber.org/zap v1.24.0 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/net v0.2.0 // indirect + golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.2.0 // indirect - golang.org/x/term v0.2.0 // indirect - golang.org/x/text v0.4.0 // indirect - golang.org/x/time v0.2.0 // indirect + golang.org/x/sys v0.3.0 // indirect + golang.org/x/term v0.3.0 // indirect + golang.org/x/text v0.5.0 // indirect + golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect google.golang.org/protobuf v1.28.1 // indirect @@ -171,7 +172,7 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/component-base v0.25.4 // indirect + k8s.io/component-base v0.26.0 // indirect k8s.io/klog/v2 v2.80.1 // indirect k8s.io/kube-openapi v0.0.0-20221123214604-86e75ddd809a // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect @@ -179,4 +180,4 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) -replace k8s.io/client-go => k8s.io/client-go v0.25.4 +replace k8s.io/client-go => k8s.io/client-go v0.26.0 diff --git a/go.sum b/go.sum index c1bd5351f..a301ee9cf 100644 --- a/go.sum +++ b/go.sum @@ -74,11 +74,9 @@ github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9mo github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= -github.com/Azure/go-autorest/autorest v0.11.27/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U= github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= -github.com/Azure/go-autorest/autorest/adal v0.9.20/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 h1:P6bYXFoao05z5uhOQzbC3Qd8JqF3jUoocoTeIxkp2cA= @@ -177,6 +175,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bnkamalesh/webgo/v4 v4.1.11/go.mod h1:taIAonQTzao8G5rnB22WgKmQuIOWHpQ0n/YLAidBXlM= github.com/bnkamalesh/webgo/v6 v6.2.2/go.mod h1:2Y+dEdTp1xC/ra+3PAVZV6hh4sCI+iPK7mcHt+t9bfM= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= @@ -237,6 +237,7 @@ github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -570,7 +571,6 @@ github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLA github.com/nutanix-cloud-native/prism-go-client v0.3.4 h1:bHY3VPrHHYnbRtkpGaKK+2ZmvUjNVRC55CYZbXIfnOk= github.com/nutanix-cloud-native/prism-go-client v0.3.4/go.mod h1:tTIH02E6o6AWSShr98QChoxuZl+jBhkXFixom9+fd1Y= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -583,7 +583,9 @@ github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3 github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= -github.com/onsi/ginkgo/v2 v2.2.0 h1:3ZNA3L1c5FYDFTTxbFeVGGD8jYvjYauHD30YgLxVsNI= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= @@ -591,7 +593,10 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= -github.com/onsi/gomega v1.20.2 h1:8uQq0zMgLEfa0vRrrBgaJF2gyW9Da9BmfGV+OyUzfkY= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 h1:+ShYlGoPriGahTTFTjQ0RtNXW0srxDodk2STdc238Rk= @@ -781,8 +786,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= -go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= @@ -801,9 +806,9 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -844,6 +849,7 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -901,8 +907,9 @@ golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.2.0 h1:sZfSu1wtKLGlWI4ZZayP0ck9Y73K1ynO6gqzTdBVdPU= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 h1:Frnccbp+ok2GkUS2tC84yAq/U9Vg+0sIO7aRL3T4Xnc= +golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1028,13 +1035,15 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0 h1:z85xZCsEl7bi/KwbNADeBYoOP0++7W1ipu+aGnpwzRM= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1044,15 +1053,16 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE= -golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1118,6 +1128,7 @@ golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpd golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1335,7 +1346,6 @@ gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= @@ -1364,22 +1374,22 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= -k8s.io/api v0.25.4 h1:3YO8J4RtmG7elEgaWMb4HgmpS2CfY1QlaOz9nwB+ZSs= -k8s.io/api v0.25.4/go.mod h1:IG2+RzyPQLllQxnhzD8KQNEu4c4YvyDTpSMztf4A0OQ= -k8s.io/apiextensions-apiserver v0.25.4 h1:7hu9pF+xikxQuQZ7/30z/qxIPZc2J1lFElPtr7f+B6U= -k8s.io/apiextensions-apiserver v0.25.4/go.mod h1:bkSGki5YBoZWdn5pWtNIdGvDrrsRWlmnvl9a+tAw5vQ= +k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I= +k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= +k8s.io/apiextensions-apiserver v0.26.0 h1:Gy93Xo1eg2ZIkNX/8vy5xviVSxwQulsnUdQ00nEdpDo= +k8s.io/apiextensions-apiserver v0.26.0/go.mod h1:7ez0LTiyW5nq3vADtK6C3kMESxadD51Bh6uz3JOlqWQ= k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apimachinery v0.25.4 h1:CtXsuaitMESSu339tfhVXhQrPET+EiWnIY1rcurKnAc= -k8s.io/apimachinery v0.25.4/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= -k8s.io/client-go v0.25.4 h1:3RNRDffAkNU56M/a7gUfXaEzdhZlYhoW8dgViGy5fn8= -k8s.io/client-go v0.25.4/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw= +k8s.io/apimachinery v0.26.0 h1:1feANjElT7MvPqp0JT6F3Ss6TWDwmcjLypwoPpEf7zg= +k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= +k8s.io/client-go v0.26.0 h1:lT1D3OfO+wIi9UFolCrifbjUUgu7CpLca0AD8ghRLI8= +k8s.io/client-go v0.26.0/go.mod h1:I2Sh57A79EQsDmn7F7ASpmru1cceh3ocVT9KlX2jEZg= k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.25.4 h1:n1bjg9Yt+G1C0WnIDJmg2fo6wbEU1UGMRiQSjmj7hNQ= -k8s.io/component-base v0.25.4/go.mod h1:nnZJU8OP13PJEm6/p5V2ztgX2oyteIaAGKGMYb2L2cY= +k8s.io/component-base v0.26.0 h1:0IkChOCohtDHttmKuz+EP3j3+qKmV55rM9gIFTXA7Vs= +k8s.io/component-base v0.26.0/go.mod h1:lqHwlfV1/haa14F/Z5Zizk5QmzaVf23nQzCwVOQpfC8= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= @@ -1390,22 +1400,21 @@ k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.70.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= -k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= k8s.io/kube-openapi v0.0.0-20221123214604-86e75ddd809a h1:UR2YSPKAb8j3uL2yK8V+t2ElG4RoBxhJTxa5gg0ZtSo= k8s.io/kube-openapi v0.0.0-20221123214604-86e75ddd809a/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/kubelet v0.25.4 h1:24MmTTQGBHr08UkMYFC/RaLjuiMREM53HfRgJKWRquI= -k8s.io/kubelet v0.25.4/go.mod h1:dWAxzvWR7B6LrSgE+6H6Dc7bOzNOzm+O+W6zLic9daA= +k8s.io/kubelet v0.26.0 h1:08bDb5IoUH/1K1t2NUwnGIIWxjm9LSqn6k3FWw1tJGI= +k8s.io/kubelet v0.26.0/go.mod h1:DluF+d8jS2nE/Hs7CC3QM+OZlIEb22NTOihQ3EDwCQ4= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v0.58.0 h1:qeNeRtD6AIJ5WVJuRXajmmXtnrO5dYchy+hpCm6QwhE= @@ -1418,8 +1427,8 @@ mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.13.1 h1:tUsRCSJVM1QQOOeViGeX3GMT3dQF1eePPw6sEE3xSlg= -sigs.k8s.io/controller-runtime v0.13.1/go.mod h1:Zbz+el8Yg31jubvAEyglRZGdLAjplZl+PgtYNI6WNTI= +sigs.k8s.io/controller-runtime v0.14.1 h1:vThDes9pzg0Y+UbCPY3Wj34CGIYPgdmspPm2GIpxpzM= +sigs.k8s.io/controller-runtime v0.14.1/go.mod h1:GaRkrY8a7UZF0kqFFbUKG7n9ICiTY5T55P1RiE3UZlU= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= diff --git a/pkg/admission/machinedeployments_validation.go b/pkg/admission/machinedeployments_validation.go index 12a65c6c9..dd4a9c1d6 100644 --- a/pkg/admission/machinedeployments_validation.go +++ b/pkg/admission/machinedeployments_validation.go @@ -40,7 +40,7 @@ func validateMachineDeployment(md v1alpha1.MachineDeployment) field.ErrorList { func validateMachineDeploymentSpec(spec *v1alpha1.MachineDeploymentSpec, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} - allErrs = append(allErrs, metav1validation.ValidateLabelSelector(&spec.Selector, fldPath.Child("selector"))...) + allErrs = append(allErrs, metav1validation.ValidateLabelSelector(&spec.Selector, metav1validation.LabelSelectorValidationOptions{}, fldPath.Child("selector"))...) if len(spec.Selector.MatchLabels)+len(spec.Selector.MatchExpressions) == 0 { allErrs = append(allErrs, field.Invalid(fldPath.Child("selector"), spec.Selector, "empty selector is not valid for MachineDeployment.")) } diff --git a/pkg/apis/cluster/v1alpha1/machineset_types.go b/pkg/apis/cluster/v1alpha1/machineset_types.go index fcd0bc1be..dadf49d07 100644 --- a/pkg/apis/cluster/v1alpha1/machineset_types.go +++ b/pkg/apis/cluster/v1alpha1/machineset_types.go @@ -172,7 +172,7 @@ func (m *MachineSet) Validate() field.ErrorList { // validate spec.selector and spec.template.labels fldPath := field.NewPath("spec") - errors = append(errors, metav1validation.ValidateLabelSelector(&m.Spec.Selector, fldPath.Child("selector"))...) + errors = append(errors, metav1validation.ValidateLabelSelector(&m.Spec.Selector, metav1validation.LabelSelectorValidationOptions{}, fldPath.Child("selector"))...) if len(m.Spec.Selector.MatchLabels)+len(m.Spec.Selector.MatchExpressions) == 0 { errors = append(errors, field.Invalid(fldPath.Child("selector"), m.Spec.Selector, "empty selector is not valid for MachineSet.")) } From 0eaa75744f8306bc5ae7481d73fc21a993e99f0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Tue, 3 Jan 2023 09:32:20 +0100 Subject: [PATCH 265/489] Allow specifying full image reference for GCP (#1520) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić Signed-off-by: Marko Mudrinić --- pkg/cloudprovider/provider/gce/config.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index 79bcd81d4..e292157a7 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -25,6 +25,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "golang.org/x/oauth2/google" "golang.org/x/oauth2/jwt" @@ -279,6 +280,12 @@ func (cfg *config) diskTypeDescriptor() string { // for the source image of an instance boot disk. func (cfg *config) sourceImageDescriptor() (string, error) { if cfg.customImage != "" { + // If a full image identifier is provided, use it + if strings.HasPrefix("projects/", cfg.customImage) { + return cfg.customImage, nil + } + + // Otherwise, make sure to properly prefix the image identifier return fmt.Sprintf("global/images/%s", cfg.customImage), nil } project, ok := imageProjects[cfg.providerConfig.OperatingSystem] From 483fd2467ffafed28af07804cb238fb64d62fbe6 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Wed, 4 Jan 2023 11:31:07 +0100 Subject: [PATCH 266/489] disable kubevirt e2e tests (#1523) Signed-off-by: Moath Qasim Signed-off-by: Moath Qasim --- .prow/provider-kubevirt.yaml | 1 + test/e2e/provisioning/all_e2e_test.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 504d72599..342b0bede 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -15,6 +15,7 @@ presubmits: - name: pull-machine-controller-e2e-kubevirt run_if_changed: "(pkg/cloudprovider/provider/kubevirt/|pkg/userdata)" + always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" max_concurrency: 1 diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index ca4ccb550..85415e1cd 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -291,7 +291,7 @@ func TestKubevirtProvisioningE2E(t *testing.T) { t.Fatalf("Unable to run kubevirt tests, KUBEVIRT_E2E_TESTS_KUBECONFIG must be set") } - selector := OsSelector("ubuntu", "centos", "flatcar", "rockylinux") + selector := Not(OsSelector("ubuntu", "centos", "flatcar", "rockylinux")) params := []string{ fmt.Sprintf("<< KUBECONFIG_BASE64 >>=%s", safeBase64Encoding(kubevirtKubeconfig)), From d3c8d412ba833d6bed6ff6740bd90c6cc590f28c Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Wed, 4 Jan 2023 13:55:07 +0100 Subject: [PATCH 267/489] support prow-based provider disabling (#1524) * support prow-based provider disabling * undo #1523 * add missing pr_has_label func --- .prow/provider-alibaba.yaml | 3 ++ .prow/provider-anexia.yaml | 2 ++ .prow/provider-aws.yaml | 29 +++++++++++++++ .prow/provider-azure.yaml | 9 +++++ .prow/provider-digitalocean.yaml | 3 ++ .prow/provider-equinix-metal.yaml | 3 ++ .prow/provider-gcp.yaml | 3 ++ .prow/provider-hetzner.yaml | 3 ++ .prow/provider-kubevirt.yaml | 4 ++- .prow/provider-linode.yaml | 3 ++ .prow/provider-nutanix.yaml | 3 ++ .prow/provider-openstack.yaml | 6 ++++ .prow/provider-scaleway.yaml | 3 ++ .prow/provider-vmware-cloud-director.yaml | 3 ++ .prow/provider-vsphere.yaml | 9 +++++ hack/ci/run-e2e-tests.sh | 4 +++ hack/lib.sh | 44 +++++++++++++++++++++++ test/e2e/provisioning/all_e2e_test.go | 2 +- 18 files changed, 134 insertions(+), 2 deletions(-) diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index ac6dec341..e579225af 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -34,6 +34,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestAlibabaProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: alibaba securityContext: privileged: true resources: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index daaec94b8..d4f443cc7 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -36,6 +36,8 @@ presubmits: # OperatingSystemManager does not yet support Anexia - name: OPERATING_SYSTEM_MANAGER value: "false" + - name: CLOUD_PROVIDER + value: anexia securityContext: privileged: true resources: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 2578a5a18..db17955f8 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -33,6 +33,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: aws securityContext: privileged: true resources: @@ -61,6 +64,8 @@ presubmits: env: - name: OPERATING_SYSTEM_MANAGER value: "false" + - name: CLOUD_PROVIDER + value: aws command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,6 +98,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSARMProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: aws securityContext: privileged: true resources: @@ -121,6 +129,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSEbsEncryptionEnabledProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: aws securityContext: privileged: true resources: @@ -149,6 +160,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSFlatcarContainerdProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: aws securityContext: privileged: true resources: @@ -178,6 +192,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSSpotInstanceProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: aws securityContext: privileged: true resources: @@ -206,6 +223,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSSLESProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: aws securityContext: privileged: true resources: @@ -234,6 +254,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSFlatcarCoreOSCloudInit8ProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: aws securityContext: privileged: true resources: @@ -262,6 +285,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSCentOS8ProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: aws securityContext: privileged: true resources: @@ -290,6 +316,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSAssumeRoleProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: aws securityContext: privileged: true resources: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 06b592cd9..8325a6df6 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -33,6 +33,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestAzureProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: azure securityContext: privileged: true resources: @@ -62,6 +65,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestAzureCustomImageReferenceProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: azure securityContext: privileged: true resources: @@ -92,6 +98,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestAzureProvisioningE2ERedhatSatellite" + env: + - name: CLOUD_PROVIDER + value: azure securityContext: privileged: true resources: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 717a87f1c..3b7dce3d2 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -32,6 +32,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestDigitalOceanProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: digitalocean securityContext: privileged: true resources: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 6a0bcb985..c7d80af7d 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -33,6 +33,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestEquinixMetalProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: metal securityContext: privileged: true resources: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 8434634f7..d879ad984 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -33,6 +33,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestGCEProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: gce securityContext: privileged: true resources: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 4ee6ddce3..de08ca29a 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -31,6 +31,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestHetznerProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: hetzner securityContext: privileged: true resources: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 342b0bede..61d12af71 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -15,7 +15,6 @@ presubmits: - name: pull-machine-controller-e2e-kubevirt run_if_changed: "(pkg/cloudprovider/provider/kubevirt/|pkg/userdata)" - always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" max_concurrency: 1 @@ -35,6 +34,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestKubevirtProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: kubevirt securityContext: privileged: true resources: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 8d8edac3c..ac9d1a895 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -33,6 +33,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestLinodeProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: linode securityContext: privileged: true resources: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index f669a460f..1b303e5c5 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -32,6 +32,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestNutanixProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: nutanix securityContext: privileged: true resources: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 87774f7f0..f17b61995 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -33,6 +33,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestOpenstackProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: openstack securityContext: privileged: true resources: @@ -62,6 +65,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestOpenstackProjectAuthProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: openstack securityContext: privileged: true resources: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index a238fdd70..9f0aab11a 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -32,6 +32,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestScalewayProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: scaleway securityContext: privileged: true resources: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index a256bda14..481fb8d64 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -34,6 +34,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestVMwareCloudDirectorProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: vcd securityContext: privileged: true resources: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 452d24c01..2343f0080 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -33,6 +33,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestVsphereProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: vsphere securityContext: privileged: true resources: @@ -62,6 +65,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestVsphereDatastoreClusterProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: vsphere securityContext: privileged: true resources: @@ -91,6 +97,9 @@ presubmits: - "./hack/ci/run-e2e-tests.sh" args: - "TestVsphereResourcePoolProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: vsphere securityContext: privileged: true resources: diff --git a/hack/ci/run-e2e-tests.sh b/hack/ci/run-e2e-tests.sh index cf848be61..2b235c06b 100755 --- a/hack/ci/run-e2e-tests.sh +++ b/hack/ci/run-e2e-tests.sh @@ -19,6 +19,10 @@ set -euo pipefail cd $(dirname $0)/../.. source hack/lib.sh +if provider_disabled "${CLOUD_PROVIDER:-}"; then + exit 0 +fi + function cleanup { set +e diff --git a/hack/lib.sh b/hack/lib.sh index e04b7a3b4..5fc9131c4 100644 --- a/hack/lib.sh +++ b/hack/lib.sh @@ -241,3 +241,47 @@ check_all_deployments_ready() { return 0 } + +pr_has_label() { + if [ -z "${REPO_OWNER:-}" ] || [ -z "${REPO_NAME:-}" ] || [ -z "${PULL_NUMBER:-}" ]; then + echo "PR check only works on CI." + return 1 + fi + + matched=$(curl \ + --header "Accept: application/vnd.github+json" \ + --silent \ + --fail \ + https://api.github.com/repos/$REPO_OWNER/$REPO_NAME/pulls/$PULL_NUMBER | + jq --arg labelName "$1" '.labels[] | select(.name == $labelName)') + + [ -n "$matched" ] +} + +provider_disabled() { + # e.g. "VSPHERE_E2E_DISABLED" + local disableEnv="${1^^}_E2E_DISABLED" + local labelName="test/require-$1" + + # tests can be globally disabled by having a special environment + # variable injected via the Prow preset; if they are not disabled, + # we are done here. + if [ -z "${!disableEnv:-}" ]; then + return 1 + fi + + # Even if tests are disabled, they can be forcefully re-enabled + # (e.g. if provider X is disabled for all tests until a certain + # pull requests fixes some underlying issue and for that certain + # PR we want to run the tests regardless). + # Importantly, one cannot use labels to _disable_ any tests, only + # _re-enable_ them. + + if pr_has_label "$labelName"; then + echodate "\$$disableEnv is set, but PR has $labelName label, so tests will not be disabled." + return 1 + fi + + echodate "\$$disableEnv is set, tests will be disabled. Apply the label $labelName to this PR to forcefully enable the tests." + return 0 +} diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 85415e1cd..ca4ccb550 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -291,7 +291,7 @@ func TestKubevirtProvisioningE2E(t *testing.T) { t.Fatalf("Unable to run kubevirt tests, KUBEVIRT_E2E_TESTS_KUBECONFIG must be set") } - selector := Not(OsSelector("ubuntu", "centos", "flatcar", "rockylinux")) + selector := OsSelector("ubuntu", "centos", "flatcar", "rockylinux") params := []string{ fmt.Sprintf("<< KUBECONFIG_BASE64 >>=%s", safeBase64Encoding(kubevirtKubeconfig)), From 067a36ebbbf73650f3227cebf495ad7044badbef Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 9 Jan 2023 19:20:31 +0500 Subject: [PATCH 268/489] Add support for Kubernetes version 1.26 (#1522) * Add support for Kubernetes version 1.26 Signed-off-by: Waleed Malik * Switch region for AWS spot instances due to InsufficientInstanceCapacity Signed-off-by: Waleed Malik * Drop configuration related to dynamic kubelet config Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- .golangci.yml | 5 - .prow/provider-openstack.yaml | 4 + README.md | 2 +- examples/alibaba-machinedeployment.yaml | 2 +- examples/anexia-machinedeployment.yaml | 2 +- examples/aws-machinedeployment.yaml | 2 +- examples/azure-machinedeployment.yaml | 2 +- examples/digitalocean-machinedeployment.yaml | 2 +- examples/equinixmetal-machinedeployment.yaml | 2 +- examples/gce-machinedeployment.yaml | 2 +- examples/hetzner-machinedeployment.yaml | 2 +- examples/kubevirt-machinedeployment.yaml | 2 +- examples/linode-machinedeployment.yaml | 2 +- examples/nutanix-machinedeployment.yaml | 2 +- examples/openstack-machinedeployment.yaml | 2 +- examples/operating-system-manager.yaml | 4 +- examples/scaleway-machinedeployment.yaml | 2 +- ...ware-cloud-director-machinedeployment.yaml | 2 +- ...e-datastore-cluster-machinedeployment.yaml | 2 +- examples/vsphere-machinedeployment.yaml | 2 +- go.mod | 93 ++-- go.sum | 394 ++++---------- pkg/admission/machines.go | 13 +- pkg/apis/cluster/v1alpha1/machine_types.go | 1 + pkg/cloudprovider/provider/azure/provider.go | 2 +- pkg/containerruntime/containerd.go | 3 - pkg/controller/machine/machine_controller.go | 12 +- pkg/controller/machine/machine_test.go | 4 +- pkg/userdata/amzn2/provider_test.go | 29 +- .../amzn2/testdata/kubelet-v1.24-aws.yaml | 4 +- ...yaml => kubelet-v1.24.9-aws-external.yaml} | 51 +- ...1.23-aws.yaml => kubelet-v1.24.9-aws.yaml} | 51 +- ...l => kubelet-v1.24.9-vsphere-mirrors.yaml} | 51 +- ...aml => kubelet-v1.24.9-vsphere-proxy.yaml} | 58 ++- ...here.yaml => kubelet-v1.24.9-vsphere.yaml} | 51 +- .../amzn2/testdata/kubelet-v1.25-aws.yaml | 4 +- .../amzn2/testdata/kubelet-v1.26-aws.yaml | 452 ++++++++++++++++ pkg/userdata/centos/provider_test.go | 32 +- ...yaml => kubelet-v1.24.9-aws-external.yaml} | 51 +- ...1.24-aws.yaml => kubelet-v1.24.9-aws.yaml} | 4 +- ...anix.yaml => kubelet-v1.24.9-nutanix.yaml} | 51 +- ...l => kubelet-v1.24.9-vsphere-mirrors.yaml} | 51 +- ...aml => kubelet-v1.24.9-vsphere-proxy.yaml} | 58 ++- ...here.yaml => kubelet-v1.24.9-vsphere.yaml} | 51 +- .../centos/testdata/kubelet-v1.25-aws.yaml | 2 +- ...-v1.23-aws.yaml => kubelet-v1.26-aws.yaml} | 51 +- pkg/userdata/flatcar/provider_test.go | 22 +- .../flatcar/testdata/cloud-init_v1.24.0.yaml | 2 +- ...t_v1.23.5.yaml => cloud-init_v1.24.9.yaml} | 46 +- .../flatcar/testdata/cloud-init_v1.25.0.yaml | 2 +- pkg/userdata/flatcar/testdata/containerd.yaml | 2 +- .../flatcar/testdata/ignition_v1.24.0.json | 2 +- ...ion_v1.23.5.json => ignition_v1.24.9.json} | 2 +- .../flatcar/testdata/ignition_v1.25.0.json | 2 +- pkg/userdata/helper/common_test.go | 6 +- .../helper/download_binaries_script.go | 2 +- .../helper/download_binaries_script_test.go | 4 +- pkg/userdata/helper/kubelet.go | 31 -- pkg/userdata/helper/kubelet_test.go | 8 +- .../download_binaries_v1.23.13.golden | 17 - .../testdata/download_binaries_v1.24.7.golden | 17 - ...olden => download_binaries_v1.24.9.golden} | 2 +- ...olden => download_binaries_v1.25.5.golden} | 2 +- ...olden => download_binaries_v1.26.0.golden} | 2 +- ...let_systemd_unit_cloud-provider-set.golden | 1 - ...t_systemd_unit_multiple-dns-servers.golden | 1 - ...kublet_systemd_unit_pause-image-set.golden | 1 - .../kublet_systemd_unit_taints-set.golden | 1 - ...temd_unit_version-v1.23.13-external.golden | 37 -- ...ublet_systemd_unit_version-v1.23.13.golden | 36 -- ...stemd_unit_version-v1.23.5-external.golden | 37 -- ...kublet_systemd_unit_version-v1.23.5.golden | 36 -- ...temd_unit_version-v1.24.9-external.golden} | 0 ...ublet_systemd_unit_version-v1.24.9.golden} | 0 ...temd_unit_version-v1.25.5-external.golden} | 0 ...ublet_systemd_unit_version-v1.25.5.golden} | 0 ...temd_unit_version-v1.26.0-external.golden} | 0 ...ublet_systemd_unit_version-v1.26.0.golden} | 0 ... => safe_download_binaries_v1.24.9.golden} | 4 +- pkg/userdata/rhel/provider_test.go | 20 +- .../testdata/kubelet-v1.24-aws-external.yaml | 2 +- .../rhel/testdata/kubelet-v1.24-aws.yaml | 2 +- ...yaml => kubelet-v1.24.9-aws-external.yaml} | 51 +- ...1.23-aws.yaml => kubelet-v1.24.9-aws.yaml} | 51 +- ...l => kubelet-v1.24.9-vsphere-mirrors.yaml} | 51 +- ...aml => kubelet-v1.24.9-vsphere-proxy.yaml} | 67 ++- ...roxy.yaml => kubelet-v1.24.9-vsphere.yaml} | 60 ++- .../rhel/testdata/kubelet-v1.25-aws.yaml | 2 +- .../rhel/testdata/kubelet-v1.25-nutanix.yaml | 2 +- .../rhel/testdata/pod-cidr-azure-rhel.yaml | 2 +- pkg/userdata/rockylinux/provider_test.go | 24 +- .../testdata/kubelet-v1.23-vsphere.yaml | 458 ---------------- .../testdata/kubelet-v1.24-aws.yaml | 2 +- ...yaml => kubelet-v1.24.9-aws-external.yaml} | 51 +- ...1.23-aws.yaml => kubelet-v1.24.9-aws.yaml} | 51 +- ...anix.yaml => kubelet-v1.24.9-nutanix.yaml} | 51 +- ...l => kubelet-v1.24.9-vsphere-mirrors.yaml} | 51 +- .../kubelet-v1.24.9-vsphere-proxy.yaml | 489 ++++++++++++++++++ ...rors.yaml => kubelet-v1.24.9-vsphere.yaml} | 60 ++- .../testdata/kubelet-v1.25-aws.yaml | 2 +- pkg/userdata/sles/provider_test.go | 8 +- .../sles/testdata/dist-upgrade-on-boot.yaml | 38 +- .../kubelet-version-without-v-prefix.yaml | 38 +- .../sles/testdata/multiple-dns-servers.yaml | 38 +- .../sles/testdata/multiple-ssh-keys.yaml | 38 +- .../openstack-overwrite-cloud-config.yaml | 38 +- pkg/userdata/sles/testdata/openstack.yaml | 38 +- .../sles/testdata/version-1.23.5.yaml | 413 --------------- .../sles/testdata/version-1.24.0.yaml | 428 --------------- ...ersion-1.24.7.yaml => version-1.24.9.yaml} | 4 +- .../sles/testdata/version-1.25.0.yaml | 415 --------------- ...ersion-1.25.3.yaml => version-1.25.5.yaml} | 4 +- ...rsion-1.23.13.yaml => version-1.26.0.yaml} | 38 +- .../sles/testdata/vsphere-mirrors.yaml | 38 +- pkg/userdata/sles/testdata/vsphere-proxy.yaml | 45 +- pkg/userdata/sles/testdata/vsphere.yaml | 38 +- pkg/userdata/ubuntu/provider_test.go | 10 +- pkg/userdata/ubuntu/testdata/containerd.yaml | 7 +- .../digitalocean-dualstack-IPv6+IPv4.yaml | 52 +- .../testdata/digitalocean-dualstack.yaml | 52 +- .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 52 +- pkg/userdata/ubuntu/testdata/docker.yaml | 67 +-- .../kubelet-version-without-v-prefix.yaml | 5 +- .../ubuntu/testdata/multiple-dns-servers.yaml | 52 +- .../ubuntu/testdata/multiple-ssh-keys.yaml | 5 +- pkg/userdata/ubuntu/testdata/nutanix.yaml | 50 +- .../openstack-dualstack-IPv6+IPv4.yaml | 52 +- .../ubuntu/testdata/openstack-dualstack.yaml | 52 +- .../openstack-overwrite-cloud-config.yaml | 5 +- pkg/userdata/ubuntu/testdata/openstack.yaml | 52 +- .../ubuntu/testdata/version-1.23.5.yaml | 436 ---------------- .../ubuntu/testdata/version-1.24.0.yaml | 451 ---------------- ...ersion-1.25.3.yaml => version-1.24.9.yaml} | 4 +- .../ubuntu/testdata/version-1.25.0.yaml | 438 ---------------- ...ersion-1.24.7.yaml => version-1.25.5.yaml} | 4 +- ...rsion-1.23.13.yaml => version-1.26.0.yaml} | 50 +- .../ubuntu/testdata/vsphere-mirrors.yaml | 5 +- .../ubuntu/testdata/vsphere-proxy.yaml | 5 +- pkg/userdata/ubuntu/testdata/vsphere.yaml | 5 +- test/e2e/provisioning/all_e2e_test.go | 29 +- test/e2e/provisioning/helper.go | 8 +- .../machinedeployment-aws-spot-instances.yaml | 2 +- ...hinedeployment-azure-redhat-satellite.yaml | 2 +- 143 files changed, 2650 insertions(+), 4506 deletions(-) rename pkg/userdata/amzn2/testdata/{kubelet-v1.23-aws-external.yaml => kubelet-v1.24.9-aws-external.yaml} (91%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.23-aws.yaml => kubelet-v1.24.9-aws.yaml} (91%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.23-vsphere-mirrors.yaml => kubelet-v1.24.9-vsphere-mirrors.yaml} (91%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.23-vsphere-proxy.yaml => kubelet-v1.24.9-vsphere-proxy.yaml} (89%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.23-vsphere.yaml => kubelet-v1.24.9-vsphere.yaml} (91%) create mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml rename pkg/userdata/centos/testdata/{kubelet-v1.23-aws-external.yaml => kubelet-v1.24.9-aws-external.yaml} (91%) rename pkg/userdata/centos/testdata/{kubelet-v1.24-aws.yaml => kubelet-v1.24.9-aws.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.23-nutanix.yaml => kubelet-v1.24.9-nutanix.yaml} (91%) rename pkg/userdata/centos/testdata/{kubelet-v1.23-vsphere-mirrors.yaml => kubelet-v1.24.9-vsphere-mirrors.yaml} (92%) rename pkg/userdata/centos/testdata/{kubelet-v1.23-vsphere-proxy.yaml => kubelet-v1.24.9-vsphere-proxy.yaml} (89%) rename pkg/userdata/centos/testdata/{kubelet-v1.23-vsphere.yaml => kubelet-v1.24.9-vsphere.yaml} (91%) rename pkg/userdata/centos/testdata/{kubelet-v1.23-aws.yaml => kubelet-v1.26-aws.yaml} (91%) rename pkg/userdata/flatcar/testdata/{cloud-init_v1.23.5.yaml => cloud-init_v1.24.9.yaml} (91%) rename pkg/userdata/flatcar/testdata/{ignition_v1.23.5.json => ignition_v1.24.9.json} (59%) delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.23.13.golden delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.24.7.golden rename pkg/userdata/helper/testdata/{download_binaries_v1.25.3.golden => download_binaries_v1.24.9.golden} (92%) rename pkg/userdata/helper/testdata/{download_binaries_v1.23.5.golden => download_binaries_v1.25.5.golden} (92%) rename pkg/userdata/helper/testdata/{download_binaries_v1.24.0.golden => download_binaries_v1.26.0.golden} (92%) delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.13-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.13.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.23.5.golden rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.24.0-external.golden => kublet_systemd_unit_version-v1.24.9-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.24.0.golden => kublet_systemd_unit_version-v1.24.9.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.24.7-external.golden => kublet_systemd_unit_version-v1.25.5-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.24.7.golden => kublet_systemd_unit_version-v1.25.5.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.25.3-external.golden => kublet_systemd_unit_version-v1.26.0-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.25.3.golden => kublet_systemd_unit_version-v1.26.0.golden} (100%) rename pkg/userdata/helper/testdata/{safe_download_binaries_v1.23.13.golden => safe_download_binaries_v1.24.9.golden} (93%) rename pkg/userdata/rhel/testdata/{kubelet-v1.23-aws-external.yaml => kubelet-v1.24.9-aws-external.yaml} (92%) rename pkg/userdata/rhel/testdata/{kubelet-v1.23-aws.yaml => kubelet-v1.24.9-aws.yaml} (92%) rename pkg/userdata/rhel/testdata/{kubelet-v1.23-vsphere-mirrors.yaml => kubelet-v1.24.9-vsphere-mirrors.yaml} (92%) rename pkg/userdata/rhel/testdata/{kubelet-v1.23-vsphere.yaml => kubelet-v1.24.9-vsphere-proxy.yaml} (88%) rename pkg/userdata/rhel/testdata/{kubelet-v1.23-vsphere-proxy.yaml => kubelet-v1.24.9-vsphere.yaml} (92%) delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml rename pkg/userdata/rockylinux/testdata/{kubelet-v1.23-aws-external.yaml => kubelet-v1.24.9-aws-external.yaml} (92%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.23-aws.yaml => kubelet-v1.24.9-aws.yaml} (92%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.23-nutanix.yaml => kubelet-v1.24.9-nutanix.yaml} (92%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.23-vsphere-proxy.yaml => kubelet-v1.24.9-vsphere-mirrors.yaml} (92%) create mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml rename pkg/userdata/rockylinux/testdata/{kubelet-v1.23-vsphere-mirrors.yaml => kubelet-v1.24.9-vsphere.yaml} (92%) delete mode 100644 pkg/userdata/sles/testdata/version-1.23.5.yaml delete mode 100644 pkg/userdata/sles/testdata/version-1.24.0.yaml rename pkg/userdata/sles/testdata/{version-1.24.7.yaml => version-1.24.9.yaml} (98%) delete mode 100644 pkg/userdata/sles/testdata/version-1.25.0.yaml rename pkg/userdata/sles/testdata/{version-1.25.3.yaml => version-1.25.5.yaml} (98%) rename pkg/userdata/sles/testdata/{version-1.23.13.yaml => version-1.26.0.yaml} (92%) delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.23.5.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.24.0.yaml rename pkg/userdata/ubuntu/testdata/{version-1.25.3.yaml => version-1.24.9.yaml} (99%) delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.25.0.yaml rename pkg/userdata/ubuntu/testdata/{version-1.24.7.yaml => version-1.25.5.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.23.13.yaml => version-1.26.0.yaml} (91%) diff --git a/.golangci.yml b/.golangci.yml index 08d4e6507..cc14d4190 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -59,8 +59,3 @@ issues: - 'cyclomatic complexity 34 of func `\(\*provider\)\.getConfig` is high' - 'cyclomatic complexity 31 of func `\(\*provider\)\.Validate` is high' - 'cyclomatic complexity 33 of func `\(\*provider\)\.Create` is high' - # SA1019: node.Spec.ConfigSource is deprecated: Previously used to specify the source of the node's configuration for the DynamicKubeletConfig feature. - # This feature is removed from Kubelets as of 1.24 and will be fully removed in 1.26. +optional - # We still support setting dynamic kubelet config feature in machine-controller. Hence, ignoring this error. - # TODO: remove this once we remove support for the feature in 1.23 - - "SA1019: node.Spec.ConfigSource is deprecated" diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index f17b61995..b787f092a 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -15,6 +15,10 @@ presubmits: - name: pull-machine-controller-e2e-openstack run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" + # We've made the E2E tests for OpenStack optional since in-tree cloud provider for OpenStack was removed with k8s v1.26. Since MC depends on the in-tree cloud provider + # the tests on k8s v1.26+ will fail. + # TODO: These tests shouldn't be marked as optional. + optional: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: diff --git a/README.md b/README.md index 5197fc1b1..cd9f9b6fb 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,9 @@ machine-controller tries to follow the Kubernetes version Currently supported K8S versions are: +- 1.26 - 1.25 - 1.24 -- 1.23 ## What does not work diff --git a/examples/alibaba-machinedeployment.yaml b/examples/alibaba-machinedeployment.yaml index 940198f90..b36c7571b 100644 --- a/examples/alibaba-machinedeployment.yaml +++ b/examples/alibaba-machinedeployment.yaml @@ -62,4 +62,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.23.13 + kubelet: 1.24.9 diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index 8702e2a29..743bddcaa 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -53,4 +53,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.23.13 + kubelet: 1.24.9 diff --git a/examples/aws-machinedeployment.yaml b/examples/aws-machinedeployment.yaml index b0ad7ea7a..088c2d348 100644 --- a/examples/aws-machinedeployment.yaml +++ b/examples/aws-machinedeployment.yaml @@ -80,4 +80,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.23.13 + kubelet: 1.24.9 diff --git a/examples/azure-machinedeployment.yaml b/examples/azure-machinedeployment.yaml index c19b27b2e..d4ed1546e 100644 --- a/examples/azure-machinedeployment.yaml +++ b/examples/azure-machinedeployment.yaml @@ -92,4 +92,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.23.13 + kubelet: 1.24.9 diff --git a/examples/digitalocean-machinedeployment.yaml b/examples/digitalocean-machinedeployment.yaml index d6373cb5a..f914a89ef 100644 --- a/examples/digitalocean-machinedeployment.yaml +++ b/examples/digitalocean-machinedeployment.yaml @@ -57,4 +57,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.23.13 + kubelet: 1.24.9 diff --git a/examples/equinixmetal-machinedeployment.yaml b/examples/equinixmetal-machinedeployment.yaml index eae33e2d5..c46b17f5d 100644 --- a/examples/equinixmetal-machinedeployment.yaml +++ b/examples/equinixmetal-machinedeployment.yaml @@ -51,4 +51,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.23.13 + kubelet: 1.24.9 diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index acb73e180..bb4392e28 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -86,4 +86,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.23.13 + kubelet: 1.24.9 diff --git a/examples/hetzner-machinedeployment.yaml b/examples/hetzner-machinedeployment.yaml index a2da21a0f..9dafc90bf 100644 --- a/examples/hetzner-machinedeployment.yaml +++ b/examples/hetzner-machinedeployment.yaml @@ -63,4 +63,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.23.13 + kubelet: 1.24.9 diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index 0f59d2e0a..81b71ed4a 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -69,4 +69,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.23.13 + kubelet: 1.24.9 diff --git a/examples/linode-machinedeployment.yaml b/examples/linode-machinedeployment.yaml index da08f50de..cf7beb50d 100644 --- a/examples/linode-machinedeployment.yaml +++ b/examples/linode-machinedeployment.yaml @@ -53,4 +53,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.23.13 + kubelet: 1.24.9 diff --git a/examples/nutanix-machinedeployment.yaml b/examples/nutanix-machinedeployment.yaml index ed4d147ae..e6978b0a6 100644 --- a/examples/nutanix-machinedeployment.yaml +++ b/examples/nutanix-machinedeployment.yaml @@ -82,4 +82,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.23.13 + kubelet: 1.24.9 diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index 8fa5eecf5..f3401d3cd 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -166,4 +166,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.23.13 + kubelet: 1.24.9 diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index 211412a85..989258c96 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -975,7 +975,7 @@ spec: serviceAccountName: operating-system-manager-webhook containers: # TODO: Update this to a semver tag before release. - - image: quay.io/kubermatic/operating-system-manager:4f7c5a6873538e922afd70bff41850795657f313 + - image: quay.io/kubermatic/operating-system-manager:22c771f4cb686427d94df83946b292d7ff3a06f8 imagePullPolicy: IfNotPresent name: webhook command: @@ -1304,7 +1304,7 @@ spec: serviceAccountName: operating-system-manager containers: # TODO: Update this to a semver tag before release. - - image: quay.io/kubermatic/operating-system-manager:4f7c5a6873538e922afd70bff41850795657f313 + - image: quay.io/kubermatic/operating-system-manager:22c771f4cb686427d94df83946b292d7ff3a06f8 imagePullPolicy: IfNotPresent name: operating-system-manager command: diff --git a/examples/scaleway-machinedeployment.yaml b/examples/scaleway-machinedeployment.yaml index 76bdb3cbf..da66040ff 100644 --- a/examples/scaleway-machinedeployment.yaml +++ b/examples/scaleway-machinedeployment.yaml @@ -61,4 +61,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.23.13 + kubelet: 1.24.9 diff --git a/examples/vmware-cloud-director-machinedeployment.yaml b/examples/vmware-cloud-director-machinedeployment.yaml index ec5baaac2..db9507890 100644 --- a/examples/vmware-cloud-director-machinedeployment.yaml +++ b/examples/vmware-cloud-director-machinedeployment.yaml @@ -76,4 +76,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.23.13 + kubelet: 1.24.9 diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml index b34ff9880..9018db197 100644 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ b/examples/vsphere-datastore-cluster-machinedeployment.yaml @@ -74,4 +74,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.23.13 + kubelet: 1.24.9 diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index 3425b432a..59ad3072a 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -74,4 +74,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.23.13 + kubelet: 1.24.9 diff --git a/go.mod b/go.mod index 4dc281bf2..cdfef6426 100644 --- a/go.mod +++ b/go.mod @@ -3,31 +3,31 @@ module github.com/kubermatic/machine-controller go 1.19 require ( - cloud.google.com/go/logging v1.5.0 - cloud.google.com/go/monitoring v1.9.0 + cloud.google.com/go/logging v1.6.1 + cloud.google.com/go/monitoring v1.9.1 github.com/Azure/azure-sdk-for-go v65.0.0+incompatible github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/BurntSushi/toml v1.2.1 github.com/Masterminds/semver/v3 v3.2.0 - github.com/Masterminds/sprig/v3 v3.2.2 - github.com/aliyun/alibaba-cloud-sdk-go v1.62.47 - github.com/aws/aws-sdk-go-v2 v1.17.1 - github.com/aws/aws-sdk-go-v2/config v1.18.3 - github.com/aws/aws-sdk-go-v2/credentials v1.13.3 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.73.0 - github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 - github.com/aws/smithy-go v1.13.4 + github.com/Masterminds/sprig/v3 v3.2.3 + github.com/aliyun/alibaba-cloud-sdk-go v1.62.112 + github.com/aws/aws-sdk-go-v2 v1.17.3 + github.com/aws/aws-sdk-go-v2/config v1.18.7 + github.com/aws/aws-sdk-go-v2/credentials v1.13.7 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 + github.com/aws/smithy-go v1.13.5 github.com/coreos/container-linux-config-transpiler v0.9.0 github.com/davecgh/go-spew v1.1.1 - github.com/digitalocean/godo v1.91.1 + github.com/digitalocean/godo v1.93.0 github.com/ghodss/yaml v1.0.0 github.com/go-test/deep v1.0.8 github.com/google/uuid v1.3.0 - github.com/gophercloud/gophercloud v1.0.0 + github.com/gophercloud/gophercloud v1.1.1 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb - github.com/hetznercloud/hcloud-go v1.37.0 - github.com/linode/linodego v1.9.3 + github.com/hetznercloud/hcloud-go v1.39.0 + github.com/linode/linodego v1.10.0 github.com/nutanix-cloud-native/prism-go-client v0.3.4 github.com/packethost/packngo v0.29.0 github.com/patrickmn/go-cache v2.1.0+incompatible @@ -38,13 +38,13 @@ require ( github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10 github.com/sethvargo/go-password v0.2.0 github.com/tinkerbell/tink v0.8.0 - github.com/vmware/go-vcloud-director/v2 v2.17.0 - github.com/vmware/govmomi v0.29.0 - go.anx.io/go-anxcloud v0.4.6 - golang.org/x/crypto v0.3.0 - golang.org/x/oauth2 v0.2.0 + github.com/vmware/go-vcloud-director/v2 v2.18.0 + github.com/vmware/govmomi v0.30.0 + go.anx.io/go-anxcloud v0.5.0 + golang.org/x/crypto v0.4.0 + golang.org/x/oauth2 v0.3.0 gomodules.xyz/jsonpatch/v2 v2.2.0 - google.golang.org/api v0.103.0 + google.golang.org/api v0.105.0 google.golang.org/grpc v1.51.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 @@ -56,15 +56,15 @@ require ( k8s.io/kubelet v0.26.0 k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 kubevirt.io/api v0.58.0 - kubevirt.io/containerized-data-importer-api v1.55.1 + kubevirt.io/containerized-data-importer-api v1.55.2 sigs.k8s.io/controller-runtime v0.14.1 sigs.k8s.io/yaml v1.3.0 ) require ( cloud.google.com/go v0.107.0 // indirect - cloud.google.com/go/compute v1.12.1 // indirect - cloud.google.com/go/compute/metadata v0.2.1 // indirect + cloud.google.com/go/compute v1.14.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/longrunning v0.3.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.28 // indirect @@ -81,16 +81,16 @@ require ( github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/ignition v0.35.0 // indirect @@ -103,25 +103,25 @@ require ( github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.2.3 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/go-resty/resty/v2 v2.7.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.4.2 // indirect + github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-version v1.6.0 // indirect - github.com/huandu/xstrings v1.3.3 // indirect + github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.13 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -142,8 +142,8 @@ require ( github.com/packethost/pkg v0.0.0-20211110202003-387414657e83 // indirect github.com/peterhellberg/link v1.2.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/common v0.39.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cast v1.5.0 // indirect @@ -151,21 +151,22 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.4 // indirect - go.opentelemetry.io/otel v1.11.1 // indirect - go.opentelemetry.io/otel/trace v1.11.1 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 // indirect + go.opentelemetry.io/otel v1.11.2 // indirect + go.opentelemetry.io/otel/metric v0.34.0 // indirect + go.opentelemetry.io/otel/trace v1.11.2 // indirect go.uber.org/atomic v1.10.0 // indirect - go.uber.org/multierr v1.8.0 // indirect + go.uber.org/multierr v1.9.0 // indirect go.uber.org/zap v1.24.0 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 // indirect + golang.org/x/net v0.4.0 // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.3.0 // indirect golang.org/x/term v0.3.0 // indirect golang.org/x/text v0.5.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect + google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect @@ -174,7 +175,7 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/component-base v0.26.0 // indirect k8s.io/klog/v2 v2.80.1 // indirect - k8s.io/kube-openapi v0.0.0-20221123214604-86e75ddd809a // indirect + k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect diff --git a/go.sum b/go.sum index a301ee9cf..cd5b4ee4b 100644 --- a/go.sum +++ b/go.sum @@ -13,22 +13,6 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= -cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww= cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= @@ -37,25 +21,19 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= -cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= -cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= -cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0= -cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= -cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48= -cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute v1.14.0 h1:hfm2+FfxVmnRlh6LpB7cg1ZNU+5edAHmW679JePztk0= +cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= -cloud.google.com/go/logging v1.5.0 h1:DcR52smaYLgeK9KPzJlBJyyBYqW/EGKiuRRl8boL1s4= -cloud.google.com/go/logging v1.5.0/go.mod h1:c/57U/aLdzSFuBtvbtFduG1Ii54uSm95HOBnp58P7/U= +cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= +cloud.google.com/go/logging v1.6.1 h1:ZBsZK+JG+oCDT+vaxwqF2egKNRjz8soXiS6Xv79benI= +cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= -cloud.google.com/go/monitoring v1.9.0 h1:O2A5HsrhvRMzD3OMUimPXF46vOzwc9vh6oGCGf9i/ws= -cloud.google.com/go/monitoring v1.9.0/go.mod h1:/FsTS0gkEFUc4cgB16s6jYDnyjzRBkRJNRzBn5Zx+wA= +cloud.google.com/go/monitoring v1.9.1 h1:y9g09cWAQaX3ZYscR/nfaFUXtuyRqD2+i0jTOw0BZFI= +cloud.google.com/go/monitoring v1.9.1/go.mod h1:iFzRDMSDMvvf/z30Ge1jwtuEe/jlPPAFusmvCkUdo+o= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -65,7 +43,6 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/99designs/gqlgen v0.15.1 h1:48bRXecwlCNTa/n2bMSp2rQsXNxwZ54QHbiULNf78ec= github.com/99designs/gqlgen v0.15.1/go.mod h1:nbeSjFkqphIqpZsYe1ULVz0yfH8hjpJdJIQoX/e0G2I= @@ -103,11 +80,10 @@ github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= -github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= -github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= +github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= +github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= @@ -132,8 +108,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.47 h1:xMEN30mWG8D0yfa3imurqQQ02tjPtOVvLRknv/R+pls= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.47/go.mod h1:Api2AkmMgGaSUAhmk76oaFObkoeCPc/bKAqcyplPODs= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.112 h1:49S6VGQeYyk2KIw85CHbAVaVF2lSgi8xrWDwSw0GCBM= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.112/go.mod h1:Api2AkmMgGaSUAhmk76oaFObkoeCPc/bKAqcyplPODs= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -143,32 +119,32 @@ github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdK github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.42.23/go.mod h1:gyRszuZ/icHmHAVE4gc/r+cfCmhA1AD+vqfWbgI+eHs= -github.com/aws/aws-sdk-go-v2 v1.17.1 h1:02c72fDJr87N8RAC2s3Qu0YuvMRZKNZJ9F+lAehCazk= -github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= -github.com/aws/aws-sdk-go-v2/config v1.18.3 h1:3kfBKcX3votFX84dm00U8RGA1sCCh3eRMOGzg5dCWfU= -github.com/aws/aws-sdk-go-v2/config v1.18.3/go.mod h1:BYdrbeCse3ZnOD5+2/VE/nATOK8fEUpBtmPMdKSyhMU= -github.com/aws/aws-sdk-go-v2/credentials v1.13.3 h1:ur+FHdp4NbVIv/49bUjBW+FE7e57HOo03ELodttmagk= -github.com/aws/aws-sdk-go-v2/credentials v1.13.3/go.mod h1:/rOMmqYBcFfNbRPU0iN9IgGqD5+V2yp3iWNmIlz0wI4= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 h1:E3PXZSI3F2bzyj6XxUXdTIfvp425HHhwKsFvmzBwHgs= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19/go.mod h1:VihW95zQpeKQWVPGkwT+2+WJNQV8UXFfMTWdU6VErL8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 h1:nBO/RFxeq/IS5G9Of+ZrgucRciie2qpLy++3UGZ+q2E= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 h1:oRHDrwCTVT8ZXi4sr9Ld+EXk7N/KGssOr2ygNeojEhw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 h1:Mza+vlnZr+fPKFKRq/lKGVvM6B/8ZZmNdEopOwSQLms= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26/go.mod h1:Y2OJ+P+MC1u1VKnavT+PshiEuGPyh/7DqxoDNij4/bg= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.73.0 h1:3AXOhjvPxEMWw5RItV47NRLuzqwlLly5GbS5aB3sXh4= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.73.0/go.mod h1:zul71QqzR4D1a90/5FloZiAnZ1CtuIjVH7R9MP997+A= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 h1:GE25AWCdNUPh9AOJzI9KIJnja7IwUc1WyUqz/JTyJ/I= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19/go.mod h1:02CP6iuYP+IVnBX5HULVdSAku/85eHB2Y9EsFhrkEwU= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 h1:GFZitO48N/7EsFDt8fMa5iYdmWqkUDDB3Eje6z3kbG0= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.25/go.mod h1:IARHuzTXmj1C0KS35vboR0FeJ89OkEy1M9mWbK2ifCI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 h1:jcw6kKZrtNfBPJkaHrscDOZoe5gvi9wjudnxvozYFJo= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8/go.mod h1:er2JHN+kBY6FcMfcBBKNGCT3CarImmdFzishsqBmSRI= -github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 h1:60SJ4lhvn///8ygCzYy2l53bFW/Q15bVfyjyAWo6zuw= -github.com/aws/aws-sdk-go-v2/service/sts v1.17.5/go.mod h1:bXcN3koeVYiJcdDU89n3kCYILob7Y34AeLopUbZgLT4= -github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk= -github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/aws-sdk-go-v2 v1.17.3 h1:shN7NlnVzvDUgPQ+1rLMSxY8OWRNDRYtiqe0p/PgrhY= +github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/config v1.18.7 h1:V94lTcix6jouwmAsgQMAEBozVAGJMFhVj+6/++xfe3E= +github.com/aws/aws-sdk-go-v2/config v1.18.7/go.mod h1:OZYsyHFL5PB9UpyS78NElgKs11qI/B5KJau2XOJDXHA= +github.com/aws/aws-sdk-go-v2/credentials v1.13.7 h1:qUUcNS5Z1092XBFT66IJM7mYkMwgZ8fcC8YDIbEwXck= +github.com/aws/aws-sdk-go-v2/credentials v1.13.7/go.mod h1:AdCcbZXHQCjJh6NaH3pFaw8LUeBFn5+88BZGMVGuBT8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 h1:j9wi1kQ8b+e0FBVHxCqCGo4kxDU175hoDHcWAi0sauU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21/go.mod h1:ugwW57Z5Z48bpvUyZuaPy4Kv+vEfJWnIrky7RmkBvJg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 h1:I3cakv2Uy1vNmmhRQmFptYDxOvBnwCdNwyw63N0RaRU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 h1:5NbbMrIzmUn/TXFqAle6mgrH5m9cOvMLRGL7pnG8tRE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9ebfHEqlhrMirFjSW0v0C9fI+KN5vk2kE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 h1:KeTxcGdNnQudb46oOl4d90f2I33DF/c6q3RnZAmvQdQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28/go.mod h1:yRZVr/iT0AqyHeep00SZ4YfBAKojXz08w3XMBscdi0c= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0 h1:m6HYlpZlTWb9vHuuRHpWRieqPHWlS0mvQ90OJNrG/Nk= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0/go.mod h1:mV0E7631M1eXdB+tlGFIw6JxfsC7Pz7+7Aw15oLVhZw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 h1:5C6XgTViSb0bunmU57b3CT+MhxULqHH2721FVA+/kDM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21/go.mod h1:lRToEJsn+DRA9lW4O9L9+/3hjTkUzlzyzHqn8MTds5k= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 h1:gItLq3zBYyRDPmqAClgzTH8PBjDQGeyptYGHIwtYYNA= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.28/go.mod h1:wo/B7uUm/7zw/dWhBJ4FXuw1sySU5lyIhVg1Bu2yL9A= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 h1:KCacyVSs/wlcPGx37hcbT3IGYO8P8Jx+TgSDhAXtQMY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11/go.mod h1:TZSH7xLO7+phDtViY/KUp9WGCJMQkLJ/VpgkTFd5gh8= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 h1:9Mtq1KM6nD8/+HStvWcvYnixJ5N85DX+P+OY3kI3W2k= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.7/go.mod h1:+lGbb3+1ugwKrNTWcf2RT05Xmp543B06zDFTwiTLp7I= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -183,20 +159,19 @@ github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx2 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/container-linux-config-transpiler v0.9.0 h1:UBGpT8qWqzi48hNLrzMAgAUNJsR0LW8Gk5/dR/caI8U= github.com/coreos/container-linux-config-transpiler v0.9.0/go.mod h1:SlcxXZQ2c42knj8pezMiQsM1f+ADxFMjGetuMKR/YSQ= @@ -223,8 +198,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= -github.com/digitalocean/godo v1.91.1 h1:1o30VOCu1aC6488qBd0SkQiBeAZ35RSTvLwCA1pQMhc= -github.com/digitalocean/godo v1.91.1/go.mod h1:NRpFznZFvhHjBoqZAaOD3khVzsJ3EibzKqFL4R60dmA= +github.com/digitalocean/godo v1.93.0 h1:N0K9z2yssZVP7nBHQ32P1Wemd5yeiJdH4ROg+7ySRxY= +github.com/digitalocean/godo v1.93.0/go.mod h1:NRpFznZFvhHjBoqZAaOD3khVzsJ3EibzKqFL4R60dmA= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= @@ -243,12 +218,9 @@ github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRr github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= -github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -278,11 +250,9 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -295,13 +265,15 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= +github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= +github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= @@ -331,8 +303,8 @@ github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptG github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs= -github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= +github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -346,8 +318,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -363,11 +333,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= @@ -386,7 +354,6 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -398,8 +365,6 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -408,37 +373,23 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= -github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.1 h1:RY7tHKZcRlk788d5WSo/e83gOyyy742E8GSs771ySpg= +github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= -github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= -github.com/gophercloud/gophercloud v1.0.0 h1:9nTGx0jizmHxDobe4mck89FyQHVyA3CaXLIUSGJjP9k= -github.com/gophercloud/gophercloud v1.0.0/go.mod h1:Q8fZtyi5zZxPS/j9aj3sSxtvj41AdQMDwyo1myduD5c= +github.com/gophercloud/gophercloud v1.1.1 h1:MuGyqbSxiuVBqkPZ3+Nhbytk1xZxhmfCB2Rg1cJWFWM= +github.com/gophercloud/gophercloud v1.1.1/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -453,21 +404,22 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb h1:tsEKRC3PU9rMw18w/uAptoijhgG4EvlA5kfJPtwrMDk= github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= -github.com/hetznercloud/hcloud-go v1.37.0 h1:Uwu7OKfZvar86LfJuzItStoO1AL7DVDCqWzRGzrvdEw= -github.com/hetznercloud/hcloud-go v1.37.0/go.mod h1:mepQwR6va27S3UQthaEPGS86jtzSY9xWL1e9dyxXpgA= +github.com/hetznercloud/hcloud-go v1.39.0 h1:RUlzI458nGnPR6dlcZlrsGXYC1hQlFbKdm8tVtEQQB0= +github.com/hetznercloud/hcloud-go v1.39.0/go.mod h1:mepQwR6va27S3UQthaEPGS86jtzSY9xWL1e9dyxXpgA= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= +github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -522,8 +474,8 @@ github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++ github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= github.com/lestrrat-go/jwx v1.2.20/go.mod h1:tLE1XszaFgd7zaS5wHe4NxA+XVhu7xgdRvDpNyi3kNM= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/linode/linodego v1.9.3 h1:+lxNZw4avRxhCqGjwfPgQ2PvMT+vOL0OMsTdzixR7hQ= -github.com/linode/linodego v1.9.3/go.mod h1:h6AuFR/JpqwwM/vkj7s8KV3iGN8/jxn+zc437F8SZ8w= +github.com/linode/linodego v1.10.0 h1:nH/BffTBQEZr48q/9UszuB5dhWpGKuVuJs/uE9Nweuc= +github.com/linode/linodego v1.10.0/go.mod h1:lRWOfS3HmRV63U6Rt+llKziobIwpySYGlCdTIHoIgps= github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -629,7 +581,6 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -642,16 +593,15 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.31.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= +github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -724,10 +674,10 @@ github.com/vektah/gqlparser/v2 v2.2.0 h1:bAc3slekAAJW6sZTi07aGq0OrfaCjj4jxARAaC7 github.com/vektah/gqlparser/v2 v2.2.0/go.mod h1:i3mQIGIrbK2PD1RrCeMTlVbkF2FJ6WkU1KJlJlC+3F4= github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= -github.com/vmware/go-vcloud-director/v2 v2.17.0 h1:msrrtEKD7H/e3cNPaXlCkZf3TMzSSyH306EXettv0c8= -github.com/vmware/go-vcloud-director/v2 v2.17.0/go.mod h1:KjnB8t5l1bRrc+jLKDJbx0vZLRzz2RPzNQ7xzg7yI3o= -github.com/vmware/govmomi v0.29.0 h1:SHJQ7DUc4fltFZv16znJNGHR1/XhiDK5iKxm2OqwkuU= -github.com/vmware/govmomi v0.29.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= +github.com/vmware/go-vcloud-director/v2 v2.18.0 h1:3kXfaLyYObVBn7SsGxPPiIcqogwnHF0FpH5oY3KVSow= +github.com/vmware/go-vcloud-director/v2 v2.18.0/go.mod h1:KjnB8t5l1bRrc+jLKDJbx0vZLRzz2RPzNQ7xzg7yI3o= +github.com/vmware/govmomi v0.30.0 h1:Fm8ugPnnlMSTSceDKY9goGvjmqc6eQLPUSUeNXdpeXA= +github.com/vmware/govmomi v0.30.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= @@ -743,8 +693,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.anx.io/go-anxcloud v0.4.6 h1:aHf5i3UCpt+wmuP07saGGWiqS5eqQbdw0SJVpae8Hqs= -go.anx.io/go-anxcloud v0.4.6/go.mod h1:OSSNrMPa/zV9Bdpx2IdN8iIYm3fJOATR/cvwH6R3OFc= +go.anx.io/go-anxcloud v0.5.0 h1:kKzAY+CRAXmQYCr+/lbEoO6JvPEVi5qjR2XgT0CMwx4= +go.anx.io/go-anxcloud v0.5.0/go.mod h1:IjUqXU0829myWH9015ES2KG2fBUnWNF5FChLwi5tUig= go.keploy.io/server v0.1.8 h1:b50vAt1+WKMscYVP5Bm8gx/iSaR7mpHox8VpaxjrQ88= go.keploy.io/server v0.1.8/go.mod h1:ZqhwTZOBb+dzx5t30Wt6eUGI6kO5QizvPg6coNPtbow= go.mongodb.org/mongo-driver v1.8.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= @@ -754,19 +704,19 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.4 h1:PRXhsszxTt5bbPriTjmaweWUsAnJYeWBhUMLRetUgBU= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.4/go.mod h1:05eWWy6ZWzmpeImD3UowLTB3VjDMU1yxQ+ENuVWDM3c= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 h1:+uFejS4DCfNH6d3xODVIGsdhzgzhh45p9gpbHQMbdZI= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0/go.mod h1:HSmzQvagH8pS2/xrK7ScWsk0vAMtRTGbMFgInXCi8Tc= go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= -go.opentelemetry.io/otel v1.11.1 h1:4WLLAmcfkmDk2ukNXJyq3/kiz/3UzCaYq6PskJsaou4= -go.opentelemetry.io/otel v1.11.1/go.mod h1:1nNhXBbWSD0nsL38H6btgnFN2k4i0sNLHNNMZMSbUGE= +go.opentelemetry.io/otel v1.11.2 h1:YBZcQlsVekzFsFbjygXMOXSs6pialIZxcjfO/mBDmR0= +go.opentelemetry.io/otel v1.11.2/go.mod h1:7p4EUV+AqgdlNV9gL97IgUZiVR3yrFXYo53f9BM3tRI= +go.opentelemetry.io/otel/metric v0.34.0 h1:MCPoQxcg/26EuuJwpYN1mZTeCYAUGx8ABxfW07YkjP8= +go.opentelemetry.io/otel/metric v0.34.0/go.mod h1:ZFuI4yQGNCupurTXCwkeD/zHBt+C2bR7bw5JqUm/AP8= go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= -go.opentelemetry.io/otel/trace v1.11.1 h1:ofxdnzsNrGBYXbP7t7zpUK281+go5rF7dvdIZXF8gdQ= -go.opentelemetry.io/otel/trace v1.11.1/go.mod h1:f/Q9G7vzk5u91PhbmKbg1Qn0rzH1LJ4vbPHFGkTPtOk= +go.opentelemetry.io/otel/trace v1.11.2 h1:Xf7hWSF2Glv0DE3MH7fBHvtpSBsjcBUe5MYAmZM/+y0= +go.opentelemetry.io/otel/trace v1.11.2/go.mod h1:4N+yC7QEz7TTsG9BSRLNAa63eg5E06ObSbKPmxQ/pKA= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -780,8 +730,8 @@ go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= @@ -796,21 +746,21 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.3.0 h1:a06MkbcxBrEFc0w0QIZWXrH/9cCX6KJyWbBOIwAn+7A= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= +golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -833,8 +783,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -844,7 +792,6 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= @@ -882,16 +829,11 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -902,36 +844,22 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10 h1:Frnccbp+ok2GkUS2tC84yAq/U9Vg+0sIO7aRL3T4Xnc= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= -golang.org/x/oauth2 v0.2.0 h1:GtQkldQ9m7yvzCL1V+LrYow3Khe0eJH0w7RbX/VbaIU= -golang.org/x/oauth2 v0.2.0/go.mod h1:Cwn6afJ8jrQwYMxQDTpISoXmXW9I6qF6vDeuuoX3Ibs= +golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= +golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -943,7 +871,6 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -988,60 +915,40 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211015200801-69063c4bb744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1049,7 +956,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= @@ -1110,19 +1016,9 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200815165600-90abf76919f3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= @@ -1133,10 +1029,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= @@ -1155,31 +1047,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= -google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= -google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg= -google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o= -google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= -google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.105.0 h1:t6P9Jj+6XTn4U9I2wycQai6Q/Kz7iOT+QzjJ3G2V4x8= +google.golang.org/api v0.105.0/go.mod h1:qh7eD5FJks5+BcE+cjBIm6Gz8vioK7EHvnlniqXBnqI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1220,60 +1089,11 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210329143202-679c6ae281ee/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= -google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20220617124728-180714bec0ad/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= -google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= -google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef h1:uQ2vjV/sHTsWSqdKeLqmwitzgvjMl7o4IdtHwUDXSJY= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1286,30 +1106,14 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/examples v0.0.0-20210728214646-ad0a2a847cdf/go.mod h1:bF8wuZSAZTcbF7ZPKrDI/qY52toTP/yxLpRRY4Eu9Js= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1406,8 +1210,8 @@ k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lV k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/kube-openapi v0.0.0-20221123214604-86e75ddd809a h1:UR2YSPKAb8j3uL2yK8V+t2ElG4RoBxhJTxa5gg0ZtSo= -k8s.io/kube-openapi v0.0.0-20221123214604-86e75ddd809a/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715 h1:tBEbstoM+K0FiBV5KGAKQ0kuvf54v/hwpldiJt69w1s= +k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= k8s.io/kubelet v0.26.0 h1:08bDb5IoUH/1K1t2NUwnGIIWxjm9LSqn6k3FWw1tJGI= k8s.io/kubelet v0.26.0/go.mod h1:DluF+d8jS2nE/Hs7CC3QM+OZlIEb22NTOihQ3EDwCQ4= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -1419,8 +1223,8 @@ k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5 k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v0.58.0 h1:qeNeRtD6AIJ5WVJuRXajmmXtnrO5dYchy+hpCm6QwhE= kubevirt.io/api v0.58.0/go.mod h1:U0CQlZR0JoJCaC+Va0wz4dMOtYDdVywJ98OT1KmOkzI= -kubevirt.io/containerized-data-importer-api v1.55.1 h1:2WJdHrbN7pOTX1KkXKME94PG8i0Shd0DK0/3jP07d/E= -kubevirt.io/containerized-data-importer-api v1.55.1/go.mod h1:92HiQEyzPoeMiCbgfG5Qe10JQVbtWMZOXucy56dKdGg= +kubevirt.io/containerized-data-importer-api v1.55.2 h1:AzYnKIUFkKwO6c0uCQZYlAIxfzbiPkJXP29hFhauaQ8= +kubevirt.io/containerized-data-importer-api v1.55.2/go.mod h1:92HiQEyzPoeMiCbgfG5Qe10JQVbtWMZOXucy56dKdGg= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 h1:fZYvD3/Vnitfkx6IJxjLAk8ugnZQ7CXVYcRfkSKmuZY= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index 86b730a31..4d7df978f 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -151,16 +151,9 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec return fmt.Errorf("kubernetes version constraint didn't allow %q kubelet version", kubeletVer) } - // Do not allow 1.24+ to use config source (dynamic kubelet configuration) - constraint124, err := semver.NewConstraint(">= 1.24") - if err != nil { - return fmt.Errorf("failed to parse 1.24 constraint: %w", err) - } - - if constraint124.Check(kubeletVer) { - if spec.ConfigSource != nil { - return fmt.Errorf("setting spec.ConfigSource is not allowed for kubelet version %q", kubeletVer) - } + // Do not allow usage of config source (dynamic kubelet configuration) since has been removed in k8s v1.24. + if spec.ConfigSource != nil { + return fmt.Errorf("setting spec.ConfigSource is not allowed for kubelet version %q", kubeletVer) } // Validate SSH keys diff --git a/pkg/apis/cluster/v1alpha1/machine_types.go b/pkg/apis/cluster/v1alpha1/machine_types.go index 1308910d0..4295c340e 100644 --- a/pkg/apis/cluster/v1alpha1/machine_types.go +++ b/pkg/apis/cluster/v1alpha1/machine_types.go @@ -89,6 +89,7 @@ type MachineSpec struct { // spec will be automatically copied to the linked NodeRef from the // status. The rest of dynamic kubelet config support should then work // as-is. + // Deprecated: This feature has been removed with k8s v1.24. // +optional ConfigSource *corev1.NodeConfigSource `json:"configSource,omitempty"` diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index dba1c5bed..85def986b 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -163,7 +163,7 @@ var imageReferences = map[providerconfigtypes.OperatingSystem]compute.ImageRefer Publisher: to.StringPtr("kinvolk"), Offer: to.StringPtr("flatcar-container-linux"), Sku: to.StringPtr("stable"), - Version: to.StringPtr("2905.2.5"), + Version: to.StringPtr("3374.2.0"), }, providerconfigtypes.OperatingSystemRockyLinux: { Publisher: to.StringPtr("procomputers"), diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index 69631ad68..17a521eaa 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -27,7 +27,6 @@ import ( ) const ( - LegacyContainerdVersion = "1.4*" DefaultContainerdVersion = "1.6*" ) @@ -73,8 +72,6 @@ func (eng *Containerd) ScriptFor(os types.OperatingSystem) (string, error) { switch os { case types.OperatingSystemAmazonLinux2: - // Amazon Linux 2 does not have containerd 1.5 - args.ContainerdVersion = LegacyContainerdVersion err := containerdAmzn2Template.Execute(&buf, args) return buf.String(), err case types.OperatingSystemCentOS, types.OperatingSystemRHEL, types.OperatingSystemRockyLinux: diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 734479675..d5f0f47e2 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -912,10 +912,10 @@ func (r *Reconciler) ensureInstanceExistsForMachine( }); err != nil { return nil, fmt.Errorf("failed to update machine after setting .status.addresses: %w", err) } - return r.ensureNodeOwnerRefAndConfigSource(ctx, providerInstance, machine, providerConfig) + return r.ensureNodeOwnerRef(ctx, providerInstance, machine, providerConfig) } -func (r *Reconciler) ensureNodeOwnerRefAndConfigSource(ctx context.Context, providerInstance instance.Instance, machine *clusterv1alpha1.Machine, providerConfig *providerconfigtypes.Config) (*reconcile.Result, error) { +func (r *Reconciler) ensureNodeOwnerRef(ctx context.Context, providerInstance instance.Instance, machine *clusterv1alpha1.Machine, providerConfig *providerconfigtypes.Config) (*reconcile.Result, error) { node, exists, err := r.getNode(ctx, providerInstance, providerConfig.CloudProvider) if err != nil { return nil, fmt.Errorf("failed to get node for machine %s: %w", machine.Name, err) @@ -930,14 +930,6 @@ func (r *Reconciler) ensureNodeOwnerRefAndConfigSource(ctx context.Context, prov } } - if node.Spec.ConfigSource == nil && machine.Spec.ConfigSource != nil { - if err := r.updateNode(ctx, node, func(node *corev1.Node) { - node.Spec.ConfigSource = machine.Spec.ConfigSource - }); err != nil { - return nil, fmt.Errorf("failed to update node %s after setting the config source: %w", node.Name, err) - } - klog.V(3).Infof("Added config source to node %s (machine %s)", node.Name, machine.Name) - } if err := r.updateMachineStatus(machine, node); err != nil { return nil, fmt.Errorf("failed to update machine status: %w", err) } diff --git a/pkg/controller/machine/machine_test.go b/pkg/controller/machine/machine_test.go index 0fc1f5ccf..9d109ffe0 100644 --- a/pkg/controller/machine/machine_test.go +++ b/pkg/controller/machine/machine_test.go @@ -314,8 +314,8 @@ func TestControllerDeletesMachinesOnJoinTimeout(t *testing.T) { joinClusterTimeout: test.joinTimeoutConfig, } - if _, err := reconciler.ensureNodeOwnerRefAndConfigSource(ctx, instance, machine, providerConfig); err != nil { - t.Fatalf("failed to call ensureNodeOwnerRefAndConfigSource: %v", err) + if _, err := reconciler.ensureNodeOwnerRef(ctx, instance, machine, providerConfig); err != nil { + t.Fatalf("failed to call ensureNodeOwnerRef: %v", err) } err := client.Get(ctx, types.NamespacedName{Name: machine.Name}, &clusterv1alpha1.Machine{}) diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index b81db4cdb..4f656d1d4 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -100,40 +100,40 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.23-aws", + name: "kubelet-v1.24.9-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, }, { - name: "kubelet-v1.23-aws-external", + name: "kubelet-v1.24.9-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.23-vsphere", + name: "kubelet-v1.24.9-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.23-vsphere-proxy", + name: "kubelet-v1.24.9-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("vsphere"), @@ -143,11 +143,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.23-vsphere-mirrors", + name: "kubelet-v1.24.9-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("vsphere"), @@ -174,6 +174,15 @@ func TestUserDataGeneration(t *testing.T) { }, }, }, + { + name: "kubelet-v1.26-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.26.0", + }, + }, + }, } defaultCloudProvider := &fakeCloudConfigProvider{ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml index eb2122fc8..c9a36f0e6 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml @@ -93,7 +93,7 @@ write_files: EOF yum install -y \ - containerd-1.4* \ + containerd-1.6* \ yum-plugin-versionlock yum versionlock add containerd @@ -104,7 +104,7 @@ write_files: opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" arch=${HOST_ARCH-} if [ -z "$arch" ] then diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml similarity index 91% rename from pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml index 41b8b3285..0cf0553e9 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml @@ -80,28 +80,31 @@ write_files: ipvsadm - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + mkdir -p /etc/systemd/system/containerd.service.d - cat < /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + + mkdir -p /etc/systemd/system/containerd.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index becb77cfc..d8fa5a6c1 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -100,50 +100,50 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.23-aws", + name: "kubelet-v1.24.9-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, }, { - name: "kubelet-v1.23-nutanix", + name: "kubelet-v1.24.9-nutanix", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("nutanix"), }, { - name: "kubelet-v1.23-aws-external", + name: "kubelet-v1.24.9-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.23-vsphere", + name: "kubelet-v1.24.9-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.23-vsphere-proxy", + name: "kubelet-v1.24.9-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("vsphere"), @@ -153,11 +153,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.23-vsphere-mirrors", + name: "kubelet-v1.24.9-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("vsphere"), @@ -167,20 +167,20 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.24-aws", + name: "kubelet-v1.25-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.0", + Kubelet: "1.25.0", }, }, }, { - name: "kubelet-v1.25-aws", + name: "kubelet-v1.26-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.25.0", + Kubelet: "1.26.0", }, }, }, diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml similarity index 91% rename from pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml index 6a137b749..f00d58d64 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml @@ -89,29 +89,28 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat < /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml index f1a87ecc7..00bff81a6 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml @@ -105,7 +105,7 @@ write_files: opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" arch=${HOST_ARCH-} if [ -z "$arch" ] then diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml similarity index 92% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml index 3d718ee13..2d337459f 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml @@ -85,28 +85,27 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat < /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + hostnamectl set-hostname node1 + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + tar \ + open-vm-tools \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + + systemctl enable --now vmtoolsd.service + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=vsphere \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + [plugins."io.containerd.grpc.v1.cri".registry.configs] + [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000".tls] + insecure_skip_verify = true + [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000".tls] + insecure_skip_verify = true + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml similarity index 92% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml index 3cfecc74c..8ede88826 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml @@ -8,14 +8,6 @@ hostname: node1 ssh_pwauth: false write_files: -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" content: | @@ -98,28 +90,27 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat < /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - - path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - - path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.23.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - - - path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - - - path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - - - path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - - - path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - - - path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - - path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - - path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - - path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - - - path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - - path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - - - path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - - - path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - - path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - - - path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: - - systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.24.0.yaml b/pkg/userdata/sles/testdata/version-1.24.0.yaml deleted file mode 100644 index c658c8962..000000000 --- a/pkg/userdata/sles/testdata/version-1.24.0.yaml +++ /dev/null @@ -1,428 +0,0 @@ -#cloud-config - -hostname: node1 - -ssh_pwauth: false -ssh_authorized_keys: - - "ssh-rsa AAABBB" - -write_files: - - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - - path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - - path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - - path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.23.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.0}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - - - path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - - - path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - - - path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - - - path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - - - path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - - path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - - path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - - path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - - - path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - - path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - - - path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - - path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - - path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - - - path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: - - systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.24.7.yaml b/pkg/userdata/sles/testdata/version-1.24.9.yaml similarity index 98% rename from pkg/userdata/sles/testdata/version-1.24.7.yaml rename to pkg/userdata/sles/testdata/version-1.24.9.yaml index 4177de049..baf817621 100644 --- a/pkg/userdata/sles/testdata/version-1.24.7.yaml +++ b/pkg/userdata/sles/testdata/version-1.24.9.yaml @@ -63,7 +63,7 @@ write_files: opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" arch=${HOST_ARCH-} if [ -z "$arch" ] then @@ -101,7 +101,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.7}" + KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/sles/testdata/version-1.25.0.yaml b/pkg/userdata/sles/testdata/version-1.25.0.yaml deleted file mode 100644 index 39d7f4c37..000000000 --- a/pkg/userdata/sles/testdata/version-1.25.0.yaml +++ /dev/null @@ -1,415 +0,0 @@ -#cloud-config - -hostname: node1 - -ssh_pwauth: false -ssh_authorized_keys: - - "ssh-rsa AAABBB" - -write_files: - - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - - path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - - path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - - path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.23.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - - - path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - - - path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - - - path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - - - path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - - - path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - - path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - - path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - - path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - - - path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - - path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - - - path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - - - path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - - path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - - - path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: - - systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.25.3.yaml b/pkg/userdata/sles/testdata/version-1.25.5.yaml similarity index 98% rename from pkg/userdata/sles/testdata/version-1.25.3.yaml rename to pkg/userdata/sles/testdata/version-1.25.5.yaml index d301bdb07..cafef8c0b 100644 --- a/pkg/userdata/sles/testdata/version-1.25.3.yaml +++ b/pkg/userdata/sles/testdata/version-1.25.5.yaml @@ -63,7 +63,7 @@ write_files: opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" arch=${HOST_ARCH-} if [ -z "$arch" ] then @@ -101,7 +101,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.3}" + KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/sles/testdata/version-1.23.13.yaml b/pkg/userdata/sles/testdata/version-1.26.0.yaml similarity index 92% rename from pkg/userdata/sles/testdata/version-1.23.13.yaml rename to pkg/userdata/sles/testdata/version-1.26.0.yaml index 0c8d538fb..87c635150 100644 --- a/pkg/userdata/sles/testdata/version-1.23.13.yaml +++ b/pkg/userdata/sles/testdata/version-1.26.0.yaml @@ -63,7 +63,7 @@ write_files: opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" arch=${HOST_ARCH-} if [ -z "$arch" ] then @@ -101,7 +101,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.13}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -156,8 +156,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -186,9 +186,8 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] @@ -376,10 +375,27 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" @@ -399,8 +415,8 @@ write_files: permissions: "0644" content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] ExecStart=/opt/bin/health-monitor.sh container-runtime diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml index 5d6625bc0..38b631858 100644 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml @@ -73,7 +73,7 @@ write_files: opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" arch=${HOST_ARCH-} if [ -z "$arch" ] then @@ -111,7 +111,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.13}" + KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -166,8 +166,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -199,9 +199,8 @@ write_files: --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] @@ -395,10 +394,27 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"registry-mirrors":["https://registry.docker-cn.com/"]} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry.docker-cn.com/"] + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" @@ -418,8 +434,8 @@ write_files: permissions: "0644" content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] ExecStart=/opt/bin/health-monitor.sh container-runtime diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml index e24d417c6..63d3098e2 100644 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/sles/testdata/vsphere-proxy.yaml @@ -73,7 +73,7 @@ write_files: opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" arch=${HOST_ARCH-} if [ -z "$arch" ] then @@ -111,7 +111,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.13}" + KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -166,8 +166,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -199,9 +199,8 @@ write_files: --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] @@ -395,10 +394,34 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"insecure-registries":["192.168.100.100:5000","10.0.0.1:5000"]} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + [plugins."io.containerd.grpc.v1.cri".registry.configs] + [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000".tls] + insecure_skip_verify = true + [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000".tls] + insecure_skip_verify = true + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" @@ -418,8 +441,8 @@ write_files: permissions: "0644" content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] ExecStart=/opt/bin/health-monitor.sh container-runtime diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml index d2b24d668..1be34fcde 100644 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ b/pkg/userdata/sles/testdata/vsphere.yaml @@ -64,7 +64,7 @@ write_files: opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" arch=${HOST_ARCH-} if [ -z "$arch" ] then @@ -102,7 +102,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.13}" + KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -157,8 +157,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -189,9 +189,8 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] @@ -385,10 +384,27 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" @@ -408,8 +424,8 @@ write_files: permissions: "0644" content: | [Unit] - Requires=docker.service - After=docker.service + Requires=containerd.service + After=containerd.service [Service] ExecStart=/opt/bin/health-monitor.sh container-runtime diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index c5701f4db..d30f210bb 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -93,7 +93,7 @@ kPe6XoSbiLm/kxk32T0= ) const ( - defaultVersion = "1.22.7" + defaultVersion = "1.25.5" ) type fakeCloudConfigProvider struct { @@ -128,9 +128,9 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.23.13"), - semver.MustParse("v1.24.7"), - semver.MustParse("v1.25.3"), + semver.MustParse("v1.24.9"), + semver.MustParse("v1.25.5"), + semver.MustParse("v1.26.0"), } var tests []userDataTestCase @@ -601,7 +601,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, ccProvider: &fakeCloudConfigProvider{ diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index a9145f640..6bbf3cee9 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -113,7 +113,7 @@ write_files: opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" arch=${HOST_ARCH-} if [ -z "$arch" ] then @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" + KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -232,9 +232,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml index 3de8da793..f8d79190e 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml @@ -90,28 +90,28 @@ write_files: curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat < /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - - path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - - path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - - - path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - - - path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - - - path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - - - path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - - - path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - - - path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - - path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - - path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - - path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - - - path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - - - path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - - - path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - - path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - -runcmd: - - systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml deleted file mode 100644 index 5e5aac8e7..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.24.0.yaml +++ /dev/null @@ -1,451 +0,0 @@ -#cloud-config - -hostname: node1 - -ssh_pwauth: false -ssh_authorized_keys: - - "ssh-rsa AAABBB" - -write_files: - - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - - path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - - path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - - path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - - - path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - - - path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - - - path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - - - path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - - - path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - - - path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - - path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - - path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - - path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - - - path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - - - path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - - path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - - path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - -runcmd: - - systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.25.3.yaml b/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.25.3.yaml rename to pkg/userdata/ubuntu/testdata/version-1.24.9.yaml index a2a512b45..90fdd8704 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.25.3.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml @@ -111,7 +111,7 @@ write_files: opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" arch=${HOST_ARCH-} if [ -z "$arch" ] then @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.3}" + KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.25.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.25.0.yaml deleted file mode 100644 index 5d6f8d440..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.25.0.yaml +++ /dev/null @@ -1,438 +0,0 @@ -#cloud-config - -hostname: node1 - -ssh_pwauth: false -ssh_authorized_keys: - - "ssh-rsa AAABBB" - -write_files: - - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - - path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - - path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - - path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - - - path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - - - path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - - - path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - - - path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - - - path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - - - path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - - path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - - path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - - path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - - - path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - - - path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - - - path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - - path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - -runcmd: - - systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.7.yaml b/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.24.7.yaml rename to pkg/userdata/ubuntu/testdata/version-1.25.5.yaml index 8dbed92ef..0c4aa9c4e 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.24.7.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml @@ -111,7 +111,7 @@ write_files: opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" arch=${HOST_ARCH-} if [ -z "$arch" ] then @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.7}" + KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.23.13.yaml b/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml similarity index 91% rename from pkg/userdata/ubuntu/testdata/version-1.23.13.yaml rename to pkg/userdata/ubuntu/testdata/version-1.26.0.yaml index ae7b981d8..289251fd6 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.23.13.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml @@ -90,28 +90,28 @@ write_files: curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat <>=%s", awsKeyID), fmt.Sprintf("<< AWS_SECRET_ACCESS_KEY >>=%s", awsSecret), @@ -558,7 +562,7 @@ func TestAWSFlatcarContainerdProvisioningE2E(t *testing.T) { scenario := scenario{ name: "flatcar with containerd in AWS", osName: "flatcar", - containerRuntime: "containerd", + containerRuntime: defaultContainerRuntime, kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } @@ -608,7 +612,7 @@ func TestAWSEbsEncryptionEnabledProvisioningE2E(t *testing.T) { scenario := scenario{ name: "AWS with ebs encryption enabled", osName: "ubuntu", - containerRuntime: "containerd", + containerRuntime: defaultContainerRuntime, kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } @@ -630,6 +634,7 @@ func TestAzureProvisioningE2E(t *testing.T) { } selector := Not(OsSelector("sles", "amzn2")) + // act params := []string{ fmt.Sprintf("<< AZURE_TENANT_ID >>=%s", azureTenantID), @@ -697,7 +702,7 @@ func TestAzureRedhatSatelliteProvisioningE2E(t *testing.T) { scenario := scenario{ name: "Azure redhat satellite server subscription", osName: "rhel", - containerRuntime: "docker", + containerRuntime: defaultContainerRuntime, kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } @@ -898,7 +903,7 @@ func TestVsphereResourcePoolProvisioningE2E(t *testing.T) { scenario := scenario{ name: "vSphere resource pool provisioning", osName: "flatcar", - containerRuntime: "docker", + containerRuntime: defaultContainerRuntime, kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } @@ -1010,7 +1015,7 @@ func TestUbuntuProvisioningWithUpgradeE2E(t *testing.T) { scenario := scenario{ name: "Ubuntu upgrade", osName: "ubuntu", - containerRuntime: "docker", + containerRuntime: defaultContainerRuntime, kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } @@ -1035,7 +1040,7 @@ func TestDeploymentControllerUpgradesMachineE2E(t *testing.T) { scenario := scenario{ name: "MachineDeployment upgrade", osName: "ubuntu", - containerRuntime: "docker", + containerRuntime: defaultContainerRuntime, kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateUpdateAndDelete, } diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index c747de3b1..104b3996e 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,9 +33,9 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.23.13"), - semver.MustParse("v1.24.7"), - semver.MustParse("v1.25.3"), + semver.MustParse("v1.24.9"), + semver.MustParse("v1.25.5"), + semver.MustParse("v1.26.0"), } operatingSystems = []providerconfigtypes.OperatingSystem{ @@ -279,7 +279,7 @@ func buildScenarios() []scenario { for _, operatingSystem := range operatingSystems { s := scenario{ name: fmt.Sprintf("%s-%s", operatingSystem, version), - containerRuntime: "docker", + containerRuntime: "containerd", kubernetesVersion: version.String(), osName: string(operatingSystem), executor: verifyCreateAndDelete, diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml index 29f9c769d..55b9c5f6e 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml @@ -29,7 +29,7 @@ spec: accessKeyId: << AWS_ACCESS_KEY_ID >> secretAccessKey: << AWS_SECRET_ACCESS_KEY >> region: "eu-central-1" - availabilityZone: "eu-central-1b" + availabilityZone: "eu-central-1a" vpcId: "vpc-819f62e9" instanceType: "t2.medium" instanceProfile: "kubernetes-v1" diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml index 5ed66030c..0af6c42a0 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml @@ -55,4 +55,4 @@ spec: rhelOrganizationName: "" rhelActivationKey: "" versions: - kubelet: 1.23.13 + kubelet: 1.24.9 From cea61b39aa740397e21f58631b740e5b700793eb Mon Sep 17 00:00:00 2001 From: Furkhat Date: Tue, 10 Jan 2023 12:26:26 +0300 Subject: [PATCH 269/489] exact string match fails, use conventional errors.Is instead (#1526) Signed-off-by: Furkhat Kasymovgeniiuulu Signed-off-by: Furkhat Kasymovgeniiuulu --- pkg/cloudprovider/provider/openstack/provider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 705b5f724..c41a68eed 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -763,7 +763,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine return false, osErrorToTerminalError(err, "failed to get compute client") } - if err := osservers.Delete(computeClient, instance.ID()).ExtractErr(); err != nil && err.Error() != "Resource not found" { + if err := osservers.Delete(computeClient, instance.ID()).ExtractErr(); err != nil && !errors.Is(err, &gophercloud.ErrDefault404{}) { return false, osErrorToTerminalError(err, "failed to delete instance") } @@ -1034,7 +1034,7 @@ func (p *provider) cleanupFloatingIP(machine *clusterv1alpha1.Machine, updater c if err != nil { return fmt.Errorf("failed to create the networkv2 client for region %s: %w", c.Region, err) } - if err := osfloatingips.Delete(netClient, floatingIPID).ExtractErr(); err != nil && err.Error() != "Resource not found" { + if err := osfloatingips.Delete(netClient, floatingIPID).ExtractErr(); err != nil && !errors.Is(err, &gophercloud.ErrDefault404{}) { return fmt.Errorf("failed to delete floating ip %s: %w", floatingIPID, err) } if err := updater(machine, func(m *clusterv1alpha1.Machine) { From 2ff7ea4ec4641d64ccff28eb089eb7823b762a45 Mon Sep 17 00:00:00 2001 From: Helene Durand <90329641+hdurand0710@users.noreply.github.com> Date: Wed, 11 Jan 2023 15:40:23 +0100 Subject: [PATCH 270/489] KubeVirt fix ci (DC) (#1527) Signed-off-by: Helene Durand Signed-off-by: Helene Durand --- test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index 6be2db2e7..f78cb46cc 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -38,7 +38,7 @@ spec: primaryDisk: osImage: http://image-repo.kube-system.svc.cluster.local/images/<< KUBEVIRT_OS_IMAGE >>.img size: "25Gi" - storageClassName: csi-rbd + storageClassName: px-csi-db dnsPolicy: "None" dnsConfig: nameservers: From 48a013651278a80b22dddbad15a40bc22411c701 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 16 Jan 2023 16:25:13 +0500 Subject: [PATCH 271/489] Drop support for SLES (#1525) Signed-off-by: Waleed Malik Signed-off-by: Waleed Malik --- .prow/provider-aws.yaml | 31 -- Makefile | 2 +- cmd/userdata/sles/main.go | 46 -- docs/operating-system.md | 28 +- examples/aws-machinedeployment.yaml | 1 - examples/azure-machinedeployment.yaml | 1 - pkg/cloudprovider/provider/aws/provider.go | 16 - .../provider/openstack/provider.go | 5 +- .../provider/vsphere/provider.go | 6 +- pkg/containerruntime/containerd.go | 2 - pkg/containerruntime/docker.go | 2 - pkg/providerconfig/types.go | 3 - pkg/providerconfig/types/types.go | 2 - pkg/userdata/manager/manager.go | 1 - pkg/userdata/sles/provider.go | 315 ------------ pkg/userdata/sles/provider_test.go | 476 ------------------ pkg/userdata/sles/sles.go | 59 --- .../sles/testdata/dist-upgrade-on-boot.yaml | 436 ---------------- .../kubelet-version-without-v-prefix.yaml | 434 ---------------- .../sles/testdata/multiple-dns-servers.yaml | 436 ---------------- .../sles/testdata/multiple-ssh-keys.yaml | 436 ---------------- .../openstack-overwrite-cloud-config.yaml | 442 ---------------- pkg/userdata/sles/testdata/openstack.yaml | 442 ---------------- .../sles/testdata/version-1.24.9.yaml | 434 ---------------- .../sles/testdata/version-1.25.5.yaml | 434 ---------------- .../sles/testdata/version-1.26.0.yaml | 434 ---------------- .../sles/testdata/vsphere-mirrors.yaml | 453 ----------------- pkg/userdata/sles/testdata/vsphere-proxy.yaml | 460 ----------------- pkg/userdata/sles/testdata/vsphere.yaml | 443 ---------------- test/e2e/provisioning/all_e2e_test.go | 30 +- test/e2e/provisioning/helper.go | 1 - .../machinedeployment-aws-arm-machines.yaml | 1 - ...deployment-aws-ebs-encryption-enabled.yaml | 1 - .../machinedeployment-aws-spot-instances.yaml | 1 - .../testdata/machinedeployment-aws.yaml | 1 - 35 files changed, 20 insertions(+), 6295 deletions(-) delete mode 100644 cmd/userdata/sles/main.go delete mode 100644 pkg/userdata/sles/provider.go delete mode 100644 pkg/userdata/sles/provider_test.go delete mode 100644 pkg/userdata/sles/sles.go delete mode 100644 pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml delete mode 100644 pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml delete mode 100644 pkg/userdata/sles/testdata/multiple-dns-servers.yaml delete mode 100644 pkg/userdata/sles/testdata/multiple-ssh-keys.yaml delete mode 100644 pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml delete mode 100644 pkg/userdata/sles/testdata/openstack.yaml delete mode 100644 pkg/userdata/sles/testdata/version-1.24.9.yaml delete mode 100644 pkg/userdata/sles/testdata/version-1.25.5.yaml delete mode 100644 pkg/userdata/sles/testdata/version-1.26.0.yaml delete mode 100644 pkg/userdata/sles/testdata/vsphere-mirrors.yaml delete mode 100644 pkg/userdata/sles/testdata/vsphere-proxy.yaml delete mode 100644 pkg/userdata/sles/testdata/vsphere.yaml diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index db17955f8..8746fe337 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -204,37 +204,6 @@ presubmits: limits: memory: 7Gi - - name: pull-machine-controller-e2e-aws-sles - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAWSSLESProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: aws - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - name: pull-machine-controller-e2e-aws-flatcar-coreos-cloud-init always_run: false decorate: true diff --git a/Makefile b/Makefile index 4118c4697..ae3ddf1df 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ IMAGE_TAG = \ $(shell echo $$(git rev-parse HEAD && if [[ -n $$(git status --porcelain) ]]; then echo '-dirty'; fi)|tr -d ' ') IMAGE_NAME ?= $(REGISTRY)/$(REGISTRY_NAMESPACE)/machine-controller:$(IMAGE_TAG) -OS = amzn2 centos ubuntu sles rhel flatcar rockylinux +OS = amzn2 centos ubuntu rhel flatcar rockylinux USERDATA_BIN = $(patsubst %, machine-controller-userdata-%, $(OS)) BASE64_ENC = \ diff --git a/cmd/userdata/sles/main.go b/cmd/userdata/sles/main.go deleted file mode 100644 index 52b8b29e9..000000000 --- a/cmd/userdata/sles/main.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for Ubuntu. -// - -package main - -import ( - "flag" - - userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" - "github.com/kubermatic/machine-controller/pkg/userdata/sles" - - "k8s.io/klog" -) - -func main() { - // Parse flags. - var debug bool - - flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging") - flag.Parse() - - // Instantiate provider and start plugin. - var provider = &sles.Provider{} - var p = userdataplugin.New(provider, debug) - - if err := p.Run(); err != nil { - klog.Fatalf("error running Ubuntu plugin: %v", err) - } -} diff --git a/docs/operating-system.md b/docs/operating-system.md index 93f16a872..8d096a529 100644 --- a/docs/operating-system.md +++ b/docs/operating-system.md @@ -4,19 +4,19 @@ ### Cloud provider -| | Ubuntu | CentOS | Flatcar | RHEL | SLES | Amazon Linux 2 | Rocky Linux | -|---|---|---|---|---|---|---|---| -| AWS | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Azure | ✓ | ✓ | ✓ | ✓ | x | x | ✓ | -| Digitalocean | ✓ | ✓ | x | x | x | x | ✓ | -| Equinix Metal | ✓ | ✓ | ✓ | x | x | x | ✓ | -| Google Cloud Platform | ✓ | x | x | x | x | x | x | -| Hetzner | ✓ | ✓ | x | x | x | x | ✓ | -| KubeVirt | ✓ | ✓ | ✓ | ✓ | x | x | ✓ | -| Nutanix | ✓ | ✓ | x | x | x | x | x | -| Openstack | ✓ | ✓ | ✓ | ✓ | x | x | ✓ | -| VMware Cloud Director | ✓ | x | x | x | x | x | x | -| VSphere | ✓ | ✓ | ✓ | ✓ | x | x | ✓ | +| | Ubuntu | CentOS | Flatcar | RHEL | Amazon Linux 2 | Rocky Linux | +|---|---|---|---|---|---|---| +| AWS | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Azure | ✓ | ✓ | ✓ | ✓ | x | ✓ | +| Digitalocean | ✓ | ✓ | x | x | x | ✓ | +| Equinix Metal | ✓ | ✓ | ✓ | x | x | ✓ | +| Google Cloud Platform | ✓ | x | x | x | x | x | +| Hetzner | ✓ | ✓ | x | x | x | ✓ | +| KubeVirt | ✓ | ✓ | ✓ | ✓ | x | ✓ | +| Nutanix | ✓ | ✓ | x | x | x | x | +| Openstack | ✓ | ✓ | ✓ | ✓ | x | ✓ | +| VMware Cloud Director | ✓ | x | x | x | x | x | +| VSphere | ✓ | ✓ | ✓ | ✓ | x | ✓ | ## Configuring a operating system @@ -28,7 +28,6 @@ Allowed values: - `flatcar` - `rhel` - `rockylinux` -- `sles` - `ubuntu` OS specific settings can be set via `machine.spec.providerConfig.operatingSystemSpec`. @@ -44,5 +43,4 @@ Machine controller may work with other OS versions that are not listed in the ta | CentOS | 7.4.x, 7.6.x, 7.7.x | | RHEL | 8.x | | Rocky Linux | 8.5 | -| SLES | SLES 15 SP3 | | Ubuntu | 20.04 LTS, 22.04 LTS | diff --git a/examples/aws-machinedeployment.yaml b/examples/aws-machinedeployment.yaml index 088c2d348..633e1ed3c 100644 --- a/examples/aws-machinedeployment.yaml +++ b/examples/aws-machinedeployment.yaml @@ -67,7 +67,6 @@ spec: assignPublicIP: false tags: "KubernetesCluster": "6qsm86c2d" - # Can be 'ubuntu', 'centos', `rhel` or 'sles' operatingSystem: "ubuntu" operatingSystemSpec: disableAutoUpdate: true diff --git a/examples/azure-machinedeployment.yaml b/examples/azure-machinedeployment.yaml index d4ed1546e..63166665e 100644 --- a/examples/azure-machinedeployment.yaml +++ b/examples/azure-machinedeployment.yaml @@ -79,7 +79,6 @@ spec: # that protects your applications and data from datacenter failures. zones: - "1" - # Can be 'ubuntu','flatcar' or 'rhel' operatingSystem: "flatcar" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 425be29a7..2891d0ab3 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -152,20 +152,6 @@ var ( owner: "099720109477", }, }, - providerconfigtypes.OperatingSystemSLES: { - awstypes.CPUArchitectureX86_64: { - // Be as precise as possible - otherwise we might get a nightly dev build - description: "SUSE Linux Enterprise Server 15 SP3 (HVM, 64-bit, SSD-Backed)", - // The AWS marketplace ID from SLES - owner: "013907871322", - }, - awstypes.CPUArchitectureARM64: { - // Be as precise as possible - otherwise we might get a nightly dev build - description: "SUSE Linux Enterprise Server 15 SP3 (HVM, 64-bit, SSD-Backed)", - // The AWS marketplace ID from SLES - owner: "013907871322", - }, - }, providerconfigtypes.OperatingSystemRHEL: { awstypes.CPUArchitectureX86_64: { // Be as precise as possible - otherwise we might get a nightly dev build @@ -351,8 +337,6 @@ func getDefaultRootDevicePath(os providerconfigtypes.OperatingSystem) (string, e return rootDevicePathSDA, nil case providerconfigtypes.OperatingSystemRockyLinux: return rootDevicePathSDA, nil - case providerconfigtypes.OperatingSystemSLES: - return rootDevicePathXVDA, nil case providerconfigtypes.OperatingSystemRHEL: return rootDevicePathSDA, nil case providerconfigtypes.OperatingSystemFlatcar: diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index c41a68eed..938d74f02 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -442,7 +442,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha } func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { - c, pc, _, err := p.getConfig(spec.ProviderSpec) + c, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) } @@ -537,9 +537,6 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) if _, err := getAvailabilityZone(computeClient, c); err != nil { return fmt.Errorf("failed to get availability zone %q: %w", c.AvailabilityZone, err) } - if pc.OperatingSystem == providerconfigtypes.OperatingSystemSLES { - return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, providerconfigtypes.ErrOSNotSupported) - } // Optional fields. if len(c.SecurityGroups) != 0 { for _, s := range c.SecurityGroups { diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 7151f15ed..3b4395294 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -205,15 +205,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { - config, pc, _, err := p.getConfig(spec.ProviderSpec) + config, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to get config: %w", err) } - if pc.OperatingSystem == providerconfigtypes.OperatingSystemSLES { - return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, providerconfigtypes.ErrOSNotSupported) - } - session, err := NewSession(ctx, config) if err != nil { return fmt.Errorf("failed to create vCenter session: %w", err) diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index 17a521eaa..02bc3e85f 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -83,8 +83,6 @@ func (eng *Containerd) ScriptFor(os types.OperatingSystem) (string, error) { case types.OperatingSystemFlatcar: err := containedFlatcarTemplate.Execute(&buf, args) return buf.String(), err - case types.OperatingSystemSLES: - return "", nil } return "", fmt.Errorf("unknown OS: %s", os) diff --git a/pkg/containerruntime/docker.go b/pkg/containerruntime/docker.go index 398368a54..801e8e0ea 100644 --- a/pkg/containerruntime/docker.go +++ b/pkg/containerruntime/docker.go @@ -107,8 +107,6 @@ func (eng *Docker) ScriptFor(os types.OperatingSystem) (string, error) { case types.OperatingSystemFlatcar: err := dockerFlatcarTemplate.Execute(&buf, args) return buf.String(), err - case types.OperatingSystemSLES: - return "", nil } return "", fmt.Errorf("unknown OS: %s", os) diff --git a/pkg/providerconfig/types.go b/pkg/providerconfig/types.go index 1988d83d5..cf1520071 100644 --- a/pkg/providerconfig/types.go +++ b/pkg/providerconfig/types.go @@ -30,7 +30,6 @@ import ( "github.com/kubermatic/machine-controller/pkg/userdata/flatcar" "github.com/kubermatic/machine-controller/pkg/userdata/rhel" "github.com/kubermatic/machine-controller/pkg/userdata/rockylinux" - "github.com/kubermatic/machine-controller/pkg/userdata/sles" "github.com/kubermatic/machine-controller/pkg/userdata/ubuntu" corev1 "k8s.io/api/core/v1" @@ -195,8 +194,6 @@ func DefaultOperatingSystemSpec( return flatcar.DefaultConfigForCloud(operatingSystemSpec, cloudProvider, externalBootstrapEnabled), nil case providerconfigtypes.OperatingSystemRHEL: return rhel.DefaultConfig(operatingSystemSpec), nil - case providerconfigtypes.OperatingSystemSLES: - return sles.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemUbuntu: return ubuntu.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemRockyLinux: diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index ac4ed465e..8e7ca8826 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -38,7 +38,6 @@ const ( OperatingSystemUbuntu OperatingSystem = "ubuntu" OperatingSystemCentOS OperatingSystem = "centos" OperatingSystemAmazonLinux2 OperatingSystem = "amzn2" - OperatingSystemSLES OperatingSystem = "sles" OperatingSystemRHEL OperatingSystem = "rhel" OperatingSystemFlatcar OperatingSystem = "flatcar" OperatingSystemRockyLinux OperatingSystem = "rockylinux" @@ -76,7 +75,6 @@ var ( OperatingSystemUbuntu, OperatingSystemCentOS, OperatingSystemAmazonLinux2, - OperatingSystemSLES, OperatingSystemRHEL, OperatingSystemFlatcar, OperatingSystemRockyLinux, diff --git a/pkg/userdata/manager/manager.go b/pkg/userdata/manager/manager.go index ffa291944..473899333 100644 --- a/pkg/userdata/manager/manager.go +++ b/pkg/userdata/manager/manager.go @@ -48,7 +48,6 @@ var ( providerconfigtypes.OperatingSystemCentOS, providerconfigtypes.OperatingSystemFlatcar, providerconfigtypes.OperatingSystemRHEL, - providerconfigtypes.OperatingSystemSLES, providerconfigtypes.OperatingSystemUbuntu, providerconfigtypes.OperatingSystemRockyLinux, } diff --git a/pkg/userdata/sles/provider.go b/pkg/userdata/sles/provider.go deleted file mode 100644 index 7e3a3c773..000000000 --- a/pkg/userdata/sles/provider.go +++ /dev/null @@ -1,315 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for SLES. -// - -package sles - -import ( - "bytes" - "errors" - "fmt" - "text/template" - - "github.com/Masterminds/semver/v3" - - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - userdatahelper "github.com/kubermatic/machine-controller/pkg/userdata/helper" -) - -// Provider is a pkg/userdata/plugin.Provider implementation. -type Provider struct{} - -// UserData renders user-data template to string. -func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { - tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(userDataTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse user-data template: %w", err) - } - - kubeletVersion, err := semver.NewVersion(req.MachineSpec.Versions.Kubelet) - if err != nil { - return "", fmt.Errorf("invalid kubelet version: %w", err) - } - - pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) - if err != nil { - return "", fmt.Errorf("failed to get providerSpec: %w", err) - } - - if pconfig.OverwriteCloudConfig != nil { - req.CloudConfig = *pconfig.OverwriteCloudConfig - } - - if pconfig.Network.IsStaticIPConfig() { - return "", errors.New("static IP config is not supported with SLES") - } - - slesConfig, err := LoadConfig(pconfig.OperatingSystemSpec) - if err != nil { - return "", fmt.Errorf("failed to get sles config from provider config: %w", err) - } - - kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) - if err != nil { - return "", err - } - - kubernetesCACert, err := userdatahelper.GetCACert(req.Kubeconfig) - if err != nil { - return "", fmt.Errorf("error extracting cacert: %w", err) - } - - crEngine := req.ContainerRuntime.Engine(kubeletVersion) - crConfig, err := crEngine.Config() - if err != nil { - return "", fmt.Errorf("failed to generate container runtime config: %w", err) - } - - crAuthConfig, err := crEngine.AuthConfig() - if err != nil { - return "", fmt.Errorf("failed to generate container runtime auth config: %w", err) - } - - data := struct { - plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - OSConfig *Config - KubeletVersion string - Kubeconfig string - KubernetesCACert string - NodeIPScript string - ExtraKubeletFlags []string - ContainerRuntimeConfigFileName string - ContainerRuntimeConfig string - ContainerRuntimeAuthConfigFileName string - ContainerRuntimeAuthConfig string - ContainerRuntimeName string - }{ - UserDataRequest: req, - ProviderSpec: pconfig, - OSConfig: slesConfig, - KubeletVersion: kubeletVersion.String(), - Kubeconfig: kubeconfigString, - KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(pconfig.Network.GetIPFamily()), - ExtraKubeletFlags: crEngine.KubeletFlags(), - ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), - ContainerRuntimeConfig: crConfig, - ContainerRuntimeAuthConfigFileName: crEngine.AuthConfigFileName(), - ContainerRuntimeAuthConfig: crAuthConfig, - ContainerRuntimeName: crEngine.String(), - } - b := &bytes.Buffer{} - err = tmpl.Execute(b, data) - if err != nil { - return "", fmt.Errorf("failed to execute user-data template: %w", err) - } - return userdatahelper.CleanupTemplateOutput(b.String()) -} - -// UserData template. -const userDataTemplate = `#cloud-config -{{ if ne .CloudProviderName "aws" }} -hostname: {{ .MachineSpec.Name }} -{{- /* Never set the hostname on AWS nodes. Kubernetes(kube-proxy) requires the hostname to be the private dns name */}} -{{ end }} - -{{- if .OSConfig.DistUpgradeOnBoot }} -package_upgrade: true -package_reboot_if_required: true -{{- end }} - -ssh_pwauth: false - -{{- if .ProviderSpec.SSHPublicKeys }} -ssh_authorized_keys: -{{- range .ProviderSpec.SSHPublicKeys }} -- "{{ . }}" -{{- end }} -{{- end }} - -write_files: -{{- if .HTTPProxy }} -- path: "/etc/environment" - content: | - PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" -{{ proxyEnvironment .HTTPProxy .NoProxy | indent 4 }} -{{- end }} - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | -{{ journalDConfig | indent 4 }} - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | -{{ kernelModulesScript | indent 4 }} - -- path: "/etc/sysctl.d/k8s.conf" - content: | -{{ kernelSettings | indent 4 }} - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail -{{- /* As we added some modules and don't want to reboot, restart the service */}} - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - {{- if or (eq .CloudProviderName "vsphere") (eq .CloudProviderName "vmware-cloud-director") }} - open-vm-tools \ - {{- end }} - ipvsadm - -{{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -{{- if ne (len .CloudConfig) 0 }} -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | -{{ .CloudConfig | indent 4 }} -{{- end }} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | -{{ .NodeIPScript | indent 4 }} - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | -{{ .Kubeconfig | indent 4 }} - -- path: "/etc/kubernetes/pki/ca.crt" - content: | -{{ .KubernetesCACert | indent 4 }} - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }} - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: {{ .ContainerRuntimeConfigFileName }} - permissions: "0644" - content: | -{{ .ContainerRuntimeConfig | indent 4 }} - -{{- if and (eq .ContainerRuntimeName "docker") .ContainerRuntimeAuthConfig }} - -- path: {{ .ContainerRuntimeAuthConfigFileName }} - permissions: "0600" - content: | -{{ .ContainerRuntimeAuthConfig | indent 4 }} -{{- end }} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | -{{ kubeletHealthCheckSystemdUnit | indent 4 }} - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | -{{ containerRuntimeHealthCheckSystemdUnit .ContainerRuntimeName | indent 4 }} - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -{{- with .ProviderSpec.CAPublicKey }} - -- path: "/etc/ssh/trusted-user-ca-keys.pem" - content: | -{{ . | indent 4 }} - -- path: "/etc/ssh/sshd_config" - content: | -{{ sshConfigAddendum | indent 4 }} - append: true -{{- end }} - -runcmd: -- systemctl start setup.service -` diff --git a/pkg/userdata/sles/provider_test.go b/pkg/userdata/sles/provider_test.go deleted file mode 100644 index d7420d6c7..000000000 --- a/pkg/userdata/sles/provider_test.go +++ /dev/null @@ -1,476 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for SLES. -// - -package sles - -import ( - "encoding/json" - "flag" - "fmt" - "net" - "testing" - - "github.com/Masterminds/semver/v3" - - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - "github.com/kubermatic/machine-controller/pkg/containerruntime" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - testhelper "github.com/kubermatic/machine-controller/pkg/test" - "github.com/kubermatic/machine-controller/pkg/userdata/cloud" - "github.com/kubermatic/machine-controller/pkg/userdata/convert" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - "k8s.io/utils/pointer" -) - -var ( - update = flag.Bool("update", false, "update testdata files") - - pemCertificate = `-----BEGIN CERTIFICATE----- -MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV -BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG -A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 -DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 -NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG -cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv -c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B -AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS -R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT -ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk -JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 -mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW -caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G -A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt -hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB -MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES -MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv -bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h -U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao -eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 -UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD -58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n -sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF -kPe6XoSbiLm/kxk32T0= ------END CERTIFICATE-----` - - kubeconfig = &clientcmdapi.Config{ - Clusters: map[string]*clientcmdapi.Cluster{ - "": { - Server: "https://server/", - CertificateAuthorityData: []byte(pemCertificate), - }, - }, - AuthInfos: map[string]*clientcmdapi.AuthInfo{ - "": { - Token: "my-token", - }, - }, - } - - kubeletFeatureGates = map[string]bool{ - "RotateKubeletServerCertificate": true, - } -) - -const ( - defaultVersion = "1.25.5" -) - -type fakeCloudConfigProvider struct { - config string - name string - err error -} - -func (p *fakeCloudConfigProvider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return p.config, p.name, p.err -} - -// userDataTestCase contains the data for a table-driven test. -type userDataTestCase struct { - name string - spec clusterv1alpha1.MachineSpec - ccProvider cloud.ConfigProvider - osConfig *Config - providerSpec *providerconfigtypes.Config - DNSIPs []net.IP - kubernetesCACert string - externalCloudProvider bool - httpProxy string - noProxy string - insecureRegistries string - registryMirrors string - pauseImage string - containerruntime string -} - -func simpleVersionTests() []userDataTestCase { - versions := []*semver.Version{ - semver.MustParse("v1.24.9"), - semver.MustParse("v1.25.5"), - semver.MustParse("v1.26.0"), - } - - var tests []userDataTestCase - for _, v := range versions { - tests = append(tests, userDataTestCase{ - name: fmt.Sprintf("version-%s", v.String()), - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: v.String(), - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }) - } - - return tests -} - -// TestUserDataGeneration runs the data generation for different -// environments. -func TestUserDataGeneration(t *testing.T) { - t.Parallel() - - tests := simpleVersionTests() - tests = append(tests, []userDataTestCase{ - { - name: "dist-upgrade-on-boot", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: true, - }, - }, - { - name: "multiple-ssh-keys", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD", "ssh-rsa EEEFFF"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "kubelet-version-without-v-prefix", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "openstack", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "openstack", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "openstack", - config: "{openstack-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "openstack-overwrite-cloud-config", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "openstack", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: pointer.String("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "openstack", - config: "{openstack-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "vsphere", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: pointer.String("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "vsphere-mirrors", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: pointer.String("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - httpProxy: "http://192.168.100.100:3128/", - noProxy: "192.168.1.0", - registryMirrors: "https://registry.docker-cn.com/", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "vsphere-proxy", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: pointer.String("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - httpProxy: "http://192.168.100.100:3128/", - noProxy: "192.168.1.0", - insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "multiple-dns-servers", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - }...) - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - rProviderSpec := test.providerSpec - osConfigByte, err := json.Marshal(test.osConfig) - if err != nil { - t.Fatal(err) - } - rProviderSpec.OperatingSystemSpec = runtime.RawExtension{ - Raw: osConfigByte, - } - - providerSpecRaw, err := json.Marshal(rProviderSpec) - if err != nil { - t.Fatal(err) - } - test.spec.ProviderSpec = clusterv1alpha1.ProviderSpec{ - Value: &runtime.RawExtension{ - Raw: providerSpecRaw, - }, - } - provider := Provider{} - - cloudConfig, cloudProviderName, err := test.ccProvider.GetCloudConfig(test.spec) - if err != nil { - t.Fatalf("failed to get cloud config: %v", err) - } - - containerRuntimeOpts := containerruntime.Opts{ - ContainerRuntime: test.containerruntime, - InsecureRegistries: test.insecureRegistries, - RegistryMirrors: test.registryMirrors, - } - containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) - if err != nil { - t.Fatalf("failed to generate container runtime config: %v", err) - } - - req := plugin.UserDataRequest{ - MachineSpec: test.spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: cloudProviderName, - KubeletCloudProviderName: cloudProviderName, - DNSIPs: test.DNSIPs, - ExternalCloudProvider: test.externalCloudProvider, - HTTPProxy: test.httpProxy, - NoProxy: test.noProxy, - PauseImage: test.pauseImage, - KubeletFeatureGates: kubeletFeatureGates, - ContainerRuntime: containerRuntimeConfig, - } - s, err := provider.UserData(req) - if err != nil { - t.Fatal(err) - } - - // Check if we can gzip it. - if _, err := convert.GzipString(s); err != nil { - t.Fatal(err) - } - goldenName := test.name + ".yaml" - testhelper.CompareOutput(t, goldenName, s, *update) - }) - } -} diff --git a/pkg/userdata/sles/sles.go b/pkg/userdata/sles/sles.go deleted file mode 100644 index 97d62c1e2..000000000 --- a/pkg/userdata/sles/sles.go +++ /dev/null @@ -1,59 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package sles - -import ( - "encoding/json" - - "k8s.io/apimachinery/pkg/runtime" -) - -// Config contains specific configuration for SLES. -type Config struct { - DistUpgradeOnBoot bool `json:"distUpgradeOnBoot"` -} - -func DefaultConfig(operatingSystemSpec runtime.RawExtension) runtime.RawExtension { - if operatingSystemSpec.Raw == nil { - operatingSystemSpec.Raw, _ = json.Marshal(Config{}) - } - - return operatingSystemSpec -} - -// LoadConfig retrieves the SLES configuration from raw data. -func LoadConfig(r runtime.RawExtension) (*Config, error) { - r = DefaultConfig(r) - cfg := Config{} - - if err := json.Unmarshal(r.Raw, &cfg); err != nil { - return nil, err - } - return &cfg, nil -} - -// Spec return the configuration as raw data. -func (cfg *Config) Spec() (*runtime.RawExtension, error) { - ext := &runtime.RawExtension{} - b, err := json.Marshal(cfg) - if err != nil { - return nil, err - } - - ext.Raw = b - return ext, nil -} diff --git a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml deleted file mode 100644 index 9bfb4a901..000000000 --- a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml +++ /dev/null @@ -1,436 +0,0 @@ -#cloud-config - -hostname: node1 - -package_upgrade: true -package_reboot_if_required: true - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml deleted file mode 100644 index cafef8c0b..000000000 --- a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml +++ /dev/null @@ -1,434 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml deleted file mode 100644 index 412be5367..000000000 --- a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml +++ /dev/null @@ -1,436 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - - 10.10.10.11 - - 10.10.10.12 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml deleted file mode 100644 index 60e462537..000000000 --- a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml +++ /dev/null @@ -1,436 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" -- "ssh-rsa CCCDDD" -- "ssh-rsa EEEFFF" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml deleted file mode 100644 index 04a9240cf..000000000 --- a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml +++ /dev/null @@ -1,442 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=openstack \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/openstack.yaml b/pkg/userdata/sles/testdata/openstack.yaml deleted file mode 100644 index a01fd693e..000000000 --- a/pkg/userdata/sles/testdata/openstack.yaml +++ /dev/null @@ -1,442 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=openstack \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {openstack-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - - 10.10.10.11 - - 10.10.10.12 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.24.9.yaml b/pkg/userdata/sles/testdata/version-1.24.9.yaml deleted file mode 100644 index baf817621..000000000 --- a/pkg/userdata/sles/testdata/version-1.24.9.yaml +++ /dev/null @@ -1,434 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.25.5.yaml b/pkg/userdata/sles/testdata/version-1.25.5.yaml deleted file mode 100644 index cafef8c0b..000000000 --- a/pkg/userdata/sles/testdata/version-1.25.5.yaml +++ /dev/null @@ -1,434 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.26.0.yaml b/pkg/userdata/sles/testdata/version-1.26.0.yaml deleted file mode 100644 index 87c635150..000000000 --- a/pkg/userdata/sles/testdata/version-1.26.0.yaml +++ /dev/null @@ -1,434 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.0}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml deleted file mode 100644 index 38b631858..000000000 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ /dev/null @@ -1,453 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: -- path: "/etc/environment" - content: | - PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - open-vm-tools \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry.docker-cn.com/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml deleted file mode 100644 index 63d3098e2..000000000 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ /dev/null @@ -1,460 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: -- path: "/etc/environment" - content: | - PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - open-vm-tools \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - [plugins."io.containerd.grpc.v1.cri".registry.configs] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000".tls] - insecure_skip_verify = true - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000".tls] - insecure_skip_verify = true - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml deleted file mode 100644 index 1be34fcde..000000000 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ /dev/null @@ -1,443 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - open-vm-tools \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 9728a608d..d52ffc30e 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -339,7 +339,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { fmt.Sprintf("<< NETWORK_NAME >>=%s", osNetwork), } - selector := Not(OsSelector("sles", "amzn2")) + selector := Not(OsSelector("amzn2")) runScenarios(t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -500,28 +500,6 @@ func TestAWSARMProvisioningE2E(t *testing.T) { runScenarios(t, selector, params, AWSManifestARM, fmt.Sprintf("aws-%s", *testRunIdentifier)) } -// TestAWSSLESProvisioningE2E - a test suite that exercises AWS provider -// by requesting nodes with different combination of container runtime type, container runtime version and the OS flavour. -func TestAWSSLESProvisioningE2E(t *testing.T) { - t.Parallel() - - // test data - awsKeyID := os.Getenv("AWS_E2E_TESTS_KEY_ID") - awsSecret := os.Getenv("AWS_E2E_TESTS_SECRET") - if len(awsKeyID) == 0 || len(awsSecret) == 0 { - t.Fatal("unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") - } - - // act - params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), - fmt.Sprintf("<< AWS_SECRET_ACCESS_KEY >>=%s", awsSecret), - } - - // We would like to test SLES image only in this test as the other images are tested in TestAWSProvisioningE2E - selector := OsSelector("sles") - runScenarios(t, selector, params, AWSManifest, fmt.Sprintf("aws-%s", *testRunIdentifier)) -} - func TestAWSFlatcarCoreOSCloudInit8ProvisioningE2E(t *testing.T) { t.Parallel() @@ -633,7 +611,7 @@ func TestAzureProvisioningE2E(t *testing.T) { t.Fatal("unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") } - selector := Not(OsSelector("sles", "amzn2")) + selector := Not(OsSelector("amzn2")) // act params := []string{ @@ -876,7 +854,7 @@ func getVSphereTestParams(t *testing.T) []string { func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() - selector := Not(OsSelector("sles", "amzn2", "rockylinux", "flatcar")) + selector := Not(OsSelector("amzn2", "rockylinux", "flatcar")) params := getVSphereTestParams(t) runScenarios(t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) @@ -937,7 +915,7 @@ func TestScalewayProvisioningE2E(t *testing.T) { t.Fatal("unable to run the test suite, SCW_E2E_TEST_PROJECT_ID environment variable cannot be empty") } - selector := Not(OsSelector("sles", "rhel", "flatcar", "rockylinux")) + selector := Not(OsSelector("rhel", "flatcar", "rockylinux")) // act params := []string{ fmt.Sprintf("<< SCW_ACCESS_KEY >>=%s", scwAccessKey), diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 104b3996e..aec9d26dc 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -42,7 +42,6 @@ var ( providerconfigtypes.OperatingSystemUbuntu, providerconfigtypes.OperatingSystemCentOS, providerconfigtypes.OperatingSystemAmazonLinux2, - providerconfigtypes.OperatingSystemSLES, providerconfigtypes.OperatingSystemRHEL, providerconfigtypes.OperatingSystemFlatcar, providerconfigtypes.OperatingSystemRockyLinux, diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml index b38cc05b5..793231b7f 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml @@ -45,7 +45,6 @@ spec: "KubernetesCluster": "randomString" # Disabling the public IP assignment requires a private subnet with internet access. assignPublicIP: true - # Can be 'ubuntu', 'centos' or 'sles' operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml index 68ff9cd82..0130744c6 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml @@ -44,7 +44,6 @@ spec: "KubernetesCluster": "randomString" # Disabling the public IP assignment requires a private subnet with internet access. assignPublicIP: true - # Can be 'ubuntu', 'centos' operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml index 55b9c5f6e..e7febdc8a 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml @@ -49,7 +49,6 @@ spec: "KubernetesCluster": "randomString" # Disabling the public IP assignment requires a private subnet with internet access. assignPublicIP: true - # Can be 'ubuntu', 'centos' or 'sles' operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws.yaml index 465f10146..915f71254 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws.yaml @@ -47,7 +47,6 @@ spec: "KubernetesCluster": "randomString" # Disabling the public IP assignment requires a private subnet with internet access. assignPublicIP: true - # Can be 'ubuntu', 'centos' or 'sles' operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false From 08e62dc29690b193a47de3899d154af12821966e Mon Sep 17 00:00:00 2001 From: Marcin Franczyk Date: Thu, 19 Jan 2023 10:57:53 +0100 Subject: [PATCH 272/489] Remove env vars that control PVC clone. (#1529) Signed-off-by: Marcin Franczyk Signed-off-by: Marcin Franczyk --- .../provider/kubevirt/provider.go | 114 +--------------- .../provider/kubevirt/provider_test.go | 126 +----------------- .../provider/kubevirt/types/types.go | 8 ++ 3 files changed, 13 insertions(+), 235 deletions(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 92415b521..28654c0f9 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -75,23 +75,6 @@ const ( httpSource imageSource = "http" // pvcSource defines the pvc source type for VM Disk Image. pvcSource imageSource = "pvc" - // kubeVirtImagesNamespace namespace contains globally available custom images and cached standard images. - kubeVirtImagesNamespace = "kubevirt-images" - dataVolumeStandardImageAnnotation = "kubevirt-initialization.k8c.io/standard-image" - osAnnotationForCustomDisk = "cdi.kubevirt.io/os-type" -) - -var ( - supportedOS = map[providerconfigtypes.OperatingSystem]*struct{}{ - providerconfigtypes.OperatingSystemCentOS: nil, - providerconfigtypes.OperatingSystemUbuntu: nil, - providerconfigtypes.OperatingSystemRHEL: nil, - providerconfigtypes.OperatingSystemFlatcar: nil, - providerconfigtypes.OperatingSystemRockyLinux: nil, - } - errInvalidOsImage = fmt.Errorf("invalid primaryDisk.osImage") - errCustomImage = fmt.Errorf("custom-image cloning not allowed") - errStandardImage = fmt.Errorf("standard-image cloning not allowed") ) type provider struct { @@ -120,8 +103,6 @@ type Config struct { SecondaryDisks []SecondaryDisks NodeAffinityPreset NodeAffinityPreset TopologySpreadConstraints []corev1.TopologySpreadConstraint - AllowPVCClone bool - AllowCustomImages bool } type AffinityType string @@ -331,16 +312,6 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf(`failed to parse "topologySpreadConstraints" field: %w`, err) } - config.AllowPVCClone, err = isImageCloningAllowed() - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "KUBEVIRT_ALLOW_PVC_CLONE" environment variable: %w`, err) - } - - config.AllowCustomImages, err = isCustomImageAllowed() - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "KUBEVIRT_ALLOW_CUSTOM_IMAGES" environment variable: %w`, err) - } - return &config, pconfig, nil } @@ -394,7 +365,7 @@ func (p *provider) parseTopologySpreadConstraint(topologyConstraints []kubevirtt return parsedTopologyConstraints, nil } -func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, nameSpace string) (*cdiv1beta1.DataVolumeSource, error) { +func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, namespace string) (*cdiv1beta1.DataVolumeSource, error) { osImage, err := p.configVarResolver.GetConfigVarStringValue(primaryDisk.OsImage) if err != nil { return nil, fmt.Errorf(`failed to get value of "primaryDisk.osImage" field: %w`, err) @@ -410,7 +381,7 @@ func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, nam if namespaceAndName := strings.Split(osImage, "/"); len(namespaceAndName) >= 2 { return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: namespaceAndName[1], Namespace: namespaceAndName[0]}}, nil } - return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: nameSpace}}, nil + return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: namespace}}, nil default: // handle old API for backward compatibility. if _, err = url.ParseRequestURI(osImage); err == nil { @@ -419,7 +390,7 @@ func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, nam if namespaceAndName := strings.Split(osImage, "/"); len(namespaceAndName) >= 2 { return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: namespaceAndName[1], Namespace: namespaceAndName[0]}}, nil } - return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: nameSpace}}, nil + return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: namespace}}, nil } } @@ -436,34 +407,6 @@ func getNamespace() string { return ns } -// isImageCloningAllowed returns whether image-cloning is allowed or not. -// Default value is `true`. -func isImageCloningAllowed() (bool, error) { - value := os.Getenv("KUBEVIRT_ALLOW_PVC_CLONE") - if value == "" { - return true, nil - } - isImageCloningEnabled, err := strconv.ParseBool(value) - if err != nil { - return false, err - } - return isImageCloningEnabled, nil -} - -// isCustomImageAllowed returns whether custom-image for cloning is allowed or not. -// Default value is `true`. -func isCustomImageAllowed() (bool, error) { - value := os.Getenv("KUBEVIRT_ALLOW_CUSTOM_IMAGES") - if value == "" { - return true, nil - } - isCustomImagesEnabled, err := strconv.ParseBool(value) - if err != nil { - return false, err - } - return isCustomImagesEnabled, nil -} - func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { @@ -541,7 +484,7 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe if err != nil { return fmt.Errorf("failed to get kubevirt client: %w", err) } - if _, ok := supportedOS[pc.OperatingSystem]; !ok { + if _, ok := kubevirttypes.SupportedOS[pc.OperatingSystem]; !ok { return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, providerconfigtypes.ErrOSNotSupported) } if c.DNSPolicy == corev1.DNSNone { @@ -555,9 +498,6 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return fmt.Errorf("failed to request VirtualMachineInstances: %w", err) } - if c.OSImageSource.PVC != nil { - return validateOsImage(ctx, c, sigClient) - } return nil } @@ -975,49 +915,3 @@ func getTopologySpreadConstraints(config *Config, matchLabels map[string]string) }, } } - -// validateOsImage with PVC as source. -func validateOsImage(ctx context.Context, c *Config, sigClient client.Client) error { - switch c.OSImageSource.PVC.Namespace { - case c.Namespace: - if !c.AllowCustomImages { - return errCustomImage - } - - case kubeVirtImagesNamespace: - existingDiskList := cdiv1beta1.DataVolumeList{} - listOption := client.ListOptions{ - Namespace: kubeVirtImagesNamespace, - } - if err := sigClient.List(ctx, &existingDiskList, &listOption); client.IgnoreNotFound(err) != nil { - return fmt.Errorf("failed to request DataVolumeList: %w", err) - } - return validateKubeVirtImages(c.OSImageSource.PVC.Name, existingDiskList, c) - - default: - return errInvalidOsImage - } - return nil -} - -// validateKubeVirtImages from kubeVirtImagesNamespace. -func validateKubeVirtImages(sourcePVC string, existingDiskList cdiv1beta1.DataVolumeList, config *Config) error { - for _, existingDV := range existingDiskList.Items { - if sourcePVC == existingDV.Name { - if existingDV.Annotations[dataVolumeStandardImageAnnotation] == "true" { - if !config.AllowPVCClone { - return errStandardImage - } - return nil - } - if existingDV.Annotations[osAnnotationForCustomDisk] != "" { - if !config.AllowCustomImages { - return errCustomImage - } - return nil - } - break - } - } - return errInvalidOsImage -} diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index ec2721df0..cdab28d29 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -20,17 +20,14 @@ import ( "bytes" "context" "embed" - "errors" "html/template" "path" "reflect" "testing" - kubevirtv1 "kubevirt.io/api/core/v1" - cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" - cloudprovidertesting "github.com/kubermatic/machine-controller/pkg/cloudprovider/testing" "github.com/kubermatic/machine-controller/pkg/providerconfig" + kubevirtv1 "kubevirt.io/api/core/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -38,7 +35,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/diff" - "k8s.io/client-go/kubernetes/scheme" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -323,123 +319,3 @@ func TestTopologySpreadConstraint(t *testing.T) { }) } } - -func TestValidateOsImage(t *testing.T) { - testClient := fakectrlruntimeclient. - NewClientBuilder(). - WithScheme(scheme.Scheme). - WithObjects(&cdiv1beta1.DataVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: "standardDV", - Namespace: kubeVirtImagesNamespace, - Annotations: map[string]string{dataVolumeStandardImageAnnotation: "true"}}, - }, - &cdiv1beta1.DataVolume{ - ObjectMeta: metav1.ObjectMeta{ - Name: "customDVByAdmin", - Namespace: kubeVirtImagesNamespace, - Annotations: map[string]string{osAnnotationForCustomDisk: "ubuntu"}}, - }, - ).Build() - - tests := []struct { - desc string - config Config - expectedErr error - }{ - { - desc: "valid osImage with cloned standard DataVolume as pvc source, cloning enabled", - config: Config{ - OSImageSource: &cdiv1beta1.DataVolumeSource{ - PVC: &cdiv1beta1.DataVolumeSourcePVC{ - Name: "standardDV", - Namespace: kubeVirtImagesNamespace, - }, - }, - AllowPVCClone: true, - }, - expectedErr: nil, - }, - { - desc: "valid osImage with cloned standard DataVolume as pvc source, cloning disabled", - config: Config{ - OSImageSource: &cdiv1beta1.DataVolumeSource{ - PVC: &cdiv1beta1.DataVolumeSourcePVC{ - Name: "standardDV", - Namespace: kubeVirtImagesNamespace, - }, - }, - AllowPVCClone: false, - }, - expectedErr: errStandardImage, - }, - { - desc: "valid osImage with custom-image-by-admin as pvc source, custom-images enabled", - config: Config{ - OSImageSource: &cdiv1beta1.DataVolumeSource{ - PVC: &cdiv1beta1.DataVolumeSourcePVC{ - Name: "customDVByAdmin", - Namespace: kubeVirtImagesNamespace, - }, - }, - AllowCustomImages: true, - }, - expectedErr: nil, - }, - { - desc: "valid osImage with custom-image-by-admin as pvc source, custom-images disabled", - config: Config{ - OSImageSource: &cdiv1beta1.DataVolumeSource{ - PVC: &cdiv1beta1.DataVolumeSourcePVC{ - Name: "customDVByAdmin", - Namespace: kubeVirtImagesNamespace, - }, - }, - AllowCustomImages: false, - }, - expectedErr: errCustomImage, - }, - { - desc: "valid osImage with custom-image-by-user as pvc source, custom-images disabled", - config: Config{ - Namespace: "cluster-test", - OSImageSource: &cdiv1beta1.DataVolumeSource{ - PVC: &cdiv1beta1.DataVolumeSourcePVC{ - Name: "customDVByUser", - Namespace: "cluster-test", - }, - }, - AllowCustomImages: false, - }, - expectedErr: errCustomImage, - }, - { - desc: "invalid osImage with non-existent pvc source, cloning enabled", - config: Config{ - OSImageSource: &cdiv1beta1.DataVolumeSource{ - PVC: &cdiv1beta1.DataVolumeSourcePVC{ - Name: "non-existent-DV", - Namespace: kubeVirtImagesNamespace, - }, - }, - AllowPVCClone: true, - }, - expectedErr: errInvalidOsImage, - }, - } - - for _, test := range tests { - t.Run(test.desc, func(t *testing.T) { - actualErr := validateOsImage(context.Background(), &test.config, testClient) - if test.expectedErr != nil { - if !errors.Is(actualErr, test.expectedErr) { - t.Errorf("expected error: %q, got: %q", test.expectedErr, actualErr) - } - } else { - if actualErr != nil { - t.Errorf("expected success, but got error: %q", actualErr) - } - } - }) - } -} diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index 427c4eae8..53f8e9dd8 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -25,6 +25,14 @@ import ( corev1 "k8s.io/api/core/v1" ) +var SupportedOS = map[providerconfigtypes.OperatingSystem]*struct{}{ + providerconfigtypes.OperatingSystemCentOS: nil, + providerconfigtypes.OperatingSystemUbuntu: nil, + providerconfigtypes.OperatingSystemRHEL: nil, + providerconfigtypes.OperatingSystemFlatcar: nil, + providerconfigtypes.OperatingSystemRockyLinux: nil, +} + type RawConfig struct { ClusterName providerconfigtypes.ConfigVarString `json:"clusterName"` Auth Auth `json:"auth,omitempty"` From 6133d147969f284f07d9f95ad99820496b2f285f Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Thu, 19 Jan 2023 15:42:46 +0100 Subject: [PATCH 273/489] Enable rockylinux and flatcar tests in vsphere (#1533) * enable rockylinux and flatcar tests in vsphere Signed-off-by: Moath Qasim * fix tests Signed-off-by: Moath Qasim Signed-off-by: Moath Qasim --- test/e2e/provisioning/all_e2e_test.go | 2 +- test/e2e/provisioning/helper.go | 4 ++-- .../testdata/machinedeployment-vsphere-resource-pool.yaml | 2 +- .../testdata/machinedeployment-vsphere-static-ip.yaml | 2 +- test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index d52ffc30e..0f0548f57 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -854,7 +854,7 @@ func getVSphereTestParams(t *testing.T) []string { func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() - selector := Not(OsSelector("amzn2", "rockylinux", "flatcar")) + selector := Not(OsSelector("amzn2", "centos")) params := getVSphereTestParams(t) runScenarios(t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index aec9d26dc..1f5d32ca3 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -57,9 +57,9 @@ var ( vSphereOSImageTemplates = map[string]string{ string(providerconfigtypes.OperatingSystemCentOS): "kkp-centos-7", - string(providerconfigtypes.OperatingSystemFlatcar): "kkp-flatcar-3033.2.2", + string(providerconfigtypes.OperatingSystemFlatcar): "kkp-flatcar-3139.2.0", string(providerconfigtypes.OperatingSystemRHEL): "kkp-rhel-8.6", - string(providerconfigtypes.OperatingSystemRockyLinux): "kkp-rockylinux-8", + string(providerconfigtypes.OperatingSystemRockyLinux): "kkp-rockylinux-8.5", string(providerconfigtypes.OperatingSystemUbuntu): "kkp-ubuntu-22.04", } diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml index e58087ecb..ca48b60fb 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml @@ -32,7 +32,7 @@ spec: datacenter: 'Hamburg' folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> - datastoreCluster: 'dsc-1' + datastore: 'vsan' resourcePool: 'e2e-resource-pool' cpus: 2 MemoryMB: 2048 diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml index 56ef49c9c..2114d717a 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml @@ -33,7 +33,7 @@ spec: folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically - datastore: ceph-vm + datastore: vsan cpus: 2 MemoryMB: 2048 allowInsecure: true diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml index a7906ed6b..f81548e04 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml @@ -33,7 +33,7 @@ spec: folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically - datastore: ceph-vm + datastore: vsan cpus: 2 MemoryMB: 4096 diskSizeGB: << DISK_SIZE >> From 9f3a1cfd04bbf56d31047173513f7096c6d1b812 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Fri, 20 Jan 2023 12:51:43 +0100 Subject: [PATCH 274/489] Update kubernetes-cni to v1.2.0 (#1534) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić Signed-off-by: Marko Mudrinić --- pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml | 2 +- .../amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml | 2 +- pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml | 2 +- .../centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml | 2 +- pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml | 2 +- pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml | 2 +- pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml | 2 +- pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml | 2 +- pkg/userdata/flatcar/testdata/containerd.yaml | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.24.0.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.24.9.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.25.0.json | 2 +- pkg/userdata/helper/download_binaries_script.go | 2 +- .../helper/testdata/safe_download_binaries_v1.24.9.golden | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws-external.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml | 2 +- pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml | 2 +- pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml | 2 +- pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml | 2 +- .../rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml | 2 +- pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml | 2 +- pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml | 2 +- .../rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml | 2 +- .../rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml | 2 +- pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml | 2 +- pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml | 2 +- pkg/userdata/ubuntu/testdata/containerd.yaml | 2 +- .../ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml | 2 +- pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml | 2 +- pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml | 2 +- pkg/userdata/ubuntu/testdata/docker.yaml | 2 +- .../ubuntu/testdata/kubelet-version-without-v-prefix.yaml | 2 +- pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml | 2 +- pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml | 2 +- pkg/userdata/ubuntu/testdata/nutanix.yaml | 2 +- pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml | 2 +- pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml | 2 +- .../ubuntu/testdata/openstack-overwrite-cloud-config.yaml | 2 +- pkg/userdata/ubuntu/testdata/openstack.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.24.9.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.25.5.yaml | 2 +- pkg/userdata/ubuntu/testdata/version-1.26.0.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere.yaml | 2 +- 62 files changed, 62 insertions(+), 62 deletions(-) diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml index c9a36f0e6..1aa43a240 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml @@ -121,7 +121,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml index 0cf0553e9..1a31ddc32 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml @@ -121,7 +121,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml index 09439f692..a7d129c4e 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml @@ -121,7 +121,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml index e0e6dbf89..e44f88cb7 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml @@ -134,7 +134,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml index 02a4e79ac..551002d1f 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml @@ -134,7 +134,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml index 312f8574e..f7d97347c 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml @@ -126,7 +126,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml index b539a12d3..2fbea37b7 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml @@ -121,7 +121,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml index b79e38f15..d7db6076f 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml @@ -121,7 +121,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml index f00d58d64..fa3f4de4d 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml @@ -127,7 +127,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml index 6ed61116e..8ae7d46e0 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml @@ -127,7 +127,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml index d3e6a33d7..1c27654f5 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml @@ -134,7 +134,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml index 2677cd283..1dcd6b3cf 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml @@ -140,7 +140,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml index 301ca683a..2f878f2b5 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml @@ -140,7 +140,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml index 661120611..e84f8a362 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml @@ -132,7 +132,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml index b469a261e..49aa5c4af 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml @@ -127,7 +127,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml index 427a5499f..6b06d692f 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml @@ -127,7 +127,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml index b180552cc..b1a8b411b 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml @@ -405,7 +405,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml index 9f983a4d3..0cde329b5 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml @@ -405,7 +405,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml index 749a4b1a6..d90ecdbc6 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml @@ -405,7 +405,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index 7c2141a12..cc14eb2a4 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -388,7 +388,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json index 87c6f3273..43a58a9cd 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.1.1%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json b/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json index 9306d10a9..639d4301f 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.1.1%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.9%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.9%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json index 5db2cd855..80ecd0304 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.1.1%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.25.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.25.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/download_binaries_script.go b/pkg/userdata/helper/download_binaries_script.go index 960ab3701..ef9dc601e 100644 --- a/pkg/userdata/helper/download_binaries_script.go +++ b/pkg/userdata/helper/download_binaries_script.go @@ -164,7 +164,7 @@ func SafeDownloadBinariesScript(kubeVersion string) (string, error) { } const ( - CNIVersion = "v1.1.1" + CNIVersion = "v1.2.0" CRIToolsVersion = "v1.22.0" ) diff --git a/pkg/userdata/helper/testdata/safe_download_binaries_v1.24.9.golden b/pkg/userdata/helper/testdata/safe_download_binaries_v1.24.9.golden index e39f4ae81..51ecb15d6 100644 --- a/pkg/userdata/helper/testdata/safe_download_binaries_v1.24.9.golden +++ b/pkg/userdata/helper/testdata/safe_download_binaries_v1.24.9.golden @@ -18,7 +18,7 @@ aarch64) ;; esac fi -CNI_VERSION="${CNI_VERSION:-v1.1.1}" +CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml index 5578055c9..ada462d8e 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml @@ -121,7 +121,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml index 1ba4b8c03..d732c7bf2 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml @@ -121,7 +121,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws-external.yaml index 604221f75..09c9cfddb 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws-external.yaml @@ -121,7 +121,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml index dc9194d95..66864183c 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml @@ -121,7 +121,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml index 05f6b3dda..fd3994457 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml @@ -135,7 +135,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml index 443b8783a..0c5a82994 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml @@ -135,7 +135,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml index 57453bf74..fbf5b83ee 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml @@ -127,7 +127,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml index f2a4f536f..e31f7838f 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml @@ -121,7 +121,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml index b84400a02..c8c131c5f 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml @@ -129,7 +129,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index ea47a5d56..dfc797b89 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -126,7 +126,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml index 00bff81a6..d7d7bc45f 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml @@ -122,7 +122,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml index 2d337459f..3732ebeb2 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml @@ -122,7 +122,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml index 16dfda265..134daaee7 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml @@ -122,7 +122,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml index 28f11187b..63a3d6d31 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml @@ -129,7 +129,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml index 659978804..47e09127a 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml @@ -135,7 +135,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml index 05049a2af..cbb1bf80e 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml @@ -135,7 +135,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml index 8ede88826..005f90d82 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml @@ -127,7 +127,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml index 7ce74fb6e..f42e711c1 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml @@ -122,7 +122,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index 6bbf3cee9..0b2f3c44b 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -130,7 +130,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml index f8d79190e..1f7321e1b 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml index f4b8ee466..ea5f28862 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index ebe109bc4..838055130 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -130,7 +130,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml index 25075ff9e..8a7ff5710 100644 --- a/pkg/userdata/ubuntu/testdata/docker.yaml +++ b/pkg/userdata/ubuntu/testdata/docker.yaml @@ -130,7 +130,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index d774439cf..e947b9369 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index c001dfa28..6c1a60d71 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index d83f08457..0653a5362 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -130,7 +130,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 19ef82285..45b98f395 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -131,7 +131,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml index bbd51553a..ed01caa76 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml index 11b46b4b9..3deb6e2cb 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index f5199c82a..b2d7b4f00 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 9c4179d64..66d087fa5 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml b/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml index 90fdd8704..5c0a62622 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml index 0c4aa9c4e..cae4bb7b1 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml index 289251fd6..3fb38e293 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml @@ -128,7 +128,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index 9df055628..b254228e2 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -138,7 +138,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 38127e887..da13baabc 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -138,7 +138,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 6f404b88d..b741cc182 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -129,7 +129,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v1.1.1}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" From 7285f2cb3b670b5008fd4bf1d22069f7730970bd Mon Sep 17 00:00:00 2001 From: Mario Reggiori Date: Tue, 24 Jan 2023 08:01:47 +0100 Subject: [PATCH 275/489] Anexia Provider: Return TerminalError on 401 and 403 engine responses (#1537) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mario Schäfer Signed-off-by: Mario Schäfer --- pkg/cloudprovider/provider/anexia/provider.go | 28 +++++- .../provider/anexia/provider_test.go | 92 +++++++++++++++++++ 2 files changed, 117 insertions(+), 3 deletions(-) diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index b89763308..1d7a8180b 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -27,6 +27,8 @@ import ( "sync" "time" + "go.anx.io/go-anxcloud/pkg/api" + "go.anx.io/go-anxcloud/pkg/client" anxclient "go.anx.io/go-anxcloud/pkg/client" anxaddr "go.anx.io/go-anxcloud/pkg/ipam/address" "go.anx.io/go-anxcloud/pkg/vsphere" @@ -119,7 +121,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, // provision machine err = provisionVM(ctx, client) if err != nil { - return nil, err + return nil, anexiaErrorToTerminalError(err, "failed waiting for vm provisioning") } return p.Get(ctx, machine, data) } @@ -431,7 +433,7 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, pd if status.InstanceID == "" { progress, err := vsphereAPI.Provisioning().Progress().Get(ctx, status.ProvisioningID) if err != nil { - return nil, fmt.Errorf("failed to get provisioning progress: %w", err) + return nil, anexiaErrorToTerminalError(err, "failed to get provisioning progress") } if len(progress.Errors) > 0 { return nil, fmt.Errorf("vm provisioning had errors: %s", strings.Join(progress.Errors, ",")) @@ -458,7 +460,7 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, pd info, err := vsphereAPI.Info().Get(timeoutCtx, status.InstanceID) if err != nil { - return nil, fmt.Errorf("failed get machine info: %w", err) + return nil, anexiaErrorToTerminalError(err, "failed getting machine info") } instance.info = &info @@ -587,3 +589,23 @@ func updateMachineStatus(machine *clusterv1alpha1.Machine, status anxtypes.Provi return nil } + +func anexiaErrorToTerminalError(err error, msg string) error { + var httpError api.HTTPError + if errors.As(err, &httpError) && (httpError.StatusCode() == http.StatusForbidden || httpError.StatusCode() == http.StatusUnauthorized) { + return cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: "Request was rejected due to invalid credentials", + } + } + + var responseError *client.ResponseError + if errors.As(err, &responseError) && (responseError.ErrorData.Code == http.StatusForbidden || responseError.ErrorData.Code == http.StatusUnauthorized) { + return cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: "Request was rejected due to invalid credentials", + } + } + + return fmt.Errorf("%s: %w", msg, err) +} diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index 08cec3257..5daa367c6 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -21,17 +21,24 @@ import ( "encoding/json" "errors" "net/http" + "net/http/httptest" + "net/url" "testing" "time" "github.com/gophercloud/gophercloud/testhelper" + "go.anx.io/go-anxcloud/pkg/api" + corev1 "go.anx.io/go-anxcloud/pkg/apis/core/v1" + "go.anx.io/go-anxcloud/pkg/client" anxclient "go.anx.io/go-anxcloud/pkg/client" + "go.anx.io/go-anxcloud/pkg/core" "go.anx.io/go-anxcloud/pkg/ipam/address" "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/progress" "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" @@ -346,3 +353,88 @@ func TestUpdateStatus(t *testing.T) { testhelper.AssertEquals(t, true, called) testhelper.AssertNoErr(t, err) } + +func Test_anexiaErrorToTerminalError(t *testing.T) { + forbiddenMockHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, err := w.Write([]byte(`{"error": {"code": 403}}`)) + testhelper.AssertNoErr(t, err) + }) + + unauthorizedMockHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, err := w.Write([]byte(`{"error": {"code": 401}}`)) + testhelper.AssertNoErr(t, err) + }) + + legacyClientRun := func(url string) error { + client, err := client.New(client.BaseURL(url), client.IgnoreMissingToken(), client.ParseEngineErrors(true)) + testhelper.AssertNoErr(t, err) + _, err = core.NewAPI(client).Location().List(context.TODO(), 1, 1, "", "") + return err + } + + apiClientRun := func(url string) error { + client, err := api.NewAPI(api.WithClientOptions( + client.BaseURL(url), + client.IgnoreMissingToken(), + )) + testhelper.AssertNoErr(t, err) + return client.Get(context.TODO(), &corev1.Location{Identifier: "foo"}) + } + + testCases := []struct { + name string + mockHandler http.HandlerFunc + run func(url string) error + }{ + { + name: "api client returns forbidden", + mockHandler: forbiddenMockHandler, + run: apiClientRun, + }, + { + name: "api client returns unauthorized", + mockHandler: unauthorizedMockHandler, + run: apiClientRun, + }, + { + name: "legacy client returns forbidden", + mockHandler: forbiddenMockHandler, + run: legacyClientRun, + }, + { + name: "legacy client returns unauthorized", + mockHandler: unauthorizedMockHandler, + run: legacyClientRun, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + srv := httptest.NewServer(testCase.mockHandler) + defer srv.Close() + + err := anexiaErrorToTerminalError(testCase.run(srv.URL), "foo") + if ok, _, _ := cloudprovidererrors.IsTerminalError(err); !ok { + t.Errorf("unexpected error %#v, expected TerminalError", err) + } + }) + } + + t.Run("api client 404 HTTPError shouldn't convert to TerminalError", func(t *testing.T) { + err := api.NewHTTPError(http.StatusNotFound, "GET", &url.URL{}, errors.New("foo")) + err = anexiaErrorToTerminalError(err, "foo") + if ok, _, _ := cloudprovidererrors.IsTerminalError(err); ok { + t.Errorf("unexpected error %#v, expected no TerminalError", err) + } + }) + + t.Run("legacy api client unspecific ResponseError shouldn't convert to TerminalError", func(t *testing.T) { + var err error = &client.ResponseError{} + err = anexiaErrorToTerminalError(err, "foo") + if ok, _, _ := cloudprovidererrors.IsTerminalError(err); ok { + t.Errorf("unexpected error %#v, expected no TerminalError", err) + } + }) +} From 50c8837f19c03d64a0cf760e25bdc0ca9e6e68c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Tue, 24 Jan 2023 14:29:06 +0100 Subject: [PATCH 276/489] Update cri-tools to v1.26.0 (#1535) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update cri-tools to v1.26.0 Signed-off-by: Marko Mudrinić * Fix checksum checks for cri-tools Signed-off-by: Marko Mudrinić * Update OSM to the latest revision Signed-off-by: Marko Mudrinić Signed-off-by: Marko Mudrinić --- examples/operating-system-manager.yaml | 4 ++-- pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml | 5 +++-- .../amzn2/testdata/kubelet-v1.24.9-aws-external.yaml | 5 +++-- pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml | 5 +++-- .../amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml | 5 +++-- .../amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml | 5 +++-- pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml | 5 +++-- pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml | 5 +++-- pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml | 5 +++-- .../centos/testdata/kubelet-v1.24.9-aws-external.yaml | 5 +++-- pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml | 5 +++-- pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml | 5 +++-- .../centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml | 5 +++-- .../centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml | 5 +++-- pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml | 5 +++-- pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml | 5 +++-- pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml | 5 +++-- pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml | 5 +++-- pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml | 5 +++-- pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml | 5 +++-- pkg/userdata/flatcar/testdata/containerd.yaml | 5 +++-- pkg/userdata/flatcar/testdata/ignition_v1.24.0.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.24.9.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.25.0.json | 2 +- pkg/userdata/helper/download_binaries_script.go | 7 ++++--- .../helper/testdata/safe_download_binaries_v1.24.9.golden | 5 +++-- pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml | 5 +++-- pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml | 5 +++-- .../rhel/testdata/kubelet-v1.24.9-aws-external.yaml | 5 +++-- pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml | 5 +++-- .../rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml | 5 +++-- .../rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml | 5 +++-- pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml | 5 +++-- pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml | 5 +++-- pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml | 5 +++-- pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml | 5 +++-- pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml | 5 +++-- .../rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml | 5 +++-- pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml | 5 +++-- .../rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml | 5 +++-- .../testdata/kubelet-v1.24.9-vsphere-mirrors.yaml | 5 +++-- .../rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml | 5 +++-- .../rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml | 5 +++-- pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml | 5 +++-- pkg/userdata/ubuntu/testdata/containerd.yaml | 5 +++-- .../ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml | 5 +++-- pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml | 5 +++-- pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml | 5 +++-- pkg/userdata/ubuntu/testdata/docker.yaml | 5 +++-- .../ubuntu/testdata/kubelet-version-without-v-prefix.yaml | 5 +++-- pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml | 5 +++-- pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml | 5 +++-- pkg/userdata/ubuntu/testdata/nutanix.yaml | 5 +++-- .../ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml | 5 +++-- pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml | 5 +++-- .../ubuntu/testdata/openstack-overwrite-cloud-config.yaml | 5 +++-- pkg/userdata/ubuntu/testdata/openstack.yaml | 5 +++-- pkg/userdata/ubuntu/testdata/version-1.24.9.yaml | 5 +++-- pkg/userdata/ubuntu/testdata/version-1.25.5.yaml | 5 +++-- pkg/userdata/ubuntu/testdata/version-1.26.0.yaml | 5 +++-- pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml | 5 +++-- pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml | 5 +++-- pkg/userdata/ubuntu/testdata/vsphere.yaml | 5 +++-- 63 files changed, 183 insertions(+), 124 deletions(-) diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index 989258c96..dadb9193b 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -975,7 +975,7 @@ spec: serviceAccountName: operating-system-manager-webhook containers: # TODO: Update this to a semver tag before release. - - image: quay.io/kubermatic/operating-system-manager:22c771f4cb686427d94df83946b292d7ff3a06f8 + - image: quay.io/kubermatic/operating-system-manager:8688a1fde001705f4c0b394ed2f4ff4a292b20a9 imagePullPolicy: IfNotPresent name: webhook command: @@ -1304,7 +1304,7 @@ spec: serviceAccountName: operating-system-manager containers: # TODO: Update this to a semver tag before release. - - image: quay.io/kubermatic/operating-system-manager:22c771f4cb686427d94df83946b292d7ff3a06f8 + - image: quay.io/kubermatic/operating-system-manager:8688a1fde001705f4c0b394ed2f4ff4a292b20a9 imagePullPolicy: IfNotPresent name: operating-system-manager command: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml index 1aa43a240..6cbbe4ad6 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml @@ -131,11 +131,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml index 1a31ddc32..0e2a2237b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml @@ -131,11 +131,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml index a7d129c4e..43a9853b2 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml @@ -131,11 +131,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml index e44f88cb7..b90489c90 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml @@ -144,11 +144,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml index 551002d1f..5e0a08c15 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml @@ -144,11 +144,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml index f7d97347c..880f0ec4b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml @@ -136,11 +136,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml index 2fbea37b7..318b1b646 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml @@ -131,11 +131,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml index d7db6076f..4115f30da 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml @@ -131,11 +131,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml index fa3f4de4d..e9998c449 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml @@ -137,11 +137,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml index 8ae7d46e0..a3213cfad 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml @@ -137,11 +137,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml index 1c27654f5..98e8e1f9a 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml @@ -144,11 +144,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml index 1dcd6b3cf..78d8f31a7 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml @@ -150,11 +150,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml index 2f878f2b5..e746b4110 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml @@ -150,11 +150,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml index e84f8a362..e2000bbb3 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml @@ -142,11 +142,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml index 49aa5c4af..be894d2a5 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml @@ -137,11 +137,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml index 6b06d692f..847bd55cf 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml @@ -137,11 +137,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml index b1a8b411b..0537cb53f 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml @@ -415,11 +415,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml index 0cde329b5..89ade7771 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml @@ -415,11 +415,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml index d90ecdbc6..4ee2b0c7b 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml @@ -415,11 +415,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index cc14eb2a4..785e85e8a 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -398,11 +398,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json index 43a58a9cd..48f3d6a2b 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json b/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json index 639d4301f..71a2b9757 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.9%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.9%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json index 80ecd0304..3549aa2a5 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.22.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22%20%7C%20sed%20's%2F%5C*%5C%2F%2F%2F')%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.25.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.25.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/download_binaries_script.go b/pkg/userdata/helper/download_binaries_script.go index ef9dc601e..ab9e248c5 100644 --- a/pkg/userdata/helper/download_binaries_script.go +++ b/pkg/userdata/helper/download_binaries_script.go @@ -79,8 +79,9 @@ cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" {{- /* download cri-tools checksum */}} -{{- /* the cri-tools checksum file has a filename prefix that breaks sha256sum so we need to drop it with sed */}} -cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') +{{- /* the cri-tools checksum file provides only the checksum without the file name, so we need to handle it specially */}} +cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") +cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" {{- /* verify cri-tools checksum */}} @@ -165,7 +166,7 @@ func SafeDownloadBinariesScript(kubeVersion string) (string, error) { const ( CNIVersion = "v1.2.0" - CRIToolsVersion = "v1.22.0" + CRIToolsVersion = "v1.26.0" ) // force v in case if it's not there diff --git a/pkg/userdata/helper/testdata/safe_download_binaries_v1.24.9.golden b/pkg/userdata/helper/testdata/safe_download_binaries_v1.24.9.golden index 51ecb15d6..98f57b9f7 100644 --- a/pkg/userdata/helper/testdata/safe_download_binaries_v1.24.9.golden +++ b/pkg/userdata/helper/testdata/safe_download_binaries_v1.24.9.golden @@ -28,11 +28,12 @@ sha256sum -c <<<"$cni_sum" tar xvf "$cni_filename" rm -f "$cni_filename" cd - -CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" +CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" -cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') +cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") +cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml index ada462d8e..9bdf86c67 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml @@ -131,11 +131,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml index d732c7bf2..d67ddaf1f 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml @@ -131,11 +131,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws-external.yaml index 09c9cfddb..d77002e64 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws-external.yaml @@ -131,11 +131,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml index 66864183c..ae510dc1d 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml @@ -131,11 +131,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml index fd3994457..a4341ba36 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml @@ -145,11 +145,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml index 0c5a82994..9a3a0586d 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml @@ -145,11 +145,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml index fbf5b83ee..854c91ec6 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml @@ -137,11 +137,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml index e31f7838f..599aa75eb 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml @@ -131,11 +131,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml index c8c131c5f..8e0a31352 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml @@ -139,11 +139,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index dfc797b89..f3bc71e64 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -136,11 +136,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml index d7d7bc45f..910ed957c 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml @@ -132,11 +132,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml index 3732ebeb2..ea98aa141 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml @@ -132,11 +132,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml index 134daaee7..eb87f3395 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml @@ -132,11 +132,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml index 63a3d6d31..9cbbf5b50 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml @@ -139,11 +139,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml index 47e09127a..b4452384f 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml @@ -145,11 +145,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml index cbb1bf80e..1070b5fd3 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml @@ -145,11 +145,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml index 005f90d82..64521cbc6 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml @@ -137,11 +137,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml index f42e711c1..ee32e7e03 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml @@ -132,11 +132,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index 0b2f3c44b..dd7dd8d91 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -140,11 +140,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml index 1f7321e1b..71c2a3f56 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml @@ -138,11 +138,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml index ea5f28862..e1bf0ec28 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml @@ -138,11 +138,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 838055130..2e56201ab 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -140,11 +140,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml index 8a7ff5710..311ae4731 100644 --- a/pkg/userdata/ubuntu/testdata/docker.yaml +++ b/pkg/userdata/ubuntu/testdata/docker.yaml @@ -140,11 +140,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index e947b9369..99f32fe25 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -138,11 +138,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 6c1a60d71..77949c18c 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -138,11 +138,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 0653a5362..38e8e3432 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -140,11 +140,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 45b98f395..3b453a2b6 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -141,11 +141,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml index ed01caa76..331eea036 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml @@ -138,11 +138,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml index 3deb6e2cb..8cc558edf 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml @@ -138,11 +138,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index b2d7b4f00..b8ada4586 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -138,11 +138,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 66d087fa5..d35ee8ecd 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -138,11 +138,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml b/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml index 5c0a62622..b38c94941 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml @@ -138,11 +138,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml index cae4bb7b1..0c53f84ed 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml @@ -138,11 +138,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml index 3fb38e293..6545373b7 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml @@ -138,11 +138,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index b254228e2..c24a42c5d 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -148,11 +148,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index da13baabc..eee9a28b5 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -148,11 +148,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index b741cc182..c5341ba32 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -139,11 +139,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" From 132ea69c770a7c937eca6471c99a85f26a2db876 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 24 Jan 2023 15:41:47 +0100 Subject: [PATCH 277/489] refactor vsphere tagging mechanism (#1532) Signed-off-by: Moath Qasim Signed-off-by: Moath Qasim --- pkg/cloudprovider/provider/vsphere/helper.go | 30 +++++-------------- .../provider/vsphere/provider.go | 10 +++++-- .../provider/vsphere/types/types.go | 1 + 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go index a11ace772..27b7ccf14 100644 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ b/pkg/cloudprovider/provider/vsphere/helper.go @@ -456,38 +456,24 @@ func resolveResourcePoolRef(ctx context.Context, config *Config, session *Sessio return nil, nil } -func createAndAttachTags(ctx context.Context, config *Config, vm *object.VirtualMachine) error { +func attachTags(ctx context.Context, config *Config, vm *object.VirtualMachine) error { restAPISession, err := NewRESTSession(ctx, config) if err != nil { return fmt.Errorf("failed to create REST API session: %w", err) } defer restAPISession.Logout(ctx) tagManager := tags.NewManager(restAPISession.Client) - klog.V(3).Info("Creating tags") + klog.V(3).Info("Attaching tags") for _, tag := range config.Tags { - tagID, err := tagManager.CreateTag(ctx, &tag) - if err != nil { - return fmt.Errorf("failed to create tag: %v %w", tag, err) - } - - if err := tagManager.AttachTag(ctx, tagID, vm.Reference()); err != nil { - // If attaching the tag to VM failed then delete this tag. It prevents orphan tags. - if errDelete := tagManager.DeleteTag(ctx, &tags.Tag{ - ID: tagID, - Description: tag.Description, - Name: tag.Name, - CategoryID: tag.CategoryID, - }); errDelete != nil { - return fmt.Errorf("failed to attach tag to VM and delete the orphan tag: %v, attach error: %v, delete error: %w", tag, err, errDelete) - } + if err := tagManager.AttachTag(ctx, tag.ID, vm.Reference()); err != nil { klog.V(3).Infof("Failed to attach tag %v. The tag was successfully deleted", tag) - return fmt.Errorf("failed to attach tag to VM: %v %w", tag, err) + return fmt.Errorf("failed to attach tag to VM: %v %w", tag.Name, err) } } return nil } -func deleteTags(ctx context.Context, config *Config, vm *object.VirtualMachine) error { +func detachTags(ctx context.Context, config *Config, vm *object.VirtualMachine) error { restAPISession, err := NewRESTSession(ctx, config) if err != nil { return fmt.Errorf("failed to create REST API session: %w", err) @@ -495,13 +481,13 @@ func deleteTags(ctx context.Context, config *Config, vm *object.VirtualMachine) defer restAPISession.Logout(ctx) tagManager := tags.NewManager(restAPISession.Client) - tags, err := tagManager.GetAttachedTags(ctx, vm.Reference()) + attachedTags, err := tagManager.GetAttachedTags(ctx, vm.Reference()) if err != nil { return fmt.Errorf("failed to get attached tags for the VM: %s, %w", vm.Name(), err) } klog.V(3).Info("Deleting tags") - for _, tag := range tags { - err := tagManager.DeleteTag(ctx, &tag) + for _, tag := range attachedTags { + err := tagManager.DetachTag(ctx, tag.ID, vm.Reference()) if err != nil { return fmt.Errorf("failed to delete tag: %v %w", tag, err) } diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 3b4395294..a7e47acae 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -196,6 +196,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p for _, tag := range rawConfig.Tags { c.Tags = append(c.Tags, tags.Tag{ Description: tag.Description, + ID: tag.ID, Name: tag.Name, CategoryID: tag.CategoryID, }) @@ -225,6 +226,9 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe tagManager := tags.NewManager(restAPISession.Client) klog.V(3).Info("Found tags") for _, tag := range config.Tags { + if tag.ID == "" { + return fmt.Errorf("one of the tags id is empty") + } if tag.Name == "" { return fmt.Errorf("one of the tags name is empty") } @@ -334,8 +338,8 @@ func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, machineInvalidConfigurationTerminalError(fmt.Errorf("failed to create cloned vm: '%w'", err)) } - if err := createAndAttachTags(ctx, config, virtualMachine); err != nil { - return nil, fmt.Errorf("failed create and attach tags: %w", err) + if err := attachTags(ctx, config, virtualMachine); err != nil { + return nil, fmt.Errorf("failed to attach tags: %w", err) } if pc.OperatingSystem != providerconfigtypes.OperatingSystemFlatcar { @@ -396,7 +400,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine return false, fmt.Errorf("failed to get instance from vSphere: %w", err) } - if err := deleteTags(ctx, config, virtualMachine); err != nil { + if err := detachTags(ctx, config, virtualMachine); err != nil { return false, fmt.Errorf("failed to delete tags: %w", err) } diff --git a/pkg/cloudprovider/provider/vsphere/types/types.go b/pkg/cloudprovider/provider/vsphere/types/types.go index 835af02e7..b584cb37d 100644 --- a/pkg/cloudprovider/provider/vsphere/types/types.go +++ b/pkg/cloudprovider/provider/vsphere/types/types.go @@ -50,6 +50,7 @@ type RawConfig struct { // Tag represents vsphere tag. type Tag struct { Description string `json:"description,omitempty"` + ID string `json:"id"` Name string `json:"name"` CategoryID string `json:"categoryID"` } From 8fa3ee3577220de31f85ae11424b6d97afcf0109 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 30 Jan 2023 19:53:17 +0500 Subject: [PATCH 278/489] Remove sig-osm from userdata and cloudprovider ownership (#1543) Signed-off-by: Waleed Malik --- OWNERS_ALIASES | 6 ------ pkg/cloudprovider/provider/OWNERS | 13 ------------- pkg/userdata/OWNERS | 13 ------------- 3 files changed, 32 deletions(-) delete mode 100644 pkg/cloudprovider/provider/OWNERS delete mode 100644 pkg/userdata/OWNERS diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index d833f9f6a..befce2745 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -19,9 +19,3 @@ aliases: - hdurand0710 - mfranczy - sankalp-r - - # Temporary SIG to oversee changes in userdata and cloudprovider sub-directories - # This SIG is responsible for ensuring that OSM and machine-controller are in sync - sig-osm: - - ahmedwaleedmalik - - moadqassem diff --git a/pkg/cloudprovider/provider/OWNERS b/pkg/cloudprovider/provider/OWNERS deleted file mode 100644 index 31bc1a729..000000000 --- a/pkg/cloudprovider/provider/OWNERS +++ /dev/null @@ -1,13 +0,0 @@ -# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md - -approvers: - - sig-osm - -reviewers: - - sig-osm - -labels: - - sig/osm - -options: - no_parent_owners: true \ No newline at end of file diff --git a/pkg/userdata/OWNERS b/pkg/userdata/OWNERS deleted file mode 100644 index 31bc1a729..000000000 --- a/pkg/userdata/OWNERS +++ /dev/null @@ -1,13 +0,0 @@ -# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md - -approvers: - - sig-osm - -reviewers: - - sig-osm - -labels: - - sig/osm - -options: - no_parent_owners: true \ No newline at end of file From bb7be77b77d1730f2097edbc96c621968aa453a2 Mon Sep 17 00:00:00 2001 From: Mario Reggiori Date: Tue, 31 Jan 2023 16:54:21 +0100 Subject: [PATCH 279/489] Anexia Provider: add support for named VM templates (#1541) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add support for named templates in anexia provider Signed-off-by: Mario Schäfer * Anexia: update docs for template configuration Signed-off-by: Mara Sophie Grosch * Resolve templateID from template in `resolveConfig()` Signed-off-by: Mario Schäfer --------- Signed-off-by: Mario Schäfer Signed-off-by: Mara Sophie Grosch Co-authored-by: Mara Sophie Grosch --- docs/anexia.md | 10 +- examples/anexia-machinedeployment.yaml | 6 +- go.mod | 2 + go.sum | 4 +- pkg/cloudprovider/provider/anexia/provider.go | 77 +++++++++--- .../provider/anexia/provider_test.go | 116 ++++++++++++++---- .../provider/anexia/types/types.go | 5 +- 7 files changed, 173 insertions(+), 47 deletions(-) diff --git a/docs/anexia.md b/docs/anexia.md index 5a224f532..cf826f8fa 100644 --- a/docs/anexia.md +++ b/docs/anexia.md @@ -10,10 +10,16 @@ An example machine deployment can be found here: [examples/anexia-machinedeploym ## Templates +You can configure the template to use by its name (using the attribute `template`) or its identifier (using the attribute `templateID`). + +When specifying the template by its name, the template build to use can optionally be set (attribute `templateBuild`). Omitting `templateBuild` will yield the latest available build (at time the time of creating the `Machine`) for the specified named template. + +Template identifiers (attribute `templateID`) always link to a given `template`-`templateBuild` combination, so using the identifier in configuration has the same drawback as specifying an exact build to use. + +Templates are rotated pretty often to include security patches and other updates. Outdated versions of templates are not retained and get removed after some time. Because of this, we do not recommend using the `templateID` attribute or pinning to a fixed build unless really required. + To retrieve all available templates against a given location: ``` https://engine.anexia-it.com/api/vsphere/v1/provisioning/templates.json//templates?page=1&limit=50&api_key= ``` - -Templates are rotated pretty often, to include updates and latest security patches. Outdated versions of templates are not retained as a result and they get removed after some time. diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index 743bddcaa..efb1564b7 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -31,8 +31,10 @@ spec: name: machine-controller-anexia key: token vlanID: "<< ANEXIA_VLAN_ID >>" - # Currently only the flatcar template is supported: 12c28aa7-604d-47e9-83fb-5f1d1f1837b3 - templateID: "<< ANEXIA_TEMPLATE_ID >>" + # Currently only the "Flatcar Linux Stable" template is supported. + # Use templateBuild to specify a build. If empty => latest + # Alternatively use templateID for a specific template. + template: "<< ANEXIA_TEMPLATE_NAME >>" locationID: "<< ANEXIA_LOCATION_ID >>" cpus: 2 memory: 2048 diff --git a/go.mod b/go.mod index cdfef6426..3a754384d 100644 --- a/go.mod +++ b/go.mod @@ -135,6 +135,8 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/onsi/ginkgo/v2 v2.6.0 // indirect + github.com/onsi/gomega v1.24.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect diff --git a/go.sum b/go.sum index cd5b4ee4b..2eb996499 100644 --- a/go.sum +++ b/go.sum @@ -529,7 +529,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= @@ -538,6 +537,7 @@ github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7 github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc= +github.com/onsi/ginkgo/v2 v2.6.0/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= @@ -549,6 +549,7 @@ github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8lu github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 h1:+ShYlGoPriGahTTFTjQ0RtNXW0srxDodk2STdc238Rk= @@ -615,6 +616,7 @@ github.com/rollbar/rollbar-go/errors v0.0.0-20210929193720-32947096267e/go.mod h github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= +github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10 h1:wsfMs0iv+MJiViM37qh5VEKISi3/ZUq2nNKNdqmumAs= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 1d7a8180b..b036d9149 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -28,6 +28,8 @@ import ( "time" "go.anx.io/go-anxcloud/pkg/api" + corev1 "go.anx.io/go-anxcloud/pkg/apis/core/v1" + vspherev1 "go.anx.io/go-anxcloud/pkg/apis/vsphere/v1" "go.anx.io/go-anxcloud/pkg/client" anxclient "go.anx.io/go-anxcloud/pkg/client" anxaddr "go.anx.io/go-anxcloud/pkg/ipam/address" @@ -94,7 +96,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, // ensure conditions are present on machine ensureConditions(&status) - config, _, err := p.getConfig(machine.Spec.ProviderSpec) + config, _, err := p.getConfig(ctx, machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("unable to get provider config: %w", err) } @@ -107,7 +109,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, Machine: machine, }) - client, err := getClient(config.Token) + _, client, err := getClient(config.Token) if err != nil { return nil, err } @@ -270,7 +272,26 @@ func ensureConditions(status *anxtypes.ProviderStatus) { } } -func (p *provider) resolveConfig(config anxtypes.RawConfig) (*resolvedConfig, error) { +func resolveTemplateID(ctx context.Context, a api.API, config anxtypes.RawConfig, configVarResolver *providerconfig.ConfigVarResolver, locationID string) (string, error) { + templateName, err := configVarResolver.GetConfigVarStringValue(config.Template) + if err != nil { + return "", fmt.Errorf("failed to get 'template': %w", err) + } + + templateBuild, err := configVarResolver.GetConfigVarStringValue(config.TemplateBuild) + if err != nil { + return "", fmt.Errorf("failed to get 'templateBuild': %w", err) + } + + template, err := vspherev1.FindNamedTemplate(ctx, a, templateName, templateBuild, corev1.Location{Identifier: locationID}) + if err != nil { + return "", fmt.Errorf("failed to retrieve named template: %w", err) + } + + return template.Identifier, nil +} + +func (p *provider) resolveConfig(ctx context.Context, config anxtypes.RawConfig) (*resolvedConfig, error) { var err error ret := resolvedConfig{ RawConfig: config, @@ -291,6 +312,21 @@ func (p *provider) resolveConfig(config anxtypes.RawConfig) (*resolvedConfig, er return nil, fmt.Errorf("failed to get 'templateID': %w", err) } + // when "templateID" is not set, we expect "template" to be + if ret.TemplateID == "" { + a, _, err := getClient(ret.Token) + if err != nil { + return nil, fmt.Errorf("failed initializing API clients: %w", err) + } + + templateID, err := resolveTemplateID(ctx, a, config, p.configVarResolver, ret.LocationID) + if err != nil { + return nil, fmt.Errorf("failed retrieving template id from named template: %w", err) + } + + ret.TemplateID = templateID + } + ret.VlanID, err = p.configVarResolver.GetConfigVarStringValue(config.VlanID) if err != nil { return nil, fmt.Errorf("failed to get 'vlanID': %w", err) @@ -325,7 +361,7 @@ func (p *provider) resolveConfig(config anxtypes.RawConfig) (*resolvedConfig, er return &ret, nil } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*resolvedConfig, *providerconfigtypes.Config, error) { +func (p *provider) getConfig(ctx context.Context, provSpec clusterv1alpha1.ProviderSpec) (*resolvedConfig, *providerconfigtypes.Config, error) { if provSpec.Value == nil { return nil, nil, fmt.Errorf("machine.spec.providerSpec.value is nil") } @@ -343,7 +379,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*resolvedCo return nil, nil, fmt.Errorf("error parsing provider config: %w", err) } - resolvedConfig, err := p.resolveConfig(*rawConfig) + resolvedConfig, err := p.resolveConfig(ctx, *rawConfig) if err != nil { return nil, nil, fmt.Errorf("error resolving config: %w", err) } @@ -362,14 +398,14 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha } // Validate returns success or failure based according to its ProviderSpec. -func (p *provider) Validate(_ context.Context, machinespec clusterv1alpha1.MachineSpec) error { - config, _, err := p.getConfig(machinespec.ProviderSpec) +func (p *provider) Validate(ctx context.Context, machinespec clusterv1alpha1.MachineSpec) error { + config, _, err := p.getConfig(ctx, machinespec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) } if config.Token == "" { - return errors.New("token is missing") + return errors.New("token not set") } if config.CPUs == 0 { @@ -399,7 +435,7 @@ func (p *provider) Validate(_ context.Context, machinespec clusterv1alpha1.Machi } if config.TemplateID == "" { - return errors.New("template id is missing") + return errors.New("no valid template configured") } if config.VlanID == "" { @@ -410,12 +446,12 @@ func (p *provider) Validate(_ context.Context, machinespec clusterv1alpha1.Machi } func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, pd *cloudprovidertypes.ProviderData) (instance.Instance, error) { - config, _, err := p.getConfig(machine.Spec.ProviderSpec) + config, _, err := p.getConfig(ctx, machine.Spec.ProviderSpec) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, "failed to retrieve config: %v", err) } - cli, err := getClient(config.Token) + _, cli, err := getClient(config.Token) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, "failed to create Anexia client: %v", err) } @@ -480,12 +516,12 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine }() ensureConditions(&status) - config, _, err := p.getConfig(machine.Spec.ProviderSpec) + config, _, err := p.getConfig(ctx, machine.Spec.ProviderSpec) if err != nil { return false, newError(common.InvalidConfigurationMachineError, "failed to parse MachineSpec: %v", err) } - cli, err := getClient(config.Token) + _, cli, err := getClient(config.Token) if err != nil { return false, newError(common.InvalidConfigurationMachineError, "failed to create Anexia client: %v", err) } @@ -544,10 +580,21 @@ func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } -func getClient(token string) (anxclient.Client, error) { +func getClient(token string) (api.API, anxclient.Client, error) { tokenOpt := anxclient.TokenFromString(token) client := anxclient.HTTPClient(&http.Client{Timeout: 120 * time.Second}) - return anxclient.New(tokenOpt, client) + + a, err := api.NewAPI(api.WithClientOptions(client, tokenOpt)) + if err != nil { + return nil, nil, fmt.Errorf("error creating generic API client: %w", err) + } + + legacyClient, err := anxclient.New(tokenOpt, client) + if err != nil { + return nil, nil, fmt.Errorf("error creating legacy client: %w", err) + } + + return a, legacyClient, nil } func getProviderStatus(machine *clusterv1alpha1.Machine) anxtypes.ProviderStatus { diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index 5daa367c6..4bda59f26 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -23,12 +23,15 @@ import ( "net/http" "net/http/httptest" "net/url" + "strings" "testing" "time" "github.com/gophercloud/gophercloud/testhelper" "go.anx.io/go-anxcloud/pkg/api" + "go.anx.io/go-anxcloud/pkg/api/mock" corev1 "go.anx.io/go-anxcloud/pkg/apis/core/v1" + vspherev1 "go.anx.io/go-anxcloud/pkg/apis/vsphere/v1" "go.anx.io/go-anxcloud/pkg/client" anxclient "go.anx.io/go-anxcloud/pkg/client" "go.anx.io/go-anxcloud/pkg/core" @@ -47,11 +50,20 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) -const TestIdentifier = "TestIdent" +const ( + TestIdentifier = "TestIdent" + testTemplateName = "test-template" +) func TestAnexiaProvider(t *testing.T) { testhelper.SetupHTTP() client, server := anxclient.NewTestClient(nil, testhelper.Mux) + + a := mock.NewMockAPI() + a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID-OLD-BUILD", Name: testTemplateName, Build: "b01"}) + a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID", Name: testTemplateName, Build: "b02"}) + a.FakeExisting(&vspherev1.Template{Identifier: "WRONG-TEMPLATE-NAME", Name: "Wrong Template Name", Build: "b02"}) + t.Cleanup(func() { testhelper.TeardownHTTP() server.Close() @@ -157,6 +169,62 @@ func TestAnexiaProvider(t *testing.T) { testhelper.AssertNoErr(t, err) }) + t.Run("Test resolve template", func(t *testing.T) { + t.Parallel() + + type testCase struct { + config anxtypes.RawConfig + expectedError string + expectedTemplateID string + } + + testCases := []testCase{ + // fail + { + // Template name does not exist + config: hookableConfig(func(c *anxtypes.RawConfig) { c.Template.Value = "non-existing-template-name" }), + expectedError: "failed to retrieve named template", + }, + { + // Template build does not exist + config: hookableConfig(func(c *anxtypes.RawConfig) { + c.Template.Value = testTemplateName + c.TemplateBuild.Value = "b42" + }), + expectedError: "failed to retrieve named template", + }, + // pass + { + // With named template + config: hookableConfig(func(c *anxtypes.RawConfig) { c.Template.Value = testTemplateName; c.TemplateID.Value = "" }), + expectedTemplateID: "TEMPLATE-ID", + }, + { + // With named template and not latest build + config: hookableConfig(func(c *anxtypes.RawConfig) { + c.Template.Value = testTemplateName + c.TemplateBuild.Value = "b01" + }), + expectedTemplateID: "TEMPLATE-ID-OLD-BUILD", + }, + } + + provider := New(nil).(*provider) + for _, testCase := range testCases { + templateID, err := resolveTemplateID(context.TODO(), a, testCase.config, provider.configVarResolver, "foo") + if testCase.expectedError != "" { + if err != nil { + testhelper.AssertErr(t, err) + testhelper.AssertEquals(t, true, strings.Contains(err.Error(), testCase.expectedError)) + continue + } + } else { + testhelper.AssertNoErr(t, err) + testhelper.AssertEquals(t, testCase.expectedTemplateID, templateID) + } + } + }) + t.Run("Test is VM Provisioning", func(t *testing.T) { t.Parallel() providerStatus := anxtypes.ProviderStatus{ @@ -209,38 +277,38 @@ func TestAnexiaProvider(t *testing.T) { }) } -func TestValidate(t *testing.T) { - t.Parallel() +// this generates a full config and allows hooking into it to e.g. remove a value. +func hookableConfig(hook func(*anxtypes.RawConfig)) anxtypes.RawConfig { + config := anxtypes.RawConfig{ + CPUs: 1, - // this generates a full config and allows hooking into it to e.g. remove a value - hookableConfig := func(hook func(*anxtypes.RawConfig)) anxtypes.RawConfig { - config := anxtypes.RawConfig{ - CPUs: 1, + Memory: 2, - Memory: 2, + Disks: []anxtypes.RawDisk{ + {Size: 5, PerformanceType: newConfigVarString("ENT6")}, + }, - Disks: []anxtypes.RawDisk{ - {Size: 5, PerformanceType: newConfigVarString("ENT6")}, - }, + Token: newConfigVarString("test-token"), + VlanID: newConfigVarString("test-vlan"), + LocationID: newConfigVarString("test-location"), + TemplateID: newConfigVarString("test-template-id"), + } - Token: newConfigVarString("test-token"), - VlanID: newConfigVarString("test-vlan"), - LocationID: newConfigVarString("test-location"), - TemplateID: newConfigVarString("test-template"), - } + if hook != nil { + hook(&config) + } - if hook != nil { - hook(&config) - } + return config +} - return config - } +func TestValidate(t *testing.T) { + t.Parallel() var configCases []ConfigTestCase configCases = append(configCases, ConfigTestCase{ Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Token.Value = "" }), - Error: errors.New("token is missing"), + Error: errors.New("token not set"), }, ConfigTestCase{ Config: hookableConfig(func(c *anxtypes.RawConfig) { c.CPUs = 0 }), @@ -270,10 +338,6 @@ func TestValidate(t *testing.T) { Config: hookableConfig(func(c *anxtypes.RawConfig) { c.LocationID.Value = "" }), Error: errors.New("location id is missing"), }, - ConfigTestCase{ - Config: hookableConfig(func(c *anxtypes.RawConfig) { c.TemplateID.Value = "" }), - Error: errors.New("template id is missing"), - }, ConfigTestCase{ Config: hookableConfig(func(c *anxtypes.RawConfig) { c.VlanID.Value = "" }), Error: errors.New("vlan id is missing"), diff --git a/pkg/cloudprovider/provider/anexia/types/types.go b/pkg/cloudprovider/provider/anexia/types/types.go index 25547e9d4..e6d8e9f22 100644 --- a/pkg/cloudprovider/provider/anexia/types/types.go +++ b/pkg/cloudprovider/provider/anexia/types/types.go @@ -57,7 +57,10 @@ type RawConfig struct { Token providerconfigtypes.ConfigVarString `json:"token,omitempty"` VlanID providerconfigtypes.ConfigVarString `json:"vlanID"` LocationID providerconfigtypes.ConfigVarString `json:"locationID"` - TemplateID providerconfigtypes.ConfigVarString `json:"templateID"` + + TemplateID providerconfigtypes.ConfigVarString `json:"templateID"` + Template providerconfigtypes.ConfigVarString `json:"template"` + TemplateBuild providerconfigtypes.ConfigVarString `json:"templateBuild"` CPUs int `json:"cpus"` Memory int `json:"memory"` From ab8cf636f59966c728c4b30488edbfeced403d38 Mon Sep 17 00:00:00 2001 From: Kubermatic Bot <41968677+kubermatic-bot@users.noreply.github.com> Date: Wed, 1 Feb 2023 13:29:03 +0100 Subject: [PATCH 280/489] Synchronize OWNERS_ALIASES file with Github teams (#1548) --- OWNERS_ALIASES | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index befce2745..879077cbf 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -1,4 +1,5 @@ -# See the OWNERS docs: https://github.com/kubernetes/community/blob/master/contributors/guide/owners.md +# This file was automatically generated by prow-aliases-syncer. DO NOT EDIT. +# To change team associations, update the GitHub teams via https://github.com/kubermatic/access. aliases: machine-controller-maintainers: @@ -13,9 +14,6 @@ aliases: - themue - xmudrii - xrstf - sig-virtualization: - - dermorz - hdurand0710 - mfranczy - - sankalp-r From 07702e76ae500b54cdaf542e574fdf74d6a074a3 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 1 Feb 2023 19:15:25 +0500 Subject: [PATCH 281/489] Make ID an optional field for vSphere tags (#1549) Either ID or a combination of Name and Category ID can be used to uniquely determine a tag Signed-off-by: Waleed Malik --- pkg/cloudprovider/provider/vsphere/helper.go | 26 +++++++++++++++++-- .../provider/vsphere/provider.go | 7 ++--- .../provider/vsphere/types/types.go | 4 +-- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go index 27b7ccf14..db08f9dbc 100644 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ b/pkg/cloudprovider/provider/vsphere/helper.go @@ -465,7 +465,12 @@ func attachTags(ctx context.Context, config *Config, vm *object.VirtualMachine) tagManager := tags.NewManager(restAPISession.Client) klog.V(3).Info("Attaching tags") for _, tag := range config.Tags { - if err := tagManager.AttachTag(ctx, tag.ID, vm.Reference()); err != nil { + tagID, err := determineTagID(ctx, tagManager, tag) + if err != nil { + return err + } + + if err := tagManager.AttachTag(ctx, tagID, vm.Reference()); err != nil { klog.V(3).Infof("Failed to attach tag %v. The tag was successfully deleted", tag) return fmt.Errorf("failed to attach tag to VM: %v %w", tag.Name, err) } @@ -487,7 +492,12 @@ func detachTags(ctx context.Context, config *Config, vm *object.VirtualMachine) } klog.V(3).Info("Deleting tags") for _, tag := range attachedTags { - err := tagManager.DetachTag(ctx, tag.ID, vm.Reference()) + tagID, err := determineTagID(ctx, tagManager, tag) + if err != nil { + return err + } + + err = tagManager.DetachTag(ctx, tagID, vm.Reference()) if err != nil { return fmt.Errorf("failed to delete tag: %v %w", tag, err) } @@ -495,3 +505,15 @@ func detachTags(ctx context.Context, config *Config, vm *object.VirtualMachine) return nil } + +func determineTagID(ctx context.Context, tagManager *tags.Manager, tag tags.Tag) (string, error) { + if tag.ID != "" { + return tag.ID, nil + } + + apiTag, err := tagManager.GetTagForCategory(ctx, tag.Name, tag.CategoryID) + if err != nil { + return "", fmt.Errorf("failed to retrieve tag: %v %w", tag.Name, err) + } + return apiTag.ID, nil +} diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index a7e47acae..481a2e519 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -226,11 +226,8 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe tagManager := tags.NewManager(restAPISession.Client) klog.V(3).Info("Found tags") for _, tag := range config.Tags { - if tag.ID == "" { - return fmt.Errorf("one of the tags id is empty") - } - if tag.Name == "" { - return fmt.Errorf("one of the tags name is empty") + if tag.ID == "" && tag.Name == "" { + return fmt.Errorf("either tag id or name must be specified") } if tag.CategoryID == "" { return fmt.Errorf("one of the tags category is empty") diff --git a/pkg/cloudprovider/provider/vsphere/types/types.go b/pkg/cloudprovider/provider/vsphere/types/types.go index b584cb37d..62331ee76 100644 --- a/pkg/cloudprovider/provider/vsphere/types/types.go +++ b/pkg/cloudprovider/provider/vsphere/types/types.go @@ -50,8 +50,8 @@ type RawConfig struct { // Tag represents vsphere tag. type Tag struct { Description string `json:"description,omitempty"` - ID string `json:"id"` - Name string `json:"name"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` CategoryID string `json:"categoryID"` } From a156f614f08ad1ff0c90e347793d9fe5c2b662a7 Mon Sep 17 00:00:00 2001 From: Mario Reggiori Date: Mon, 6 Feb 2023 17:55:25 +0100 Subject: [PATCH 282/489] Anexia Provider: configure rpc-statd service as kubelet dependency (#1553) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Anexia Provider: configure rpc-statd service as kubelet dependency Signed-off-by: Mario Schäfer * Update flatcar cloudinit fixtures Signed-off-by: Mario Schäfer --------- Signed-off-by: Mario Schäfer --- pkg/userdata/flatcar/provider.go | 6 ++++++ pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml | 4 ++++ pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml | 4 ++++ pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml | 4 ++++ 4 files changed, 18 insertions(+) diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index 6a22a14a6..daf3cba76 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -639,6 +639,12 @@ coreos: [Unit] Requires=download-script.service After=download-script.service +{{- if eq .CloudProviderName "anexia" }} + - name: 50-rpc-statd.conf + content: | + [Unit] + Wants=rpc-statd.service +{{- end }} content: | {{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags false | indent 6 }} diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml index 0537cb53f..55cd7654d 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml @@ -97,6 +97,10 @@ coreos: [Unit] Requires=download-script.service After=download-script.service + - name: 50-rpc-statd.conf + content: | + [Unit] + Wants=rpc-statd.service content: | [Unit] After=containerd.service diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml index 89ade7771..4cea15a05 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml @@ -97,6 +97,10 @@ coreos: [Unit] Requires=download-script.service After=download-script.service + - name: 50-rpc-statd.conf + content: | + [Unit] + Wants=rpc-statd.service content: | [Unit] After=containerd.service diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml index 4ee2b0c7b..9be63ca86 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml @@ -97,6 +97,10 @@ coreos: [Unit] Requires=download-script.service After=download-script.service + - name: 50-rpc-statd.conf + content: | + [Unit] + Wants=rpc-statd.service content: | [Unit] After=containerd.service From 86f2a378df3d0528e64301bd9bf186e64427124a Mon Sep 17 00:00:00 2001 From: Matthias Osthues Date: Wed, 8 Feb 2023 11:11:21 +0100 Subject: [PATCH 283/489] Adjust kubevirt image repo (#1554) --- test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index f78cb46cc..90a46bfc1 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -36,7 +36,7 @@ spec: cpus: "1" memory: "4096M" primaryDisk: - osImage: http://image-repo.kube-system.svc.cluster.local/images/<< KUBEVIRT_OS_IMAGE >>.img + osImage: http://image-repo.kube-system.svc/images/<< KUBEVIRT_OS_IMAGE >>.img size: "25Gi" storageClassName: px-csi-db dnsPolicy: "None" From c175998eebd4e98e9944d519d69854599dd93a2f Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Wed, 8 Feb 2023 13:41:31 +0100 Subject: [PATCH 284/489] Generate correct node affinity if no values are passed (#1557) * Generate correct node affinity if no values are passed Signed-off-by: Marvin Beckers * Add test for affinity without values Signed-off-by: Marvin Beckers * Fix yamllint complaint Signed-off-by: Marvin Beckers --------- Signed-off-by: Marvin Beckers --- .../provider/kubevirt/provider.go | 29 +++---- .../provider/kubevirt/provider_test.go | 13 ++- .../kubevirt/testdata/affinity-no-values.yaml | 83 +++++++++++++++++++ 3 files changed, 108 insertions(+), 17 deletions(-) create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 28654c0f9..044439b1b 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -852,6 +852,19 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. func getAffinity(config *Config, matchKey, matchValue string) *corev1.Affinity { affinity := &corev1.Affinity{} + expressions := []corev1.NodeSelectorRequirement{ + { + Key: config.NodeAffinityPreset.Key, + Operator: corev1.NodeSelectorOperator(metav1.LabelSelectorOpExists), + }, + } + + // change the operator if any values were passed for node affinity matching + if len(config.NodeAffinityPreset.Values) > 0 { + expressions[0].Operator = corev1.NodeSelectorOperator(metav1.LabelSelectorOpIn) + expressions[0].Values = config.NodeAffinityPreset.Values + } + // NodeAffinity switch config.NodeAffinityPreset.Type { case softAffinityType: @@ -860,13 +873,7 @@ func getAffinity(config *Config, matchKey, matchValue string) *corev1.Affinity { { Weight: 1, Preference: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: config.NodeAffinityPreset.Key, - Values: config.NodeAffinityPreset.Values, - Operator: corev1.NodeSelectorOperator(metav1.LabelSelectorOpIn), - }, - }, + MatchExpressions: expressions, }, }, }, @@ -876,13 +883,7 @@ func getAffinity(config *Config, matchKey, matchValue string) *corev1.Affinity { RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ NodeSelectorTerms: []corev1.NodeSelectorTerm{ { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: config.NodeAffinityPreset.Key, - Values: config.NodeAffinityPreset.Values, - Operator: corev1.NodeSelectorOperator(metav1.LabelSelectorOpIn), - }, - }, + MatchExpressions: expressions, }, }, }, diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index cdab28d29..1ec62cc0d 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -60,6 +60,7 @@ type kubevirtProviderSpecConf struct { OperatingSystem string TopologySpreadConstraint bool Affinity bool + AffinityValues bool SecondaryDisks bool OsImageSource imageSource } @@ -86,9 +87,11 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { "affinity": { "nodeAffinityPreset": { "type": "hard", - "key": "key1", - "values": [ + "key": "key1" + {{- if .AffinityValues }} + , "values": [ "foo1", "foo2" ] + {{- end }} } }, {{- end }} @@ -194,7 +197,11 @@ func TestNewVirtualMachine(t *testing.T) { }, { name: "affinity", - specConf: kubevirtProviderSpecConf{Affinity: true}, + specConf: kubevirtProviderSpecConf{Affinity: true, AffinityValues: true}, + }, + { + name: "affinity-no-values", + specConf: kubevirtProviderSpecConf{Affinity: true, AffinityValues: false}, }, { name: "secondary-disks", diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml new file mode 100644 index 000000000..f304b2a6e --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml @@ -0,0 +1,83 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: affinity-no-values + md: md-name + name: affinity-no-values + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + creationTimestamp: null + name: affinity-no-values + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: affinity-no-values + md: md-name + spec: + affinity: + nodeAffinity: # Section present if nodeAffinityPreset.type != "" + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: key1 + operator: Exists + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: affinity-no-values + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External From 61c4e782f1574c1b7f62f985567254fb7a84153a Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 15 Feb 2023 18:14:27 +0500 Subject: [PATCH 285/489] Always use latest OSM image in e2e tests (#1561) * Always use latest OSM image in e2e tests Signed-off-by: Waleed Malik * Update k8s versions for e2e tests Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- .prow/provider-azure.yaml | 2 +- examples/operating-system-manager.yaml | 6 ++---- test/e2e/provisioning/helper.go | 6 +++--- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 8325a6df6..4fb60bb79 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-azure - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/azure/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index dadb9193b..403f14e2e 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -974,8 +974,7 @@ spec: spec: serviceAccountName: operating-system-manager-webhook containers: - # TODO: Update this to a semver tag before release. - - image: quay.io/kubermatic/operating-system-manager:8688a1fde001705f4c0b394ed2f4ff4a292b20a9 + - image: quay.io/kubermatic/operating-system-manager:latest imagePullPolicy: IfNotPresent name: webhook command: @@ -1303,8 +1302,7 @@ spec: spec: serviceAccountName: operating-system-manager containers: - # TODO: Update this to a semver tag before release. - - image: quay.io/kubermatic/operating-system-manager:8688a1fde001705f4c0b394ed2f4ff4a292b20a9 + - image: quay.io/kubermatic/operating-system-manager:latest imagePullPolicy: IfNotPresent name: operating-system-manager command: diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 1f5d32ca3..5fed62df6 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,9 +33,9 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.24.9"), - semver.MustParse("v1.25.5"), - semver.MustParse("v1.26.0"), + semver.MustParse("v1.24.10"), + semver.MustParse("v1.25.6"), + semver.MustParse("v1.26.1"), } operatingSystems = []providerconfigtypes.OperatingSystem{ From f0130c944b2df51a6445b929ce03f64ac1f075f4 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 20 Feb 2023 11:17:32 +0500 Subject: [PATCH 286/489] Use AWS spot instances for tests that always run (#1565) Signed-off-by: Waleed Malik --- .prow/provider-aws.yaml | 39 ++----------------- .../provider/anexia/instance_test.go | 2 +- pkg/cloudprovider/provider/aws/provider.go | 1 - .../provider/kubevirt/provider_test.go | 3 +- 4 files changed, 7 insertions(+), 38 deletions(-) diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 8746fe337..67468708c 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-aws - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -45,7 +45,7 @@ presubmits: limits: memory: 7Gi - - name: pull-machine-controller-e2e-aws-legacy-userdata + - name: pull-machine-controller-e2e-aws-spot-instance-legacy-userdata always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" @@ -69,7 +69,7 @@ presubmits: command: - "./hack/ci/run-e2e-tests.sh" args: - - "TestAWSProvisioningE2E" + - "TestAWSSpotInstanceProvisioningE2E" securityContext: privileged: true resources: @@ -141,39 +141,8 @@ presubmits: limits: memory: 7Gi - - name: pull-machine-controller-e2e-aws-flatcar-containerd - run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAWSFlatcarContainerdProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: aws - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - name: pull-machine-controller-e2e-aws-spot-instance - run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" + always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: diff --git a/pkg/cloudprovider/provider/anexia/instance_test.go b/pkg/cloudprovider/provider/anexia/instance_test.go index 0d7641a5f..8340752a9 100644 --- a/pkg/cloudprovider/provider/anexia/instance_test.go +++ b/pkg/cloudprovider/provider/anexia/instance_test.go @@ -20,8 +20,8 @@ import ( "testing" "github.com/gophercloud/gophercloud/testhelper" - "go.anx.io/go-anxcloud/pkg/vsphere/info" + v1 "k8s.io/api/core/v1" ) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 2891d0ab3..9637476ff 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -34,7 +34,6 @@ import ( ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/aws/smithy-go" - gocache "github.com/patrickmn/go-cache" "github.com/prometheus/client_golang/prometheus" diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index 1ec62cc0d..16cb82573 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -25,9 +25,10 @@ import ( "reflect" "testing" + kubevirtv1 "kubevirt.io/api/core/v1" + cloudprovidertesting "github.com/kubermatic/machine-controller/pkg/cloudprovider/testing" "github.com/kubermatic/machine-controller/pkg/providerconfig" - kubevirtv1 "kubevirt.io/api/core/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" From 3ca5c411ba1bfbaa71781699160457c7ba03486a Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 20 Feb 2023 15:41:32 +0500 Subject: [PATCH 287/489] Add providerID support for cloud providers (#1568) Signed-off-by: Waleed Malik --- pkg/cloudprovider/provider/linode/provider.go | 3 +-- pkg/cloudprovider/provider/nutanix/provider.go | 3 +-- pkg/cloudprovider/provider/vmwareclouddirector/provider.go | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index 030fe4585..67b200588 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -403,9 +403,8 @@ func (d *linodeInstance) ID() string { return strconv.Itoa(d.linode.ID) } -// TODO: Implement once we start supporting Linode CCM. func (d *linodeInstance) ProviderID() string { - return "" + return fmt.Sprintf("linode://%s", d.ID()) } func (d *linodeInstance) Addresses() map[string]v1.NodeAddressType { diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 8e43ce3ed..f2dcbf884 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -86,9 +86,8 @@ func (nutanixServer Server) ID() string { return nutanixServer.id } -// NB: Nutanix doesn't have a CCM. func (nutanixServer Server) ProviderID() string { - return "" + return fmt.Sprintf("nutanix://%s", nutanixServer.ID()) } func (nutanixServer Server) Addresses() map[string]corev1.NodeAddressType { diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go index 9c06152d1..4c3cf16ce 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -120,9 +120,8 @@ func (s Server) ID() string { return s.id } -// TODO: Implement once we start supporting vCloud Director CCM. func (s Server) ProviderID() string { - return "" + return fmt.Sprintf("vmware-cloud-director://%s", s.ID()) } func (s Server) Addresses() map[string]corev1.NodeAddressType { From 1176b4cdcaa80d396e10129cfdc2ce9d6fe09c4c Mon Sep 17 00:00:00 2001 From: Yakul Garg <2000yeshu@gmail.com> Date: Mon, 20 Feb 2023 18:45:31 +0530 Subject: [PATCH 288/489] Add basic support for vultr cloud provider (#1531) Signed-off-by: Yakul Garg <2000yeshu@gmail.com> --- .wwhrd.yml | 1 + docs/cloud-provider.md | 9 + examples/vultr-machinedeployment.yaml | 60 +++ go.mod | 2 + go.sum | 6 + pkg/cloudprovider/provider.go | 4 + pkg/cloudprovider/provider/vultr/provider.go | 375 ++++++++++++++++++ .../provider/vultr/types/types.go | 36 ++ pkg/providerconfig/types/types.go | 2 + test/e2e/provisioning/all_e2e_test.go | 19 + .../testdata/machinedeployment-vultr.yaml | 38 ++ 11 files changed, 552 insertions(+) create mode 100644 examples/vultr-machinedeployment.yaml create mode 100644 pkg/cloudprovider/provider/vultr/provider.go create mode 100644 pkg/cloudprovider/provider/vultr/types/types.go create mode 100644 test/e2e/provisioning/testdata/machinedeployment-vultr.yaml diff --git a/.wwhrd.yml b/.wwhrd.yml index 497ede2eb..677ba2b2f 100644 --- a/.wwhrd.yml +++ b/.wwhrd.yml @@ -33,3 +33,4 @@ exceptions: - github.com/ajeddeloh/go-json # Since it's a fork, https://github.com/golang/go/blob/master/LICENSE - github.com/hashicorp/go-version # MPL-2.0 - github.com/hashicorp/go-cleanhttp # MPL-2.0 + - github.com/hashicorp/go-retryablehttp # MPL-2.0 diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index 75a065640..39c701b6c 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -321,3 +321,12 @@ memory: "2048M" ## vSphere Refer to the [VSphere](./vsphere.md#provider-configuration) specific documentation. + +## Vultr + +### machine.spec.providerConfig.cloudProviderSpec +```yaml +apiKey: "<< VULTR_API_KEY >>" +plan: "vhf-8c-32gb" +region: "" +osId: 127 \ No newline at end of file diff --git a/examples/vultr-machinedeployment.yaml b/examples/vultr-machinedeployment.yaml new file mode 100644 index 000000000..390c88453 --- /dev/null +++ b/examples/vultr-machinedeployment.yaml @@ -0,0 +1,60 @@ +apiVersion: v1 +kind: Secret +metadata: + # If you change the namespace/name, you must also + # adjust the rbac rules + name: machine-controller-vultr + namespace: kube-system +type: Opaque +stringData: + apiKey: << VULTR_API_KEY >> +--- +apiVersion: "cluster.k8s.io/v1alpha1" +kind: MachineDeployment +metadata: + name: vultr-machinedeployment + namespace: kube-system +spec: + paused: false + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + minReadySeconds: 0 + selector: + matchLabels: + foo: bar + template: + metadata: + labels: + foo: bar + spec: + providerSpec: + value: + sshPublicKeys: + - "<< YOUR_PUBLIC_KEY >>" + cloudProvider: "vultr" + cloudProviderSpec: + # Can also be set via the env var 'VULTR_API_KEY' on the machine-controller + apiKey: + secretKeyRef: + namespace: kube-system + name: machine-controller-vultr + key: apiKey + region: blr + plan: 'vhf-8c-32gb' + # Required: app_id, image_id, os_id, snapshot_id, or iso_id must be provided. Currently only os_id is supported. + osId: 215 + # Optional + tags: + - tag1 + - tag2 + - tag3 + operatingSystem: "ubuntu" + operatingSystemSpec: + distUpgradeOnBoot: false + disableAutoUpdate: true + versions: + kubelet: 1.24.9 diff --git a/go.mod b/go.mod index 3a754384d..457b639c2 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/tinkerbell/tink v0.8.0 github.com/vmware/go-vcloud-director/v2 v2.18.0 github.com/vmware/govmomi v0.30.0 + github.com/vultr/govultr/v2 v2.17.2 go.anx.io/go-anxcloud v0.5.0 golang.org/x/crypto v0.4.0 golang.org/x/oauth2 v0.3.0 @@ -118,6 +119,7 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.2 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.13 // indirect diff --git a/go.sum b/go.sum index 2eb996499..62c96732b 100644 --- a/go.sum +++ b/go.sum @@ -398,6 +398,10 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= +github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -680,6 +684,8 @@ github.com/vmware/go-vcloud-director/v2 v2.18.0 h1:3kXfaLyYObVBn7SsGxPPiIcqogwnH github.com/vmware/go-vcloud-director/v2 v2.18.0/go.mod h1:KjnB8t5l1bRrc+jLKDJbx0vZLRzz2RPzNQ7xzg7yI3o= github.com/vmware/govmomi v0.30.0 h1:Fm8ugPnnlMSTSceDKY9goGvjmqc6eQLPUSUeNXdpeXA= github.com/vmware/govmomi v0.30.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= +github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs= +github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= diff --git a/pkg/cloudprovider/provider.go b/pkg/cloudprovider/provider.go index 8447be854..55546bf90 100644 --- a/pkg/cloudprovider/provider.go +++ b/pkg/cloudprovider/provider.go @@ -37,6 +37,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/scaleway" vcd "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vmwareclouddirector" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vsphere" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vultr" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -76,6 +77,9 @@ var ( providerconfigtypes.CloudProviderEquinixMetal: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return equinixmetal.New(cvr) }, + providerconfigtypes.CloudProviderVultr: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return vultr.New(cvr) + }, // NB: This is explicitly left to allow old Packet machines to be deleted. // We can handle those machines in the same way as Equinix Metal machines // because there are no API changes. diff --git a/pkg/cloudprovider/provider/vultr/provider.go b/pkg/cloudprovider/provider/vultr/provider.go new file mode 100644 index 000000000..739036c01 --- /dev/null +++ b/pkg/cloudprovider/provider/vultr/provider.go @@ -0,0 +1,375 @@ +/* +Copyright 2023 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vultr + +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/vultr/govultr/v2" + "golang.org/x/oauth2" + + cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" + + "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" + vultrtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vultr/types" + cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "github.com/kubermatic/machine-controller/pkg/providerconfig" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" +) + +type provider struct { + configVarResolver *providerconfig.ConfigVarResolver +} + +// New returns a new vultr provider. +func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return &provider{configVarResolver: configVarResolver} +} + +type Config struct { + APIKey string + Region string + Plan string + OsID string + Tags []string +} + +func getIDForOS(os providerconfigtypes.OperatingSystem) (int, error) { + switch os { + case providerconfigtypes.OperatingSystemUbuntu: + return 1743, nil + // name: CentOS 7 x64 + case providerconfigtypes.OperatingSystemCentOS: + return 167, nil + // name: Rocky Linux 9 x64 + case providerconfigtypes.OperatingSystemRockyLinux: + return 1869, nil + } + return 0, providerconfigtypes.ErrOSNotSupported +} + +func getClient(ctx context.Context, apiKey string) *govultr.Client { + config := &oauth2.Config{} + ts := config.TokenSource(ctx, &oauth2.Token{AccessToken: apiKey}) + return govultr.NewClient(oauth2.NewClient(ctx, ts)) +} + +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { + if provSpec.Value == nil { + return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") + } + + pconfig, err := providerconfigtypes.GetConfig(provSpec) + if err != nil { + return nil, nil, err + } + + if pconfig.OperatingSystemSpec.Raw == nil { + return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") + } + + rawConfig, err := vultrtypes.GetConfig(*pconfig) + if err != nil { + return nil, nil, err + } + + c := Config{} + c.APIKey, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.APIKey, "VULTR_API_KEY") + if err != nil { + return nil, nil, fmt.Errorf("failed to get the value of \"apiKey\" field, error = %w", err) + } + + c.Plan, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Plan) + if err != nil { + return nil, nil, err + } + + c.Region, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Region) + if err != nil { + return nil, nil, err + } + + c.OsID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.OsID) + if err != nil { + return nil, nil, err + } + + c.Tags = rawConfig.Tags + + return &c, pconfig, err +} + +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { + return spec, nil +} + +func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { + c, pc, err := p.getConfig(spec.ProviderSpec) + if err != nil { + return fmt.Errorf("failed to parse config: %w", err) + } + + if c.APIKey == "" { + return errors.New("apiKey is missing") + } + + if c.Region == "" { + return errors.New("region is missing") + } + + if c.Plan == "" { + return errors.New("plan is missing") + } + + if c.OsID == "" { + return errors.New("osID is missing") + } + + _, err = getIDForOS(pc.OperatingSystem) + if err != nil { + return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, err) + } + + client := getClient(ctx, c.APIKey) + + plans, err := client.Region.Availability(ctx, c.Region, "") + + // TODO: Validate region separately + if err != nil { + return fmt.Errorf("invalid/not supported region specified %q: %w", c.Region, err) + } + + planFound := false + + // Check if given plan present in the returned list + for _, plan := range plans.AvailablePlans { + if plan == c.Plan { + planFound = true + } + } + if !planFound { + return fmt.Errorf("invalid/not supported plan specified %q: %w", c.Plan, err) + } + return nil +} + +func (p *provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (*vultrInstance, error) { + c, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), + } + } + + client := getClient(ctx, c.APIKey) + + instances, _, err := client.Instance.List(ctx, &govultr.ListOptions{ + Tag: string(machine.UID), + }) + if err != nil { + return nil, vltErrorToTerminalError(err, "failed to list servers") + } + + for _, instance := range instances { + for _, tag := range instance.Tags { + if tag == string(machine.UID) { + return &vultrInstance{instance: &instance}, nil + } + } + } + + return nil, cloudprovidererrors.ErrInstanceNotFound +} + +func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { + return p.get(ctx, machine) +} + +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { + return "", "", nil +} + +func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { + c, pc, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), + } + } + + client := getClient(ctx, c.APIKey) + + if c.OsID == "" { + osID, err := getIDForOS(pc.OperatingSystem) + if err != nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Invalid operating system specified %q, details = %v", pc.OperatingSystem, err), + } + } + c.OsID = strconv.Itoa(osID) + } + + if c.Tags == nil { + c.Tags = []string{} + } + + c.Tags = append(c.Tags, string(machine.UID)) + + strOsID, err := strconv.Atoi(c.OsID) + if err != nil { + return nil, err + } + + instanceCreateRequest := govultr.InstanceCreateReq{ + Region: c.Region, + Plan: c.Plan, + Label: machine.Spec.Name, + UserData: userdata, + Tags: c.Tags, + OsID: strOsID, + } + + res, err := client.Instance.Create(ctx, &instanceCreateRequest) + if err != nil { + return nil, vltErrorToTerminalError(err, "failed to create server") + } + + return &vultrInstance{instance: res}, nil +} + +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { + instance, err := p.Get(ctx, machine, data) + if err != nil { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { + return true, nil + } + return false, err + } + + c, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return false, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), + } + } + client := getClient(ctx, c.APIKey) + + if err = client.Instance.Delete(ctx, instance.ID()); err != nil { + return false, vltErrorToTerminalError(err, "failed to delete server") + } + + return false, nil +} + +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { + labels := make(map[string]string) + + c, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err == nil { + labels["plan"] = c.Plan + labels["region"] = c.Region + } + + return labels, err +} + +func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { + c, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return fmt.Errorf("failed to decode providerconfig: %w", err) + } + client := getClient(ctx, c.APIKey) + instances, _, err := client.Instance.List(ctx, &govultr.ListOptions{PerPage: 1000}) + if err != nil { + return fmt.Errorf("failed to list instances: %w", err) + } + + for _, instance := range instances { + if instance.Label == machine.Spec.Name && sets.NewString(instance.Tags...).Has(string(machine.UID)) { + _, err = client.Instance.Update(ctx, instance.ID, &govultr.InstanceUpdateReq{ + Tags: sets.NewString(instance.Tags...).Delete(string(machine.UID)).Insert(string(newUID)).List(), + }) + if err != nil { + return fmt.Errorf("failed to tag instance with new UID tag: %w", err) + } + } + } + + return nil +} + +type vultrInstance struct { + instance *govultr.Instance +} + +func (v *vultrInstance) Name() string { + return v.instance.Label +} + +func (v *vultrInstance) ID() string { + return v.instance.ID +} + +func (v *vultrInstance) ProviderID() string { + return "vultr://" + v.instance.ID +} + +func (v *vultrInstance) Addresses() map[string]v1.NodeAddressType { + addresses := map[string]v1.NodeAddressType{} + addresses[v.instance.MainIP] = v1.NodeExternalIP + addresses[v.instance.InternalIP] = v1.NodeInternalIP + return addresses +} + +func (v *vultrInstance) Status() instance.Status { + switch v.instance.Status { + case "active": + return instance.StatusRunning + case "pending": + return instance.StatusCreating + // "suspending" or "resizing" + default: + return instance.StatusUnknown + } +} + +func vltErrorToTerminalError(err error, msg string) error { + prepareAndReturnError := func() error { + return fmt.Errorf("%s, due to %w", msg, err) + } + if err != nil { + return prepareAndReturnError() + } + return err +} + +func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { + return nil +} diff --git a/pkg/cloudprovider/provider/vultr/types/types.go b/pkg/cloudprovider/provider/vultr/types/types.go new file mode 100644 index 000000000..f4b61aee2 --- /dev/null +++ b/pkg/cloudprovider/provider/vultr/types/types.go @@ -0,0 +1,36 @@ +/* +Copyright 2023 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" +) + +type RawConfig struct { + APIKey providerconfigtypes.ConfigVarString `json:"apiKey,omitempty"` + Region providerconfigtypes.ConfigVarString `json:"region"` + Plan providerconfigtypes.ConfigVarString `json:"plan"` + OsID providerconfigtypes.ConfigVarString `json:"osId"` + Tags []string `json:"tags,omitempty"` +} + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 8e7ca8826..02a589fbb 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -58,6 +58,7 @@ const ( CloudProviderNutanix CloudProvider = "nutanix" CloudProviderOpenstack CloudProvider = "openstack" CloudProviderVsphere CloudProvider = "vsphere" + CloudProviderVultr CloudProvider = "vultr" CloudProviderVMwareCloudDirector CloudProvider = "vmware-cloud-director" CloudProviderFake CloudProvider = "fake" CloudProviderAlibaba CloudProvider = "alibaba" @@ -100,6 +101,7 @@ var ( CloudProviderAnexia, CloudProviderScaleway, CloudProviderBaremetal, + CloudProviderVultr, } ) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 0f0548f57..fb021f1db 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -78,6 +78,7 @@ const ( alibabaManifest = "./testdata/machinedeployment-alibaba.yaml" anexiaManifest = "./testdata/machinedeployment-anexia.yaml" nutanixManifest = "./testdata/machinedeployment-nutanix.yaml" + vultrManifest = "./testdata/machinedeployment-vultr.yaml" ) const ( @@ -1047,3 +1048,21 @@ func TestAnexiaProvisioningE2E(t *testing.T) { runScenarios(t, selector, params, anexiaManifest, fmt.Sprintf("anexia-%s", *testRunIdentifier)) } + +// TestVultrProvisioning - a test suite that exercises Vultr provider +// by requesting nodes with different combination of container runtime type, container runtime version and the OS flavour. +func TestVultrProvisioningE2E(t *testing.T) { + t.Parallel() + + // test data + apiKey := os.Getenv("VULTR_API_KEY") + if len(apiKey) == 0 { + t.Fatal("unable to run the test suite, VULTR_API_KEY environment variable cannot be empty") + } + + selector := OsSelector("ubuntu", "centos", "rockylinux") + + // act + params := []string{fmt.Sprintf("<< VULTR_API_KEY >>=%s", apiKey)} + runScenarios(t, selector, params, vultrManifest, fmt.Sprintf("vlt-%s", *testRunIdentifier)) +} diff --git a/test/e2e/provisioning/testdata/machinedeployment-vultr.yaml b/test/e2e/provisioning/testdata/machinedeployment-vultr.yaml new file mode 100644 index 000000000..e7baddeb7 --- /dev/null +++ b/test/e2e/provisioning/testdata/machinedeployment-vultr.yaml @@ -0,0 +1,38 @@ +apiVersion: "cluster.k8s.io/v1alpha1" +kind: MachineDeployment +metadata: + name: << MACHINE_NAME >> + namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + name: << MACHINE_NAME >> + template: + metadata: + labels: + name: << MACHINE_NAME >> + spec: + providerSpec: + value: + sshPublicKeys: + - "<< YOUR_PUBLIC_KEY >>" + cloudProvider: "vultr" + cloudProviderSpec: + apiKey: << VULTR_API_KEY >> + region: blr + plan: 'vhf-8c-32gb' + osId: 127 + operatingSystem: "<< OS_NAME >>" + operatingSystemSpec: + distUpgradeOnBoot: false + disableAutoUpdate: true + versions: + kubelet: "<< KUBERNETES_VERSION >>" From c099bf2a3cb8d84863a210a124fbe3272607924c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Feb 2023 16:37:38 +0100 Subject: [PATCH 289/489] Bump golang.org/x/net from 0.4.0 to 0.7.0 (#1566) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.4.0 to 0.7.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.4.0...v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 13 ++++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 457b639c2..eb6a40591 100644 --- a/go.mod +++ b/go.mod @@ -163,11 +163,11 @@ require ( go.uber.org/multierr v1.9.0 // indirect go.uber.org/zap v1.24.0 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/net v0.4.0 // indirect + golang.org/x/net v0.7.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.3.0 // indirect - golang.org/x/term v0.3.0 // indirect - golang.org/x/text v0.5.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/term v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect diff --git a/go.sum b/go.sum index 62c96732b..dcb2aa01d 100644 --- a/go.sum +++ b/go.sum @@ -857,8 +857,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= -golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -950,15 +950,17 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -968,8 +970,9 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 085dfeb26092ae77018db6ce4dc3db92e43c9243 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 22 Feb 2023 13:11:34 +0500 Subject: [PATCH 290/489] Increase default timeout for OpenStack provider (#1570) Signed-off-by: Waleed Malik --- pkg/cloudprovider/provider/openstack/provider.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 938d74f02..6c48b7fae 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -57,6 +57,7 @@ import ( const ( floatingIPReleaseFinalizer = "kubermatic.io/release-openstack-floating-ip" floatingIPIDAnnotationKey = "kubermatic.io/release-openstack-floating-ip" + clientTimeout = 1 * time.Minute ) // clientGetterFunc returns an OpenStack client. @@ -344,7 +345,10 @@ func getClient(c *Config) (*gophercloud.ProviderClient, error) { } if pc != nil { // use the util's HTTP client to benefit, among other things, from its CA bundle. - pc.HTTPClient = cloudproviderutil.HTTPClientConfig{LogPrefix: "[OpenStack API]"}.New() + pc.HTTPClient = cloudproviderutil.HTTPClientConfig{ + LogPrefix: "[OpenStack API]", + Timeout: clientTimeout, + }.New() } err = goopenstack.Authenticate(pc, opts) From d7106626c272716834a66675487dd20c8132332a Mon Sep 17 00:00:00 2001 From: eiabea Date: Wed, 22 Feb 2023 16:02:36 +0100 Subject: [PATCH 291/489] Add metrics for provisioning nodes (#1572) Add machine_controller_provisioning_time_seconds bucket Add machine_controller_deprovisioning_time_seconds bucket Signed-off-by: Manuel Zangl --- pkg/controller/machine/machine_controller.go | 24 ++++++++++++++++---- pkg/controller/machine/metrics.go | 10 ++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index d5f0f47e2..c4da1a05c 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -155,12 +155,19 @@ type KubeconfigProvider interface { // MetricsCollection is a struct of all metrics used in // this controller. type MetricsCollection struct { - Workers prometheus.Gauge - Errors prometheus.Counter + Workers prometheus.Gauge + Errors prometheus.Counter + Provisioning prometheus.Histogram + Deprovisioning prometheus.Histogram } func (mc *MetricsCollection) MustRegister(registerer prometheus.Registerer) { - registerer.MustRegister(mc.Errors, mc.Workers) + registerer.MustRegister( + mc.Errors, + mc.Workers, + mc.Provisioning, + mc.Deprovisioning, + ) } func Add( @@ -458,6 +465,9 @@ func (r *Reconciler) ensureMachineHasNodeReadyCondition(machine *clusterv1alpha1 return nil } } + + r.metrics.Provisioning.Observe(time.Until(machine.CreationTimestamp.Time).Abs().Seconds()) + return r.updateMachine(machine, func(m *clusterv1alpha1.Machine) { m.Status.Conditions = append(m.Status.Conditions, corev1.NodeCondition{Type: corev1.NodeReady, Status: corev1.ConditionTrue, @@ -595,7 +605,13 @@ func (r *Reconciler) deleteMachine(ctx context.Context, prov cloudprovidertypes. return nil, err } - return nil, r.deleteNodeForMachine(ctx, nodes, machine) + if err := r.deleteNodeForMachine(ctx, nodes, machine); err != nil { + return nil, err + } + + r.metrics.Deprovisioning.Observe(time.Until(machine.DeletionTimestamp.Time).Abs().Seconds()) + + return nil, nil } func (r *Reconciler) retrieveNodesRelatedToMachine(ctx context.Context, machine *clusterv1alpha1.Machine) ([]*corev1.Node, error) { diff --git a/pkg/controller/machine/metrics.go b/pkg/controller/machine/metrics.go index a5d375a93..8906279d6 100644 --- a/pkg/controller/machine/metrics.go +++ b/pkg/controller/machine/metrics.go @@ -50,6 +50,16 @@ func NewMachineControllerMetrics() *MetricsCollection { Name: metricsPrefix + "errors_total", Help: "The total number or unexpected errors the controller encountered", }), + Provisioning: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: metricsPrefix + "provisioning_time_seconds", + Help: "Histogram of times spent from creating a Machine to ready state in the cluster", + Buckets: prometheus.ExponentialBuckets(32, 1.5, 10), + }), + Deprovisioning: prometheus.NewHistogram(prometheus.HistogramOpts{ + Name: metricsPrefix + "deprovisioning_time_seconds", + Help: "Histogram of times spent from deleting a Machine to be removed from cluster and cloud provider", + Buckets: prometheus.ExponentialBuckets(32, 1.5, 10), + }), } // Set default values, so that these metrics always show up From e80f6119cb69c78b124a51be7d9bd47702c77b83 Mon Sep 17 00:00:00 2001 From: Mario Reggiori Date: Thu, 23 Feb 2023 13:10:39 +0100 Subject: [PATCH 292/489] Postpone machine cleanup when instance is still being created (#1571) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mario Schäfer --- pkg/cloudprovider/provider/anexia/instance.go | 4 ++++ pkg/cloudprovider/provider/anexia/provider.go | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/pkg/cloudprovider/provider/anexia/instance.go b/pkg/cloudprovider/provider/anexia/instance.go index cd67d80c5..d84d90f7c 100644 --- a/pkg/cloudprovider/provider/anexia/instance.go +++ b/pkg/cloudprovider/provider/anexia/instance.go @@ -29,6 +29,7 @@ import ( type anexiaInstance struct { isCreating bool + isDeleting bool info *info.Info reservedAddresses []string } @@ -86,6 +87,9 @@ func (ai *anexiaInstance) Addresses() map[string]v1.NodeAddressType { } func (ai *anexiaInstance) Status() instance.Status { + if ai.isDeleting { + return instance.StatusDeleting + } if ai.isCreating { return instance.StatusCreating } diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index b036d9149..e51835ce9 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -466,6 +466,11 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, pd return nil, cloudprovidererrors.ErrInstanceNotFound } + if status.DeprovisioningID != "" { + // info endpoint no longer available for vm -> stop here + return &anexiaInstance{isDeleting: true}, nil + } + if status.InstanceID == "" { progress, err := vsphereAPI.Provisioning().Progress().Get(ctx, status.ProvisioningID) if err != nil { @@ -508,6 +513,13 @@ func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string } func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (isDeleted bool, retErr error) { + if inst, err := p.Get(ctx, machine, data); err != nil { + return false, err + } else if inst.Status() == instance.StatusCreating { + klog.Warningf("Unable to cleanup machine %q. Instance is still creating", machine.Name) + return false, nil + } + status := getProviderStatus(machine) // make sure status is reflected in Machine Object defer func() { @@ -525,11 +537,8 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine if err != nil { return false, newError(common.InvalidConfigurationMachineError, "failed to create Anexia client: %v", err) } - vsphereAPI := vsphere.NewAPI(cli) - if err != nil { - return false, newError(common.InvalidConfigurationMachineError, "failed to get machine status: %v", err) - } + vsphereAPI := vsphere.NewAPI(cli) deleteCtx, cancel := context.WithTimeout(ctx, anxtypes.DeleteRequestTimeout) defer cancel() From b63dfa1a995573ae4de61d98234de851d59180fe Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 28 Feb 2023 10:14:47 +0100 Subject: [PATCH 293/489] Support Private IP for Hetzner Cloud (#1579) * support private ip for hetzner cloud Signed-off-by: Moath Qasim * document hetzner ips assignment Signed-off-by: Moath Qasim * fix linting Signed-off-by: Moath Qasim --------- Signed-off-by: Moath Qasim --- examples/hetzner-machinedeployment.yaml | 4 ++ .../provider/hetzner/provider.go | 45 ++++++++++++++++++- .../provider/hetzner/types/types.go | 2 + 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/examples/hetzner-machinedeployment.yaml b/examples/hetzner-machinedeployment.yaml index 9dafc90bf..57ce82784 100644 --- a/examples/hetzner-machinedeployment.yaml +++ b/examples/hetzner-machinedeployment.yaml @@ -53,6 +53,10 @@ spec: # Optional: network IDs or names networks: - "<< YOUR_NETWORK >>" + # Optional: assignPublicIPv4 whether a public ipv4 should be assigned or not + assignPublicIPv4: true + # Optional: assignPublicIPv4 whether an ipv6 should be assigned or not + assignPublicIPv6: true # Optional: firewall IDs or names firewalls: - "<< YOUR_FIREWALL >>" diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 9c385e809..4252e9ce8 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -65,6 +65,8 @@ type Config struct { Networks []string Firewalls []string Labels map[string]string + AssignIPv4 bool + AssignIPv6 bool } func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { @@ -149,6 +151,14 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c.Firewalls = append(c.Firewalls, firewallValue) } + ipv4, ipv6, err := p.publicIPsAssignment(rawConfig) + if err != nil { + return nil, nil, err + } + + c.AssignIPv4 = ipv4 + c.AssignIPv6 = ipv6 + c.Labels = rawConfig.Labels return &c, pconfig, err @@ -239,6 +249,10 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe } } + if !c.AssignIPv4 && !c.AssignIPv6 && len(c.Networks) < 1 { + return errors.New("server should have either a public ipv4, ipv6 or dedicated network") + } + if _, _, err = client.ServerType.Get(ctx, c.ServerType); err != nil { return fmt.Errorf("failed to get server type: %w", err) } @@ -273,10 +287,15 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, } c.Labels[machineUIDLabelKey] = string(machine.UID) + serverCreateOpts := hcloud.ServerCreateOpts{ Name: machine.Spec.Name, UserData: userdata, Labels: c.Labels, + PublicNet: &hcloud.ServerCreatePublicNet{ + EnableIPv4: c.AssignIPv4, + EnableIPv6: c.AssignIPv6, + }, } if c.Datacenter != "" { @@ -548,7 +567,7 @@ func (s *hetznerServer) Addresses() map[string]v1.NodeAddressType { addresses[s.server.PublicNet.IPv4.IP.String()] = v1.NodeExternalIP // For a given IPv6 network of 2001:db8:1234::/64, the instance address is 2001:db8:1234::1 // Reference: https://github.com/hetznercloud/hcloud-cloud-controller-manager/blob/v1.12.1/hcloud/instances.go#L165-167 - if !s.server.PublicNet.IPv6.IP.IsUnspecified() { + if s.server.PublicNet.IPv6.IP != nil && !s.server.PublicNet.IPv6.IP.IsUnspecified() { s.server.PublicNet.IPv6.IP[len(s.server.PublicNet.IPv6.IP)-1] |= 0x01 addresses[s.server.PublicNet.IPv6.IP.String()] = v1.NodeExternalIP } @@ -591,6 +610,30 @@ func hzErrorToTerminalError(err error, msg string) error { return err } +func (p *provider) publicIPsAssignment(rawConfig *hetznertypes.RawConfig) (bool, bool, error) { + assignIPv4, ipv4Set, err := p.configVarResolver.GetConfigVarBoolValue(rawConfig.AssignPublicIPv4) + if err != nil { + return false, false, err + } + + assignIPv6, ipv6Set, err := p.configVarResolver.GetConfigVarBoolValue(rawConfig.AssignPublicIPv6) + if err != nil { + return false, false, err + } + + // hetzner default behaviour assigns public ips when users don't set them explicitly for the server. In order to + // retain this behaviour, if the field AssignPublicIPv4/AssignPublicIPv6 in MachineDeployment is not set, machine controller + // default them to true. + if !ipv4Set { + assignIPv4 = true + } + if !ipv6Set { + assignIPv6 = true + } + + return assignIPv4, assignIPv6, nil +} + func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/hetzner/types/types.go b/pkg/cloudprovider/provider/hetzner/types/types.go index dd1b86471..1148497ae 100644 --- a/pkg/cloudprovider/provider/hetzner/types/types.go +++ b/pkg/cloudprovider/provider/hetzner/types/types.go @@ -31,6 +31,8 @@ type RawConfig struct { Networks []providerconfigtypes.ConfigVarString `json:"networks"` Firewalls []providerconfigtypes.ConfigVarString `json:"firewalls"` Labels map[string]string `json:"labels,omitempty"` + AssignPublicIPv4 providerconfigtypes.ConfigVarBool `json:"assignPublicIPv4,omitempty"` + AssignPublicIPv6 providerconfigtypes.ConfigVarBool `json:"assignPublicIPv6,omitempty"` } func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { From ce45ab9bc1dc8b221182ddf170dd603429484cc6 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Tue, 14 Mar 2023 09:48:42 +0100 Subject: [PATCH 294/489] adjust VPC to the new one after we recently switched AWS accounts (#1591) * adjust VPC to the new one after we recently switched AWS accounts * fix misleading documentation --- docs/cloud-provider.md | 7 +++---- examples/aws-machinedeployment.yaml | 2 +- .../clusterv1alpha1machineWithProviderConfig/aws.yaml | 2 +- .../conversions/testdata/machinesv1alpha1machine/aws.yaml | 2 +- .../testdata/migrated_clusterv1alpha1machine/aws.yaml | 2 +- .../aws.yaml | 2 +- .../testdata/machinedeployment-aws-arm-machines.yaml | 4 ++-- .../machinedeployment-aws-ebs-encryption-enabled.yaml | 4 ++-- .../testdata/machinedeployment-aws-spot-instances.yaml | 4 ++-- test/e2e/provisioning/testdata/machinedeployment-aws.yaml | 4 ++-- 10 files changed, 16 insertions(+), 17 deletions(-) diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index 39c701b6c..38295e68a 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -57,7 +57,7 @@ region: "eu-central-1" # avaiability zone for the instance availabilityZone: "eu-central-1a" # vpc id for the instance -vpcId: "vpc-819f62e9" +vpcId: "vpc-079f7648481a11e77" # subnet id for the instance subnetId: "subnet-2bff4f43" # enable public IP assignment, default is true @@ -80,8 +80,7 @@ ami: "" # When not set a 'kubernetes-v1' security group will get created securityGroupIDs: - "" -# name of the instance profile to use. -# When not set a 'kubernetes-v1' instance profile will get created +# name of the instance profile to use, required. instanceProfile : "" # instance tags ("KubernetesCluster": "my-cluster" is a required tag. @@ -329,4 +328,4 @@ Refer to the [VSphere](./vsphere.md#provider-configuration) specific documentati apiKey: "<< VULTR_API_KEY >>" plan: "vhf-8c-32gb" region: "" -osId: 127 \ No newline at end of file +osId: 127 diff --git a/examples/aws-machinedeployment.yaml b/examples/aws-machinedeployment.yaml index 633e1ed3c..ce04e8aef 100644 --- a/examples/aws-machinedeployment.yaml +++ b/examples/aws-machinedeployment.yaml @@ -52,7 +52,7 @@ spec: key: secretAccessKey region: "eu-central-1" availabilityZone: "eu-central-1a" - vpcId: "vpc-819f62e9" + vpcId: "vpc-079f7648481a11e77" subnetId: "subnet-2bff4f43" instanceType: "t2.micro" instanceProfile: "kubernetes-v1" diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/aws.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/aws.yaml index 9e94d17e3..52c66c07e 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/aws.yaml +++ b/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/aws.yaml @@ -27,7 +27,7 @@ spec: subnetId: subnet-2bff4f43 tags: KubernetesCluster: 6qsm86c2d - vpcId: vpc-819f62e9 + vpcId: vpc-079f7648481a11e77 operatingSystem: flatcar operatingSystemSpec: disableAutoUpdate: true diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/aws.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/aws.yaml index e9a7245d7..89b00babe 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/aws.yaml +++ b/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/aws.yaml @@ -16,7 +16,7 @@ spec: secretAccessKey: "val" region: "eu-central-1" availabilityZone: "eu-central-1a" - vpcId: "vpc-819f62e9" + vpcId: "vpc-079f7648481a11e77" subnetId: "subnet-2bff4f43" instanceType: "t2.micro" diskSize: 50 diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/aws.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/aws.yaml index 77d13a080..9d89343b9 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/aws.yaml +++ b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/aws.yaml @@ -22,7 +22,7 @@ spec: subnetId: subnet-2bff4f43 tags: KubernetesCluster: 6qsm86c2d - vpcId: vpc-819f62e9 + vpcId: vpc-079f7648481a11e77 operatingSystem: flatcar operatingSystemSpec: disableAutoUpdate: true diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/aws.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/aws.yaml index ec50f95a3..470fc9632 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/aws.yaml +++ b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/aws.yaml @@ -30,7 +30,7 @@ spec: subnetId: subnet-2bff4f43 tags: KubernetesCluster: 6qsm86c2d - vpcId: vpc-819f62e9 + vpcId: vpc-079f7648481a11e77 operatingSystem: flatcar operatingSystemSpec: disableAutoUpdate: true diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml index 793231b7f..a090f9ad8 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml @@ -30,7 +30,7 @@ spec: secretAccessKey: << AWS_SECRET_ACCESS_KEY >> region: "eu-central-1" availabilityZone: "eu-central-1a" - vpcId: "vpc-819f62e9" + vpcId: "vpc-079f7648481a11e77" instanceType: "a1.medium" instanceProfile: "kubernetes-v1" diskSize: 50 @@ -38,7 +38,7 @@ spec: ebsVolumeEncrypted: false ami: "<< AMI >>" securityGroupIDs: - - "sg-a2c195ca" + - "sg-0f1f62df28fb378b7" tags: # you have to set this flag to real clusterID when running against our dev or prod # otherwise you might have issues with your nodes not joining the cluster diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml index 0130744c6..a496f0693 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml @@ -30,14 +30,14 @@ spec: secretAccessKey: << AWS_SECRET_ACCESS_KEY >> region: "eu-central-1" availabilityZone: "eu-central-1a" - vpcId: "vpc-819f62e9" + vpcId: "vpc-079f7648481a11e77" instanceType: "t2.medium" instanceProfile: "kubernetes-v1" diskSize: 50 diskType: "gp2" ebsVolumeEncrypted: true securityGroupIDs: - - "sg-a2c195ca" + - "sg-0f1f62df28fb378b7" tags: # you have to set this flag to real clusterID when running against our dev or prod # otherwise you might have issues with your nodes not joining the cluster diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml index e7febdc8a..880fc8fe2 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml @@ -30,7 +30,7 @@ spec: secretAccessKey: << AWS_SECRET_ACCESS_KEY >> region: "eu-central-1" availabilityZone: "eu-central-1a" - vpcId: "vpc-819f62e9" + vpcId: "vpc-079f7648481a11e77" instanceType: "t2.medium" instanceProfile: "kubernetes-v1" diskSize: 50 @@ -42,7 +42,7 @@ spec: maxPrice: "<< MAX_PRICE >>" persistentRequest: false securityGroupIDs: - - "sg-a2c195ca" + - "sg-0f1f62df28fb378b7" tags: # you have to set this flag to real clusterID when running against our dev or prod # otherwise you might have issues with your nodes not joining the cluster diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws.yaml index 915f71254..c42382332 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws.yaml @@ -32,7 +32,7 @@ spec: assumeRoleExternalID: "<< AWS_ASSUME_ROLE_EXTERNAL_ID >>" region: "eu-central-1" availabilityZone: "eu-central-1a" - vpcId: "vpc-819f62e9" + vpcId: "vpc-079f7648481a11e77" instanceType: "t2.medium" instanceProfile: "kubernetes-v1" diskSize: 50 @@ -40,7 +40,7 @@ spec: ebsVolumeEncrypted: false ami: "<< AMI >>" securityGroupIDs: - - "sg-a2c195ca" + - "sg-0f1f62df28fb378b7" tags: # you have to set this flag to real clusterID when running against our dev or prod # otherwise you might have issues with your nodes not joining the cluster From 20b44fba05821f2f9e24d501e760e5ab1fd07476 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Tue, 14 Mar 2023 11:00:44 +0100 Subject: [PATCH 295/489] do not cache cancelled/timed out validations (#1593) --- pkg/cloudprovider/validationwrapper.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/validationwrapper.go b/pkg/cloudprovider/validationwrapper.go index 965e2205b..77302b075 100644 --- a/pkg/cloudprovider/validationwrapper.go +++ b/pkg/cloudprovider/validationwrapper.go @@ -18,6 +18,7 @@ package cloudprovider import ( "context" + "errors" "fmt" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -56,8 +57,13 @@ func (w *cachingValidationWrapper) Validate(ctx context.Context, spec v1alpha1.M klog.V(6).Infof("Got cache miss for validation") err = w.actualProvider.Validate(ctx, spec) - if err := cache.Set(spec, err); err != nil { - return fmt.Errorf("failed to set cache after validation: %w", err) + + // do not cache canceled contexts (e.g. the validation request was canceled client-side) + // and timeouts (assumed to be temporary) + if !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + if err := cache.Set(spec, err); err != nil { + return fmt.Errorf("failed to set cache after validation: %w", err) + } } return err From 523298e7456d2d8f898240c74e4c23afd9edda98 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Tue, 14 Mar 2023 11:00:52 +0100 Subject: [PATCH 296/489] store MC/OSM logs as artifacts during E2E tests (#1594) --- hack/ci/setup-machine-controller-in-kind.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/hack/ci/setup-machine-controller-in-kind.sh b/hack/ci/setup-machine-controller-in-kind.sh index d34d5b934..7686b7574 100755 --- a/hack/ci/setup-machine-controller-in-kind.sh +++ b/hack/ci/setup-machine-controller-in-kind.sh @@ -54,6 +54,8 @@ if [ ! -f machine-controller-deployed ]; then make deploy touch machine-controller-deployed + + protokol --kubeconfig "$KUBECONFIG" --flat --output "$ARTIFACTS/logs" --namespace kube-system 'machine-controller-*' > /dev/null 2>&1 & fi if [[ "$OPERATING_SYSTEM_MANAGER" == "true" ]]; then @@ -75,6 +77,8 @@ if [[ "$OPERATING_SYSTEM_MANAGER" == "true" ]]; then sed -i -e 's/-worker-count=5/-worker-count=50/g' examples/operating-system-manager.yaml kubectl apply -f examples/operating-system-manager.yaml ) + + protokol --kubeconfig "$KUBECONFIG" --flat --output "$ARTIFACTS/logs" --namespace kube-system 'operating-system-manager-*' > /dev/null 2>&1 & fi sleep 10 From ee0d1cd22731caa42cd769501950959fdf0e42a8 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Thu, 23 Mar 2023 15:19:52 +0100 Subject: [PATCH 297/489] reduce default log level to prevent credentials from leaking (#1597) --- examples/machine-controller.yaml | 3 ++- examples/operating-system-manager.yaml | 4 ++-- pkg/admission/admission.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/machine-controller.yaml b/examples/machine-controller.yaml index b724ba5e8..a06dc9c2f 100644 --- a/examples/machine-controller.yaml +++ b/examples/machine-controller.yaml @@ -260,7 +260,8 @@ spec: command: - /usr/local/bin/webhook - -logtostderr - - -v=6 + # Starting with v=6, full Machine objects with inline credentials are logged, beware! + - -v=4 - -use-osm=true - -namespace=kube-system - -listen-address=0.0.0.0:9876 diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index 403f14e2e..745d362d7 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -980,7 +980,7 @@ spec: command: - /usr/local/bin/webhook - -logtostderr - - -v=6 + - -v=4 - -namespace=kube-system volumeMounts: - name: operating-system-manager-admission-cert @@ -1308,7 +1308,7 @@ spec: command: - /usr/local/bin/osm-controller - -logtostderr - - -v=5 + - -v=4 - -worker-count=5 - -cluster-dns=10.10.10.10 - -metrics-address=0.0.0.0:8080 diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go index cf5ae12ea..beaf4ed90 100644 --- a/pkg/admission/admission.go +++ b/pkg/admission/admission.go @@ -125,7 +125,7 @@ func createAdmissionResponse(original, mutated runtime.Object) (*admissionv1.Adm if err != nil { return nil, fmt.Errorf("failed to marshal json patch: %w", err) } - klog.V(3).Infof("Produced jsonpatch: %s", string(patchRaw)) + klog.V(6).Infof("Produced jsonpatch: %s", string(patchRaw)) response.Patch = patchRaw response.PatchType = &jsonPatch From 4d09804f547b3e241434df21c3b20c15ec1c363e Mon Sep 17 00:00:00 2001 From: Mario Reggiori Date: Mon, 27 Mar 2023 10:54:51 +0200 Subject: [PATCH 298/489] Anexia Provider: fix failed vm deletion deadlock (#1600) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mario Schäfer --- pkg/cloudprovider/provider/anexia/provider.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index e51835ce9..47aa46a83 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -514,6 +514,10 @@ func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (isDeleted bool, retErr error) { if inst, err := p.Get(ctx, machine, data); err != nil { + if cloudprovidererrors.IsNotFound(err) { + return true, nil + } + return false, err } else if inst.Status() == instance.StatusCreating { klog.Warningf("Unable to cleanup machine %q. Instance is still creating", machine.Name) From 28324658dccb6061c78cc32b476e41ceee77e3a9 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 27 Mar 2023 18:20:23 +0500 Subject: [PATCH 299/489] Fix E2E tests for VMware Cloud Director (#1601) * Fix E2E tests for VMware Cloud Director Signed-off-by: Waleed Malik * Compute and pull latest image for OSM in CI Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- hack/ci/setup-machine-controller-in-kind.sh | 15 +++++++++++++++ .../machinedeployment-vmware-cloud-director.yaml | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/hack/ci/setup-machine-controller-in-kind.sh b/hack/ci/setup-machine-controller-in-kind.sh index 7686b7574..bfbf42fa3 100755 --- a/hack/ci/setup-machine-controller-in-kind.sh +++ b/hack/ci/setup-machine-controller-in-kind.sh @@ -23,6 +23,8 @@ fi export MC_VERSION="${MC_VERSION:-$(git rev-parse HEAD)}" export OPERATING_SYSTEM_MANAGER="${OPERATING_SYSTEM_MANAGER:-true}" +OSM_REPO_URL="${OSM_REPO_URL:-https://github.com/kubermatic/operating-system-manager.git}" +OSM_REPO_TAG="${OSM_REPO_TAG:-main}" # Build the Docker image for machine-controller beforeDockerBuild=$(nowms) @@ -69,8 +71,21 @@ if [[ "$OPERATING_SYSTEM_MANAGER" == "true" ]]; then kubectl -n cert-manager rollout status deploy/cert-manager-webhook ) + OSM_TMP_DIR=/tmp/osm + echodate "Clone OSM respository" + ( + # Clone OSM repo + mkdir -p $OSM_TMP_DIR + echodate "Cloning cluster exposer" + git clone --depth 1 --branch "${OSM_REPO_TAG}" "${OSM_REPO_URL}" $OSM_TMP_DIR + ) + echodate "Installing operating-system-manager" ( + OSM_TAG="$(git -C $OSM_TMP_DIR rev-parse HEAD)" + # In release branches we'll have this pinned to a specific semver instead of latest. + sed -i "s;:latest;:$OSM_TAG;g" examples/operating-system-manager.yaml + # This is required for running e2e tests in KIND url="-override-bootstrap-kubelet-apiserver=$MASTER_URL" sed -i "s;-container-runtime=containerd;$url;g" examples/operating-system-manager.yaml diff --git a/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml b/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml index c696987e0..8ce2a4fc0 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml @@ -33,10 +33,10 @@ spec: organization: "<< VCD_ORG >>" vdc: "<< VCD_VDC >>" allowInsecure: false - vapp: "machine-controller-e2e" + vapp: "kubermatic-e2e" catalog: "kubermatic" template: "machine-controller-<< OS_NAME >>" - network: "machine-controller-e2e" + network: "kubermatic-e2e-routed-network" ipAllocationMode: "DHCP" cpus: 2 cpuCores: 1 From d054e612a40b06224fc2f9f544c43fc2a43c2cb0 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Mon, 27 Mar 2023 16:42:17 +0200 Subject: [PATCH 300/489] Validate the AWS VPC has DNS hostnames enabled (#1590) * validate the AWS VPC has DNS hostnames enabled * lint * try different RHEL AMI --- pkg/cloudprovider/provider/aws/provider.go | 26 ++++++++++++++++++++++ test/e2e/provisioning/helper.go | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 9637476ff..bf7bb3398 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -588,6 +588,15 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return fmt.Errorf(util.ErrUnknownNetworkFamily, f) } + dnsHostnames, err := areVpcDNSHostnamesEnabled(ctx, ec2Client, config.VpcID) + if err != nil { + return fmt.Errorf("failed to retrieve VPC attributes: %w", err) + } + + if !dnsHostnames { + return fmt.Errorf("vpc %s does not have the enableDnsHostname attribute enabled, new machines in this VPC would be incompatible with Kubernetes", config.VpcID) + } + _, err = ec2Client.DescribeAvailabilityZones(ctx, &ec2.DescribeAvailabilityZonesInput{ZoneNames: []string{config.AvailabilityZone}}) if err != nil { return fmt.Errorf("invalid zone %q specified: %w", config.AvailabilityZone, err) @@ -639,6 +648,23 @@ func getVpc(ctx context.Context, client *ec2.Client, id string) (*ec2types.Vpc, return &vpcOut.Vpcs[0], nil } +func areVpcDNSHostnamesEnabled(ctx context.Context, client *ec2.Client, id string) (bool, error) { + out, err := client.DescribeVpcAttribute(ctx, &ec2.DescribeVpcAttributeInput{ + VpcId: &id, + Attribute: ec2types.VpcAttributeNameEnableDnsHostnames, + }) + + if err != nil { + return false, awsErrorToTerminalError(err, "failed to describe vpc attributes") + } + + if out.EnableDnsHostnames == nil { + return false, errors.New("API response does not include expected field enableDnsHostnames") + } + + return *out.EnableDnsHostnames.Value, nil +} + func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 5fed62df6..2125d148a 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -206,7 +206,7 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_DISK_SIZE >>=%v", 0)) scenarioParams = append(scenarioParams, fmt.Sprintf("<< DATA_DISK_SIZE >>=%v", 0)) scenarioParams = append(scenarioParams, fmt.Sprintf("<< CUSTOM-IMAGE >>=%v", "rhel-8-1-custom")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< AMI >>=%s", "ami-08c04369895785ac4")) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< AMI >>=%s", "ami-08b25fe3ad2fc9b18")) scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.08")) } else { scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_DISK_SIZE >>=%v", 30)) From 94700596df6b19240f4e599626d7a53095d142a6 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 28 Mar 2023 10:06:17 +0200 Subject: [PATCH 301/489] remove custom AMI from rhel os (#1607) Signed-off-by: Moath Qasim --- test/e2e/provisioning/helper.go | 2 -- .../testdata/machinedeployment-aws-arm-machines.yaml | 1 - .../testdata/machinedeployment-aws-spot-instances.yaml | 1 - test/e2e/provisioning/testdata/machinedeployment-aws.yaml | 1 - 4 files changed, 5 deletions(-) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 2125d148a..c3eab368f 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -206,12 +206,10 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_DISK_SIZE >>=%v", 0)) scenarioParams = append(scenarioParams, fmt.Sprintf("<< DATA_DISK_SIZE >>=%v", 0)) scenarioParams = append(scenarioParams, fmt.Sprintf("<< CUSTOM-IMAGE >>=%v", "rhel-8-1-custom")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< AMI >>=%s", "ami-08b25fe3ad2fc9b18")) scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.08")) } else { scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_DISK_SIZE >>=%v", 30)) scenarioParams = append(scenarioParams, fmt.Sprintf("<< DATA_DISK_SIZE >>=%v", 30)) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< AMI >>=%s", "")) scenarioParams = append(scenarioParams, fmt.Sprintf("<< DISK_SIZE >>=%v", 25)) scenarioParams = append(scenarioParams, fmt.Sprintf("<< CUSTOM-IMAGE >>=%v", "")) scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.03")) diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml index a090f9ad8..37567a878 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml @@ -36,7 +36,6 @@ spec: diskSize: 50 diskType: "gp2" ebsVolumeEncrypted: false - ami: "<< AMI >>" securityGroupIDs: - "sg-0f1f62df28fb378b7" tags: diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml index 880fc8fe2..aa35c5cfa 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml @@ -36,7 +36,6 @@ spec: diskSize: 50 diskType: "gp2" ebsVolumeEncrypted: false - ami: "<< AMI >>" isSpotInstance: true spotInstanceConfig: maxPrice: "<< MAX_PRICE >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws.yaml index c42382332..77ac2bb3b 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws.yaml @@ -38,7 +38,6 @@ spec: diskSize: 50 diskType: "gp2" ebsVolumeEncrypted: false - ami: "<< AMI >>" securityGroupIDs: - "sg-0f1f62df28fb378b7" tags: From 09b7c7deb4023e4156f8b88519a34e44dcdfc5e3 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 28 Mar 2023 15:58:18 +0500 Subject: [PATCH 302/489] Fix vSphere e2e tests (#1608) Signed-off-by: Waleed Malik --- test/e2e/provisioning/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index c3eab368f..d3595c343 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -59,7 +59,7 @@ var ( string(providerconfigtypes.OperatingSystemCentOS): "kkp-centos-7", string(providerconfigtypes.OperatingSystemFlatcar): "kkp-flatcar-3139.2.0", string(providerconfigtypes.OperatingSystemRHEL): "kkp-rhel-8.6", - string(providerconfigtypes.OperatingSystemRockyLinux): "kkp-rockylinux-8.5", + string(providerconfigtypes.OperatingSystemRockyLinux): "kkp-rockylinux-8", string(providerconfigtypes.OperatingSystemUbuntu): "kkp-ubuntu-22.04", } From 0c507ddb86265e72521ee31dbeae6aa04d6a0269 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 28 Mar 2023 20:24:14 +0500 Subject: [PATCH 303/489] Upgrade to Go 1.20.2 (#1584) * Upgrade to Go 1.20.2 Signed-off-by: Waleed Malik * Remove package updates Signed-off-by: Waleed Malik * Fix e2e tests for VCD Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 16 ++++++++-------- .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 6 +++--- .prow/verify.yaml | 8 ++++---- Dockerfile | 4 ++-- Makefile | 2 +- hack/ci/setup-cni-in-kind.sh | 2 +- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- pkg/cloudprovider/provider/hetzner/provider.go | 2 +- pkg/cloudprovider/provider/nutanix/provider.go | 2 +- .../provider/vmwareclouddirector/provider.go | 2 +- pkg/cloudprovider/provider/vsphere/provider.go | 6 +++--- test/e2e/provisioning/verify.go | 2 +- 28 files changed, 50 insertions(+), 50 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 4515e5b39..1842855dc 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -91,7 +91,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -118,7 +118,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 4145a8407..fe1fddf1d 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index e579225af..a8f49cc04 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index d4f443cc7..1f153cc02 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 67468708c..54ccefbf2 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -156,7 +156,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -187,7 +187,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -218,7 +218,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -249,7 +249,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 4fb60bb79..7de7cc483 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 3b7dce3d2..e7244834e 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index c7d80af7d..3a223bbad 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index d879ad984..4ab6572cb 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index de08ca29a..c6d1e8806 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 61d12af71..2a87125ac 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index ac9d1a895..785b340f1 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 1b303e5c5..8f1973a7c 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index b787f092a..3bf604cbb 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -32,7 +32,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -64,7 +64,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 9f0aab11a..e26ad4907 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 481fb8d64..afbf64194 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 2343f0080..f3c53ff7f 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 6aad15648..f3f224410 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.19.4 + - image: golang:1.20.2 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.19.4 + - image: golang:1.20.2 command: - make args: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.19.4 + - image: golang:1.20.2 command: - make args: diff --git a/Dockerfile b/Dockerfile index 380cbde28..8a30556a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.19.4 +ARG GO_VERSION=1.20.2 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . RUN make all -FROM alpine:3.16 +FROM alpine:3.17 RUN apk add --no-cache ca-certificates cdrkit diff --git a/Makefile b/Makefile index ae3ddf1df..a02665e4a 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.19.4 +GO_VERSION ?= 1.20.2 GOOS ?= $(shell go env GOOS) diff --git a/hack/ci/setup-cni-in-kind.sh b/hack/ci/setup-cni-in-kind.sh index 50b075a8d..430a0042d 100755 --- a/hack/ci/setup-cni-in-kind.sh +++ b/hack/ci/setup-cni-in-kind.sh @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -CNI_VERSION="${CNI_VERSION:-v0.8.7}" +CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_bin_dir=/opt/cni/bin mkdir -p /etc/cni/net.d "$cni_bin_dir" diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index b17ff3596..d68fc5b41 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.19.4 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.20.2 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 7d71c7113..3160e2a1f 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 containerize ./hack/verify-licenses.sh go mod vendor diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 4252e9ce8..7af032611 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -425,7 +425,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine client := getClient(c.Token) hzServer := instance.(*hetznerServer).server - res, err := client.Server.Delete(ctx, hzServer) + _, res, err := client.Server.DeleteWithResult(ctx, hzServer) if err != nil { return false, hzErrorToTerminalError(err, "failed to delete the server") } diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index f2dcbf884..0020be4c3 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -260,7 +260,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, if err != nil { _, cleanupErr := p.Cleanup(ctx, machine, data) if cleanupErr != nil { - return nil, fmt.Errorf("cleaning up failed with err %v after creation failed with err %w", cleanupErr, err) + return nil, fmt.Errorf("cleaning up failed with err %w after creation failed with err %w", cleanupErr, err) } return nil, err } diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go index 4c3cf16ce..bb425eb38 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -200,7 +200,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, if err != nil { _, cleanupErr := p.Cleanup(ctx, machine, data) if cleanupErr != nil { - return nil, fmt.Errorf("cleaning up failed with err %v after creation failed with err %w", cleanupErr, err) + return nil, fmt.Errorf("cleaning up failed with err %w after creation failed with err %w", cleanupErr, err) } return nil, err } diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 481a2e519..f5cee0dc9 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -300,7 +300,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, if err != nil { _, cleanupErr := p.Cleanup(ctx, machine, data) if cleanupErr != nil { - return nil, fmt.Errorf("cleaning up failed with err %v after creation failed with err %w", cleanupErr, err) + return nil, fmt.Errorf("cleaning up failed with err %w after creation failed with err %w", cleanupErr, err) } return nil, err } @@ -356,10 +356,10 @@ func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, // Destroy VM to avoid a leftover. destroyTask, vmErr := virtualMachine.Destroy(ctx) if vmErr != nil { - return nil, fmt.Errorf("failed to destroy vm %s after failing upload and attach userdata iso: %w / %v", virtualMachine.Name(), err, vmErr) + return nil, fmt.Errorf("failed to destroy vm %s after failing upload and attach userdata iso: %w / %w", virtualMachine.Name(), err, vmErr) } if vmErr := destroyTask.Wait(ctx); vmErr != nil { - return nil, fmt.Errorf("failed to destroy vm %s after failing upload and attach userdata iso: %w / %v", virtualMachine.Name(), err, vmErr) + return nil, fmt.Errorf("failed to destroy vm %s after failing upload and attach userdata iso: %w / %w", virtualMachine.Name(), err, vmErr) } return nil, machineInvalidConfigurationTerminalError(fmt.Errorf("failed to upload and attach userdata iso: %w", err)) } diff --git a/test/e2e/provisioning/verify.go b/test/e2e/provisioning/verify.go index 2dc7233db..4e1f9bdc2 100644 --- a/test/e2e/provisioning/verify.go +++ b/test/e2e/provisioning/verify.go @@ -175,7 +175,7 @@ func createAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien return false, nil }) if err != nil { - return nil, fmt.Errorf("failed waiting for MachineDeployment %s to get a node: %w (%v)", machineDeployment.Name, err, pollErr) + return nil, fmt.Errorf("failed waiting for MachineDeployment %s to get a node: %w (%w)", machineDeployment.Name, err, pollErr) } klog.Infof("Found a node for MachineDeployment %s", machineDeployment.Name) From e62960d14ff29dfcbf094458518d903eb2325095 Mon Sep 17 00:00:00 2001 From: lucakuendig Date: Tue, 28 Mar 2023 22:14:21 +0200 Subject: [PATCH 304/489] initialize iscsi if cloudprovider is Nutanix (#1596) Signed-off-by: Luca Kuendig --- pkg/userdata/flatcar/provider.go | 6 ++++++ pkg/userdata/flatcar/testdata/ignition_v1.24.0.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.24.9.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.25.0.json | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index daf3cba76..ec4752723 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -490,6 +490,12 @@ storage: {{- end }} systemctl disable setup.service + # Creates iscsi InitiatorName on Nutanix machines for CSI driver to attach volumes. + {{- if eq .CloudProviderName "nutanix" }} + systemctl start iscsi-init.service + systemctl enable --now iscsid.service + {{- end }} + - path: /opt/bin/download.sh filesystem: root mode: 0755 diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json index 48f3d6a2b..72cd8f383 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json b/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json index 71a2b9757..65be0a9d9 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.9%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.9%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json index 3549aa2a5..6baf1897e 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.25.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.25.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file From 0598624c5881a79ec440eb582f1550034d23161c Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 3 Apr 2023 19:25:24 +0500 Subject: [PATCH 305/489] API token authentication support for VMware Cloud Director (#1612) * API token authentication support for VMware Cloud Director Signed-off-by: Waleed Malik * Refactored code Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- ...ware-cloud-director-machinedeployment.yaml | 7 +++ go.mod | 2 +- go.sum | 4 +- .../provider/vmwareclouddirector/client.go | 25 ++++++++--- .../provider/vmwareclouddirector/provider.go | 43 ++++++++++++------- .../vmwareclouddirector/types/types.go | 1 + 6 files changed, 57 insertions(+), 25 deletions(-) diff --git a/examples/vmware-cloud-director-machinedeployment.yaml b/examples/vmware-cloud-director-machinedeployment.yaml index db9507890..15b7011f6 100644 --- a/examples/vmware-cloud-director-machinedeployment.yaml +++ b/examples/vmware-cloud-director-machinedeployment.yaml @@ -48,6 +48,13 @@ spec: namespace: kube-system name: machine-controller-vmware-cloud-director key: password + # Can also be set via the env var 'VCD_API_TOKEN' on the machine-controller + # Either username, password or apiToken should be used for authentication. + apiToken: + secretKeyRef: + namespace: kube-system + name: machine-controller-vmware-cloud-director + key: apiToken # Can also be set via the env var 'VCD_ORG' on the machine-controller organization: "<< VCD_ORG >>" # Can also be set via the env var 'VCD_VDC' on the machine-controller diff --git a/go.mod b/go.mod index eb6a40591..076a82b7a 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10 github.com/sethvargo/go-password v0.2.0 github.com/tinkerbell/tink v0.8.0 - github.com/vmware/go-vcloud-director/v2 v2.18.0 + github.com/vmware/go-vcloud-director/v2 v2.19.0 github.com/vmware/govmomi v0.30.0 github.com/vultr/govultr/v2 v2.17.2 go.anx.io/go-anxcloud v0.5.0 diff --git a/go.sum b/go.sum index dcb2aa01d..be6269960 100644 --- a/go.sum +++ b/go.sum @@ -680,8 +680,8 @@ github.com/vektah/gqlparser/v2 v2.2.0 h1:bAc3slekAAJW6sZTi07aGq0OrfaCjj4jxARAaC7 github.com/vektah/gqlparser/v2 v2.2.0/go.mod h1:i3mQIGIrbK2PD1RrCeMTlVbkF2FJ6WkU1KJlJlC+3F4= github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= -github.com/vmware/go-vcloud-director/v2 v2.18.0 h1:3kXfaLyYObVBn7SsGxPPiIcqogwnHF0FpH5oY3KVSow= -github.com/vmware/go-vcloud-director/v2 v2.18.0/go.mod h1:KjnB8t5l1bRrc+jLKDJbx0vZLRzz2RPzNQ7xzg7yI3o= +github.com/vmware/go-vcloud-director/v2 v2.19.0 h1:A9p95VLn50dm7JbXqg5q+VmQxu3RxoMH6OD5ZeLK9EQ= +github.com/vmware/go-vcloud-director/v2 v2.19.0/go.mod h1:QPxGFgrUcSyzy9IlpwDE4UNT3tsOy2047tJOPEJ4nlw= github.com/vmware/govmomi v0.30.0 h1:Fm8ugPnnlMSTSceDKY9goGvjmqc6eQLPUSUeNXdpeXA= github.com/vmware/govmomi v0.30.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs= diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/client.go b/pkg/cloudprovider/provider/vmwareclouddirector/client.go index 3cd03ad83..a714abda2 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/client.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/client.go @@ -34,11 +34,12 @@ type Client struct { VCDClient *govcd.VCDClient } -func NewClient(username, password, org, url, vdc string, allowInsecure bool) (*Client, error) { +func NewClient(username, password, apiToken, org, url, vdc string, allowInsecure bool) (*Client, error) { client := Client{ Auth: &Auth{ Username: username, Password: password, + APIToken: apiToken, Organization: org, URL: url, VDC: vdc, @@ -61,11 +62,15 @@ func (c *Client) GetAuthenticatedClient() (*govcd.VCDClient, error) { if c.Auth == nil { return nil, fmt.Errorf("authentication configuration not provided") } - if c.Auth.Username == "" { - return nil, fmt.Errorf("username not provided") - } - if c.Auth.Password == "" { - return nil, fmt.Errorf("password not provided") + + // If API token is provided, use it for authentication. + if c.Auth.APIToken == "" { + if c.Auth.Username == "" { + return nil, fmt.Errorf("username not provided") + } + if c.Auth.Password == "" { + return nil, fmt.Errorf("password not provided") + } } if c.Auth.URL == "" { return nil, fmt.Errorf("URL not provided") @@ -85,6 +90,14 @@ func (c *Client) GetAuthenticatedClient() (*govcd.VCDClient, error) { vcdClient := govcd.NewVCDClient(*apiEndpoint, c.Auth.AllowInsecure) + if c.Auth.APIToken != "" { + err = vcdClient.SetToken(c.Auth.Organization, govcd.ApiTokenHeader, c.Auth.APIToken) + if err != nil { + return nil, fmt.Errorf("failed to authenticate with VMware Cloud Director using API Token: %w", err) + } + return vcdClient, nil + } + err = vcdClient.Authenticate(c.Auth.Username, c.Auth.Password, c.Auth.Organization) if err != nil { return nil, fmt.Errorf("failed to authenticate with VMware Cloud Director: %w", err) diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go index bb425eb38..0e568548b 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -61,6 +61,7 @@ type provider struct { type Auth struct { Username string Password string + APIToken string Organization string URL string VDC string @@ -160,7 +161,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine return false, fmt.Errorf("failed to parse config: %w", err) } - client, err := NewClient(c.Username, c.Password, c.Organization, c.URL, c.VDC, c.AllowInsecure) + client, err := NewClient(c.Username, c.Password, c.APIToken, c.Organization, c.URL, c.VDC, c.AllowInsecure) if err != nil { return false, fmt.Errorf("failed to create VMware Cloud Director client: %w", err) } @@ -213,7 +214,7 @@ func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, fmt.Errorf("failed to parse config: %w", err) } - client, err := NewClient(c.Username, c.Password, c.Organization, c.URL, c.VDC, c.AllowInsecure) + client, err := NewClient(c.Username, c.Password, c.APIToken, c.Organization, c.URL, c.VDC, c.AllowInsecure) if err != nil { return nil, fmt.Errorf("failed to create VMware Cloud Director client: %w", err) } @@ -291,7 +292,7 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, da return nil, fmt.Errorf("failed to parse config: %w", err) } - client, err := NewClient(c.Username, c.Password, c.Organization, c.URL, c.VDC, c.AllowInsecure) + client, err := NewClient(c.Username, c.Password, c.APIToken, c.Organization, c.URL, c.VDC, c.AllowInsecure) if err != nil { return nil, fmt.Errorf("failed to create VMware Cloud Director client: %w", err) } @@ -328,34 +329,40 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } c := Config{} + + c.APIToken, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.APIToken, "VCD_API_TOKEN") + if err != nil { + return nil, nil, nil, fmt.Errorf(`failed to get the value of "apiToken" field, error = %w`, err) + } + c.Username, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Username, "VCD_USER") if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"username\" field, error = %w", err) + return nil, nil, nil, fmt.Errorf(`failed to get the value of "username" field, error = %w`, err) } c.Password, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Password, "VCD_PASSWORD") if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"password\" field, error = %w", err) + return nil, nil, nil, fmt.Errorf(`failed to get the value of "password" field, error = %w`, err) } c.Organization, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Organization, "VCD_ORG") if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"organization\" field, error = %w", err) + return nil, nil, nil, fmt.Errorf(`failed to get the value of "organization" field, error = %w`, err) } c.URL, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.URL, "VCD_URL") if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"url\" field, error = %w", err) + return nil, nil, nil, fmt.Errorf(`failed to get the value of "url" field, error = %w`, err) } c.VDC, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.VDC, "VCD_VDC") if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"vdc\" field, error = %w", err) + return nil, nil, nil, fmt.Errorf(`failed to get the value of "vdc" field, error = %w`, err) } c.AllowInsecure, err = p.configVarResolver.GetConfigVarBoolValueOrEnv(rawConfig.AllowInsecure, "VCD_ALLOW_UNVERIFIED_SSL") if err != nil { - return nil, nil, nil, fmt.Errorf("failed to get the value of \"allowInsecure\" field, error = %w", err) + return nil, nil, nil, fmt.Errorf(`failed to get the value of "allowInsecure" field, error = %w`, err) } c.VApp, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VApp) @@ -381,30 +388,30 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c.IPAllocationMode = rawConfig.IPAllocationMode if rawConfig.DiskSizeGB != nil && *rawConfig.DiskSizeGB < 0 { - return nil, nil, nil, fmt.Errorf("value for \"diskSizeGB\" should either be nil or greater than or equal to 0") + return nil, nil, nil, fmt.Errorf(`value for "diskSizeGB" should either be nil or greater than or equal to 0`) } c.DiskSizeGB = rawConfig.DiskSizeGB if rawConfig.DiskIOPS != nil && *rawConfig.DiskIOPS < 0 { - return nil, nil, nil, fmt.Errorf("value for \"diskIOPS\" should either be nil or greater than or equal to 0") + return nil, nil, nil, fmt.Errorf(`value for "diskIOPS" should either be nil or greater than or equal to 0`) } c.DiskIOPS = rawConfig.DiskIOPS if rawConfig.CPUs <= 0 { - return nil, nil, nil, fmt.Errorf("value for \"cpus\" should be greater than 0") + return nil, nil, nil, fmt.Errorf(`value for "cpus" should be greater than 0`) } c.CPUs = rawConfig.CPUs if rawConfig.CPUCores <= 0 { - return nil, nil, nil, fmt.Errorf("value for \"cpuCores\" should be greater than 0") + return nil, nil, nil, fmt.Errorf(`value for "cpuCores" should be greater than 0`) } c.CPUCores = rawConfig.CPUCores if rawConfig.MemoryMB <= 4 { - return nil, nil, nil, fmt.Errorf("value for \"memoryMB\" should be greater than 0") + return nil, nil, nil, fmt.Errorf(`value for "memoryMB" should be greater than 0`) } if rawConfig.MemoryMB%4 != 0 { - return nil, nil, nil, fmt.Errorf("value for \"memoryMB\" should be a multiple of 4") + return nil, nil, nil, fmt.Errorf(`value for "memoryMB" should be a multiple of 4`) } c.MemoryMB = rawConfig.MemoryMB @@ -474,7 +481,11 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) return fmt.Errorf("failed to parse config: %w", err) } - client, err := NewClient(c.Username, c.Password, c.Organization, c.URL, c.VDC, c.AllowInsecure) + if c.APIToken != "" && (c.Password != "" || c.Username != "") { + return fmt.Errorf(`either "apiToken" or "username" and "password" must be specified`) + } + + client, err := NewClient(c.Username, c.Password, c.APIToken, c.Organization, c.URL, c.VDC, c.AllowInsecure) if err != nil { return fmt.Errorf("failed to create VMware Cloud Director client: %w", err) } diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go b/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go index 188e3c2bd..1b4cb7b6d 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go @@ -33,6 +33,7 @@ type RawConfig struct { // Provider configuration. Username providerconfigtypes.ConfigVarString `json:"username"` Password providerconfigtypes.ConfigVarString `json:"password"` + APIToken providerconfigtypes.ConfigVarString `json:"apiToken"` Organization providerconfigtypes.ConfigVarString `json:"organization"` URL providerconfigtypes.ConfigVarString `json:"url"` VDC providerconfigtypes.ConfigVarString `json:"vdc"` From 4ec7ddfc27090722d3df44e11483063f47d51618 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 5 Apr 2023 17:19:34 +0500 Subject: [PATCH 306/489] Use OSM with Anexia E2E tests (#1582) * Use OSM with Anexia E2E tests Signed-off-by: Waleed Malik * Always pull latest image for OSM Signed-off-by: Waleed Malik * Compute image for OSM Signed-off-by: Waleed Malik * Hardcode the OSP name for Anexia Signed-off-by: Waleed Malik * Refactored code Signed-off-by: Waleed Malik * Make Anexia E2E tests optional Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- .prow/provider-anexia.yaml | 7 ++++--- examples/operating-system-manager.yaml | 4 ++-- hack/ci/setup-machine-controller-in-kind.sh | 3 ++- .../provisioning/testdata/machinedeployment-anexia.yaml | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 1f153cc02..b6c2983c7 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -14,6 +14,10 @@ presubmits: - name: pull-machine-controller-e2e-anexia + # We've made the E2E tests for Anexia optional since it doesn't support k8s v1.26 at the moment. + # the tests on k8s v1.26+ will fail. + # TODO: These tests shouldn't be marked as optional. + optional: true always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" @@ -33,9 +37,6 @@ presubmits: args: - "TestAnexiaProvisioningE2E" env: - # OperatingSystemManager does not yet support Anexia - - name: OPERATING_SYSTEM_MANAGER - value: "false" - name: CLOUD_PROVIDER value: anexia securityContext: diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index 745d362d7..86e945774 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -975,7 +975,7 @@ spec: serviceAccountName: operating-system-manager-webhook containers: - image: quay.io/kubermatic/operating-system-manager:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always name: webhook command: - /usr/local/bin/webhook @@ -1303,7 +1303,7 @@ spec: serviceAccountName: operating-system-manager containers: - image: quay.io/kubermatic/operating-system-manager:latest - imagePullPolicy: IfNotPresent + imagePullPolicy: Always name: operating-system-manager command: - /usr/local/bin/osm-controller diff --git a/hack/ci/setup-machine-controller-in-kind.sh b/hack/ci/setup-machine-controller-in-kind.sh index bfbf42fa3..4778e7654 100755 --- a/hack/ci/setup-machine-controller-in-kind.sh +++ b/hack/ci/setup-machine-controller-in-kind.sh @@ -80,9 +80,10 @@ if [[ "$OPERATING_SYSTEM_MANAGER" == "true" ]]; then git clone --depth 1 --branch "${OSM_REPO_TAG}" "${OSM_REPO_URL}" $OSM_TMP_DIR ) - echodate "Installing operating-system-manager" ( OSM_TAG="$(git -C $OSM_TMP_DIR rev-parse HEAD)" + echodate "Installing operating-system-manager with image: $OSM_TAG" + # In release branches we'll have this pinned to a specific semver instead of latest. sed -i "s;:latest;:$OSM_TAG;g" examples/operating-system-manager.yaml diff --git a/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml b/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml index 87e539fd8..2507cb58d 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml @@ -4,7 +4,7 @@ metadata: name: << MACHINE_NAME >> namespace: kube-system annotations: - k8c.io/operating-system-profile: osp-<< OS_NAME >> + k8c.io/operating-system-profile: osp-flatcar-cloud-init spec: replicas: 1 strategy: From dc7a8a26a023106f6d5f85f25f60bf28421e2a87 Mon Sep 17 00:00:00 2001 From: pprzekwas Date: Thu, 6 Apr 2023 13:22:09 +0200 Subject: [PATCH 307/489] Update Golang version to 1.20.3 (#1614) --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 16 ++++++++-------- .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 6 +++--- .prow/verify.yaml | 8 ++++---- Dockerfile | 2 +- Makefile | 2 +- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- 22 files changed, 41 insertions(+), 41 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 1842855dc..c0d2df969 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -91,7 +91,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -118,7 +118,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index fe1fddf1d..65ba53edb 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index a8f49cc04..c4a0375c1 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index b6c2983c7..d00c6bdc5 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 54ccefbf2..6e1d29d08 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -156,7 +156,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -187,7 +187,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -218,7 +218,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -249,7 +249,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 7de7cc483..e0cfdc5a7 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index e7244834e..6145fb7f4 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 3a223bbad..7e1b222f0 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 4ab6572cb..f12c38ae7 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index c6d1e8806..602f5706e 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 2a87125ac..b6c3e1d78 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 785b340f1..db653095f 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 8f1973a7c..d7b2b58ad 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 3bf604cbb..1a6913f88 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -32,7 +32,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -64,7 +64,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index e26ad4907..15c9a6a95 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index afbf64194..b51fe7283 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index f3c53ff7f..86f0db674 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index f3f224410..9ab548f3e 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.20.2 + - image: golang:1.20.3 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.20.2 + - image: golang:1.20.3 command: - make args: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.20.2 + - image: golang:1.20.3 command: - make args: diff --git a/Dockerfile b/Dockerfile index 8a30556a9..2d0c45d20 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.20.2 +ARG GO_VERSION=1.20.3 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index a02665e4a..e6f40e700 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.20.2 +GO_VERSION ?= 1.20.3 GOOS ?= $(shell go env GOOS) diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index d68fc5b41..790c2e140 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.20.2 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.20.3 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 3160e2a1f..4eaa345c0 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-2 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 containerize ./hack/verify-licenses.sh go mod vendor From 51e71701544a8ea69739c249de75fd69f198d936 Mon Sep 17 00:00:00 2001 From: Furkhat Date: Thu, 6 Apr 2023 16:24:14 +0300 Subject: [PATCH 308/489] fix not found check: errors.As for type match whereas errors.Is for value equality (#1616) Signed-off-by: Furkhat Kasymovgeniiuulu --- pkg/cloudprovider/provider/openstack/provider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 6c48b7fae..90acf3423 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -764,7 +764,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine return false, osErrorToTerminalError(err, "failed to get compute client") } - if err := osservers.Delete(computeClient, instance.ID()).ExtractErr(); err != nil && !errors.Is(err, &gophercloud.ErrDefault404{}) { + if err := osservers.Delete(computeClient, instance.ID()).ExtractErr(); err != nil && !errors.As(err, &gophercloud.ErrDefault404{}) { return false, osErrorToTerminalError(err, "failed to delete instance") } @@ -1035,7 +1035,7 @@ func (p *provider) cleanupFloatingIP(machine *clusterv1alpha1.Machine, updater c if err != nil { return fmt.Errorf("failed to create the networkv2 client for region %s: %w", c.Region, err) } - if err := osfloatingips.Delete(netClient, floatingIPID).ExtractErr(); err != nil && !errors.Is(err, &gophercloud.ErrDefault404{}) { + if err := osfloatingips.Delete(netClient, floatingIPID).ExtractErr(); err != nil && !errors.As(err, &gophercloud.ErrDefault404{}) { return fmt.Errorf("failed to delete floating ip %s: %w", floatingIPID, err) } if err := updater(machine, func(m *clusterv1alpha1.Machine) { From dd2680d56c209ddec98a0b75c7455ab9e014868f Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Sun, 9 Apr 2023 12:56:41 +0200 Subject: [PATCH 309/489] Refactor gcp client creating (#1613) * refactor gcp client creating Signed-off-by: Moath Qasim Signed-off-by: Moath Qasim * addressing pr reviews Signed-off-by: Moath Qasim Signed-off-by: Moath Qasim --------- Signed-off-by: Moath Qasim --- examples/gce-machinedeployment.yaml | 4 ++ pkg/cloudprovider/provider/gce/config.go | 37 +++++++++++++++---- pkg/cloudprovider/provider/gce/provider.go | 2 +- pkg/cloudprovider/provider/gce/service.go | 17 ++++++--- pkg/cloudprovider/provider/gce/types/types.go | 1 + 5 files changed, 48 insertions(+), 13 deletions(-) diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index bb4392e28..cdef9d4cd 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -50,6 +50,10 @@ spec: key: serviceAccount # See https://cloud.google.com/compute/docs/regions-zones/ zone: "europe-west3-a" + # Is the id of the GCP project that can be used to create machines in. Usually this id is taken from the + # service account however, it should be possible to create a machine in another project, as long as the + # machine controller has the right permissions + projectID: "" # See https://cloud.google.com/compute/docs/machine-types machineType: "n1-standard-2" # In GB diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index e292157a7..d47d16635 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -21,14 +21,15 @@ limitations under the License. package gce import ( + "context" "encoding/base64" "encoding/json" "errors" "fmt" "strings" - "golang.org/x/oauth2/google" - "golang.org/x/oauth2/jwt" + "golang.org/x/oauth2" + googleoauth "golang.org/x/oauth2/google" "google.golang.org/api/compute/v1" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -106,7 +107,6 @@ type config struct { provisioningModel *string labels map[string]string tags []string - jwtConfig *jwt.Config providerConfig *providerconfigtypes.Config assignPublicIPAddress bool multizone bool @@ -116,6 +116,12 @@ type config struct { enableNestedVirtualization bool minCPUPlatform string guestOSFeatures []string + clientConfig *clientConfig +} + +type clientConfig struct { + ClientEmail string + TokenSource oauth2.TokenSource } // newConfig creates a Provider configuration out of the passed resolver and spec. @@ -140,6 +146,11 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide return nil, fmt.Errorf("cannot retrieve service account: %w", err) } + cfg.projectID, err = resolver.GetConfigVarStringValue(cpSpec.ProjectID) + if err != nil { + return nil, fmt.Errorf("failed to retrieve project id: %w", err) + } + err = cfg.postprocessServiceAccount() if err != nil { return nil, fmt.Errorf("cannot prepare JWT: %w", err) @@ -251,16 +262,28 @@ func (cfg *config) postprocessServiceAccount() error { sa = string(decoded) } + creds, err := googleoauth.CredentialsFromJSON(context.TODO(), []byte(sa), compute.ComputeScope) + if err != nil { + return fmt.Errorf("failed to parse credentials from google service account: %w", err) + } + + if cfg.projectID == "" { + cfg.projectID = creds.ProjectID + } + sam := map[string]string{} err = json.Unmarshal([]byte(sa), &sam) if err != nil { return fmt.Errorf("failed unmarshalling service account: %w", err) } - cfg.projectID = sam["project_id"] - cfg.jwtConfig, err = google.JWTConfigFromJSON([]byte(sa), compute.ComputeScope) - if err != nil { - return fmt.Errorf("failed preparing JWT: %w", err) + + // if the project id is not set in the machine deployment, we fallback to the project id that is embedded in the + // google service account json object. + cfg.clientConfig = &clientConfig{ + ClientEmail: sam["client_email"], + TokenSource: creds.TokenSource, } + return nil } diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 72f13602a..ba4f0e947 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -261,7 +261,7 @@ func (p *Provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, if !cfg.disableMachineServiceAccount { inst.ServiceAccounts = []*compute.ServiceAccount{ { - Email: cfg.jwtConfig.Email, + Email: cfg.clientConfig.ClientEmail, Scopes: append( monitoring.DefaultAuthScopes(), compute.ComputeScope, diff --git a/pkg/cloudprovider/provider/gce/service.go b/pkg/cloudprovider/provider/gce/service.go index d58a9268d..3a18c1b01 100644 --- a/pkg/cloudprovider/provider/gce/service.go +++ b/pkg/cloudprovider/provider/gce/service.go @@ -22,9 +22,11 @@ package gce import ( "context" + "errors" "fmt" "time" + "golang.org/x/oauth2" "google.golang.org/api/compute/v1" "google.golang.org/api/option" @@ -54,12 +56,17 @@ type service struct { // connectComputeService establishes a service connection to the Compute Engine. func connectComputeService(cfg *config) (*service, error) { - client := cfg.jwtConfig.Client(context.Background()) - svc, err := compute.NewService(context.Background(), option.WithHTTPClient(client)) - if err != nil { - return nil, fmt.Errorf("cannot connect to Google Cloud: %w", err) + if cfg.clientConfig != nil && + cfg.clientConfig.TokenSource != nil { + client := oauth2.NewClient(context.Background(), cfg.clientConfig.TokenSource) + svc, err := compute.NewService(context.Background(), option.WithHTTPClient(client)) + if err != nil { + return nil, fmt.Errorf("cannot connect to Google Cloud: %w", err) + } + return &service{svc}, nil } - return &service{svc}, nil + + return nil, errors.New("gcp token source was not found") } // networkInterfaces returns the configured network interfaces for an instance creation. diff --git a/pkg/cloudprovider/provider/gce/types/types.go b/pkg/cloudprovider/provider/gce/types/types.go index 8b8736bbe..096bc3f88 100644 --- a/pkg/cloudprovider/provider/gce/types/types.go +++ b/pkg/cloudprovider/provider/gce/types/types.go @@ -51,6 +51,7 @@ type CloudProviderSpec struct { EnableNestedVirtualization providerconfigtypes.ConfigVarBool `json:"enableNestedVirtualization,omitempty"` MinCPUPlatform providerconfigtypes.ConfigVarString `json:"minCPUPlatform,omitempty"` GuestOSFeatures []string `json:"guestOSFeatures,omitempty"` + ProjectID providerconfigtypes.ConfigVarString `json:"projectID,omitempty"` } // UpdateProviderSpec updates the given provider spec with changed From 052a4cad4a2565362bb42427ab69bad08e21642b Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Thu, 20 Apr 2023 13:06:38 +0200 Subject: [PATCH 310/489] Use zap instead of klog (#1606) * add logging bootstrapping * replace klog everywhere in the controllers --- cmd/machine-controller/main.go | 72 ++++-- cmd/userdata/amzn2/main.go | 22 +- cmd/userdata/centos/main.go | 22 +- cmd/userdata/flatcar/main.go | 21 +- cmd/userdata/rhel/main.go | 22 +- cmd/userdata/rockylinux/main.go | 22 +- cmd/userdata/ubuntu/main.go | 22 +- cmd/webhook/main.go | 48 ++-- examples/machine-controller.yaml | 10 +- go.mod | 8 +- hack/ci/setup-machine-controller-in-kind.sh | 3 + hack/run-machine-controller.sh | 2 +- pkg/admission/admission.go | 25 +- pkg/admission/machinedeployments.go | 6 +- pkg/admission/machines.go | 11 +- pkg/apis/cluster/common/plugins.go | 8 +- .../cluster/v1alpha1/migrations/migrations.go | 108 ++++---- .../provider/alibaba/provider.go | 15 +- pkg/cloudprovider/provider/anexia/provider.go | 68 +++-- .../provider/anexia/provider_test.go | 12 +- .../provider/anexia/types/types.go | 2 +- pkg/cloudprovider/provider/aws/provider.go | 26 +- .../provider/azure/create_delete_resources.go | 18 +- pkg/cloudprovider/provider/azure/provider.go | 68 ++--- .../baremetal/plugins/tinkerbell/hardware.go | 3 - .../provider/baremetal/provider.go | 14 +- .../provider/digitalocean/provider.go | 24 +- .../provider/equinixmetal/provider.go | 22 +- pkg/cloudprovider/provider/fake/provider.go | 19 +- pkg/cloudprovider/provider/gce/provider.go | 17 +- .../provider/gce/provider_test.go | 4 +- pkg/cloudprovider/provider/gce/service.go | 8 +- .../provider/hetzner/provider.go | 24 +- .../provider/kubevirt/provider.go | 18 +- pkg/cloudprovider/provider/linode/provider.go | 15 +- .../provider/nutanix/provider.go | 16 +- .../provider/openstack/helper.go | 9 +- .../provider/openstack/provider.go | 159 ++++++------ .../provider/openstack/provider_test.go | 5 +- .../provider/scaleway/provider.go | 16 +- .../provider/vmwareclouddirector/client.go | 2 +- .../provider/vmwareclouddirector/helper.go | 4 +- .../provider/vmwareclouddirector/provider.go | 15 +- pkg/cloudprovider/provider/vsphere/helper.go | 34 +-- .../provider/vsphere/helper_test.go | 3 +- .../provider/vsphere/provider.go | 38 +-- .../provider/vsphere/provider_test.go | 3 +- pkg/cloudprovider/provider/vultr/provider.go | 18 +- pkg/cloudprovider/types/types.go | 17 +- pkg/cloudprovider/validationwrapper.go | 31 +-- pkg/clusterinfo/configmap.go | 8 +- pkg/clusterinfo/configmap_test.go | 3 +- .../{machine_controller.go => controller.go} | 242 ++++++++++-------- .../{machine_test.go => controller_test.go} | 16 +- pkg/controller/machine/kubeconfig.go | 6 +- ...deployment_controller.go => controller.go} | 79 +++--- pkg/controller/machinedeployment/rolling.go | 31 +-- pkg/controller/machinedeployment/sync.go | 38 +-- ...machineset_controller.go => controller.go} | 91 ++++--- pkg/controller/machineset/machine.go | 19 +- pkg/controller/machineset/status.go | 29 ++- .../{node_csr_approver.go => controller.go} | 57 +++-- ...sr_approver_test.go => controller_test.go} | 0 pkg/controller/util/machine_deployment.go | 46 ++-- pkg/health/readiness.go | 10 +- pkg/log/zap.go | 168 ++++++++++++ pkg/node/eviction/eviction.go | 50 ++-- pkg/node/eviction/eviction_test.go | 5 +- pkg/node/nodemanager/node_manager.go | 20 +- pkg/node/poddeletion/pod_deletion.go | 51 ++-- pkg/rhsm/satellite_subscription_manager.go | 13 +- .../satellite_subscription_manager_test.go | 4 +- pkg/rhsm/subscription_manager.go | 16 +- pkg/rhsm/subscription_manager_test.go | 4 +- pkg/userdata/amzn2/provider.go | 5 +- pkg/userdata/amzn2/provider_test.go | 4 +- pkg/userdata/centos/provider.go | 5 +- pkg/userdata/centos/provider_test.go | 4 +- pkg/userdata/convert/ignition-converter.go | 5 +- pkg/userdata/flatcar/provider.go | 5 +- pkg/userdata/flatcar/provider_test.go | 4 +- .../helper/download_binaries_script.go | 10 +- .../helper/download_binaries_script_test.go | 6 +- pkg/userdata/helper/kubelet.go | 21 +- pkg/userdata/helper/kubelet_test.go | 2 + pkg/userdata/helper/template_functions.go | 40 ++- pkg/userdata/manager/manager.go | 14 +- pkg/userdata/manager/plugin.go | 21 +- pkg/userdata/plugin/plugin.go | 8 +- pkg/userdata/rhel/provider.go | 7 +- pkg/userdata/rhel/provider_test.go | 4 +- pkg/userdata/rockylinux/provider.go | 5 +- pkg/userdata/rockylinux/provider_test.go | 4 +- pkg/userdata/ubuntu/provider.go | 5 +- pkg/userdata/ubuntu/provider_test.go | 3 +- test/e2e/provisioning/all_e2e_test.go | 68 ++--- test/e2e/provisioning/helper.go | 2 +- test/e2e/provisioning/migrateuidscenario.go | 20 +- test/e2e/provisioning/verify.go | 4 +- 99 files changed, 1487 insertions(+), 974 deletions(-) rename pkg/controller/machine/{machine_controller.go => controller.go} (83%) rename pkg/controller/machine/{machine_test.go => controller_test.go} (96%) rename pkg/controller/machinedeployment/{machinedeployment_controller.go => controller.go} (81%) rename pkg/controller/machineset/{machineset_controller.go => controller.go} (83%) rename pkg/controller/nodecsrapprover/{node_csr_approver.go => controller.go} (86%) rename pkg/controller/nodecsrapprover/{node_csr_approver_test.go => controller_test.go} (100%) create mode 100644 pkg/log/zap.go diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 2a1b0a1a3..1e591ff7a 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -20,13 +20,16 @@ import ( "context" "flag" "fmt" + "log" "net" "net/http" "net/http/pprof" "strings" "time" + "github.com/go-logr/zapr" "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1/migrations" @@ -39,6 +42,7 @@ import ( machinesetcontroller "github.com/kubermatic/machine-controller/pkg/controller/machineset" "github.com/kubermatic/machine-controller/pkg/controller/nodecsrapprover" "github.com/kubermatic/machine-controller/pkg/health" + machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" machinesv1alpha1 "github.com/kubermatic/machine-controller/pkg/machines/v1alpha1" "github.com/kubermatic/machine-controller/pkg/node" @@ -48,8 +52,8 @@ import ( "k8s.io/client-go/kubernetes/scheme" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog" "sigs.k8s.io/controller-runtime/pkg/healthz" + ctrlruntimelog "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" "sigs.k8s.io/controller-runtime/pkg/metrics" @@ -137,12 +141,15 @@ type controllerRunOptions struct { nodePortRange string overrideBootstrapKubeletAPIServer string + + log *zap.SugaredLogger } func main() { nodeFlags := node.NewFlags(flag.CommandLine) + logFlags := machinecontrollerlog.NewDefaultOptions() + logFlags.AddFlags(flag.CommandLine) - klog.InitFlags(nil) // This is also being registered in kubevirt.io/kubevirt/pkg/kubecli/kubecli.go so // we have to guard it. // TODO: Evaluate alternatives to importing the CLI. Generate our own client? Use a dynamic client? @@ -183,12 +190,23 @@ func main() { flag.StringVar(&overrideBootstrapKubeletAPIServer, "override-bootstrap-kubelet-apiserver", "", "Override for the API server address used in worker nodes bootstrap-kubelet.conf") flag.Parse() + + if err := logFlags.Validate(); err != nil { + log.Fatalf("Invalid options: %v", err) + } + + rawLog := machinecontrollerlog.New(logFlags.Debug, logFlags.Format) + log := rawLog.Sugar() + + // set the logger used by controller-runtime + ctrlruntimelog.SetLogger(zapr.NewLogger(rawLog.WithOptions(zap.AddCallerSkip(1)))) + kubeconfig = flag.Lookup("kubeconfig").Value.(flag.Getter).Get().(string) masterURL = flag.Lookup("master").Value.(flag.Getter).Get().(string) clusterDNSIPs, err := parseClusterDNSIPs(clusterDNSIPs) if err != nil { - klog.Fatalf("invalid cluster dns specified: %v", err) + log.Fatalw("Invalid cluster dns specified", zap.Error(err)) } var parsedJoinClusterTimeout *time.Duration @@ -196,29 +214,29 @@ func main() { parsedJoinClusterTimeoutLiteral, err := time.ParseDuration(joinClusterTimeout) parsedJoinClusterTimeout = &parsedJoinClusterTimeoutLiteral if err != nil { - klog.Fatalf("failed to parse join-cluster-timeout as duration: %v", err) + log.Fatalw("Failed to parse join-cluster-timeout as duration", zap.Error(err)) } } // Needed for migrations if err := machinesv1alpha1.AddToScheme(scheme.Scheme); err != nil { - klog.Fatalf("failed to add machinesv1alpha1 api to scheme: %v", err) + log.Fatalw("Failed to add api to scheme", "api", machinesv1alpha1.SchemeGroupVersion, zap.Error(err)) } if err := apiextensionsv1.AddToScheme(scheme.Scheme); err != nil { - klog.Fatalf("failed to add apiextensionsv1 api to scheme: %v", err) + log.Fatalw("Failed to add api to scheme", "api", apiextensionsv1.SchemeGroupVersion, zap.Error(err)) } if err := clusterv1alpha1.AddToScheme(scheme.Scheme); err != nil { - klog.Fatalf("failed to add clusterv1alpha1 api to scheme: %v", err) + log.Fatalw("Failed to add api to scheme", "api", clusterv1alpha1.SchemeGroupVersion, zap.Error(err)) } cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) if err != nil { - klog.Fatalf("error building kubeconfig: %v", err) + log.Fatalw("Failed to build kubeconfig", zap.Error(err)) } if caBundleFile != "" { if err := util.SetCABundleFile(caBundleFile); err != nil { - klog.Fatalf("-ca-bundle is invalid: %v", err) + log.Fatalw("-ca-bundle is invalid", zap.Error(err)) } } @@ -228,12 +246,12 @@ func main() { // QPS and Burst config there machineCfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) if err != nil { - klog.Fatalf("error building kubeconfig for machines: %v", err) + log.Fatalw("Failed to build kubeconfig for machines", zap.Error(err)) } kubeClient, err := kubernetes.NewForConfig(cfg) if err != nil { - klog.Fatalf("error building kubernetes clientset for kubeClient: %v", err) + log.Fatalw("Failed to build kubernetes clientset for kubeClient", zap.Error(err)) } kubeconfigProvider := clusterinfo.New(cfg, kubeClient) @@ -251,10 +269,11 @@ func main() { } containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) if err != nil { - klog.Fatalf("failed to generate container runtime config: %v", err) + log.Fatalw("Failed to generate container runtime config", zap.Error(err)) } runOptions := controllerRunOptions{ + log: log, kubeClient: kubeClient, kubeconfigProvider: kubeconfigProvider, name: name, @@ -277,7 +296,7 @@ func main() { } if err := nodeFlags.UpdateNodeSettings(&runOptions.node); err != nil { - klog.Fatalf("failed to update nodesettings: %v", err) + log.Fatalw("Failed to update nodesettings", zap.Error(err)) } if parsedJoinClusterTimeout != nil { @@ -287,7 +306,7 @@ func main() { if bootstrapTokenServiceAccountName != "" { flagParts := strings.Split(bootstrapTokenServiceAccountName, "/") if flagPartsLen := len(flagParts); flagPartsLen != 2 { - klog.Fatalf("Splitting the bootstrap-token-service-account-name flag value in '/' returned %d parts, expected exactly two", flagPartsLen) + log.Fatalf("Splitting the bootstrap-token-service-account-name flag value in '/' returned %d parts, expected exactly two", flagPartsLen) } runOptions.bootstrapTokenServiceAccountName = &types.NamespacedName{Namespace: flagParts[0], Name: flagParts[1]} } @@ -295,16 +314,16 @@ func main() { ctx := signals.SetupSignalHandler() go func() { <-ctx.Done() - klog.Info("caught signal, shutting down...") + log.Info("Caught signal, shutting down...") }() mgr, err := createManager(5*time.Minute, runOptions) if err != nil { - klog.Fatalf("failed to create runtime manager: %v", err) + log.Fatalw("Failed to create runtime manager", zap.Error(err)) } if err := mgr.Start(ctx); err != nil { - klog.Errorf("failed to start kubebuilder manager: %v", err) + log.Errorw("Failed to start manager", zap.Error(err)) } } @@ -323,14 +342,14 @@ func createManager(syncPeriod time.Duration, options controllerRunOptions) (mana MetricsBindAddress: metricsAddress, }) if err != nil { - return nil, fmt.Errorf("error building ctrlruntime manager: %w", err) + return nil, fmt.Errorf("failed to build ctrlruntime manager: %w", err) } if err := mgr.AddReadyzCheck("alive", healthz.Ping); err != nil { return nil, fmt.Errorf("failed to add readiness check: %w", err) } - if err := mgr.AddHealthzCheck("kubeconfig", health.KubeconfigAvailable(options.kubeconfigProvider)); err != nil { + if err := mgr.AddHealthzCheck("kubeconfig", health.KubeconfigAvailable(options.kubeconfigProvider, options.log)); err != nil { return nil, fmt.Errorf("failed to add health check: %w", err) } @@ -384,12 +403,12 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { } // Migrate MachinesV1Alpha1Machine to ClusterV1Alpha1Machine. - if err := migrations.MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary(ctx, client, bs.opt.kubeClient, providerData); err != nil { + if err := migrations.MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary(ctx, bs.opt.log, client, bs.opt.kubeClient, providerData); err != nil { return fmt.Errorf("migration to clusterv1alpha1 failed: %w", err) } // Migrate providerConfig field to providerSpec field. - if err := migrations.MigrateProviderConfigToProviderSpecIfNecessary(ctx, bs.opt.cfg, client); err != nil { + if err := migrations.MigrateProviderConfigToProviderSpecIfNecessary(ctx, bs.opt.log, bs.opt.cfg, client); err != nil { return fmt.Errorf("migration of providerConfig field to providerSpec field failed: %w", err) } @@ -398,6 +417,7 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { if err := machinecontroller.Add( ctx, + bs.opt.log, bs.mgr, bs.opt.kubeClient, workerCount, @@ -416,21 +436,21 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { return fmt.Errorf("failed to add Machine controller to manager: %w", err) } - if err := machinesetcontroller.Add(bs.mgr); err != nil { + if err := machinesetcontroller.Add(bs.mgr, bs.opt.log); err != nil { return fmt.Errorf("failed to add MachineSet controller to manager: %w", err) } - if err := machinedeploymentcontroller.Add(bs.mgr); err != nil { + if err := machinedeploymentcontroller.Add(bs.mgr, bs.opt.log); err != nil { return fmt.Errorf("failed to add MachineDeployment controller to manager: %w", err) } if bs.opt.nodeCSRApprover { - if err := nodecsrapprover.Add(bs.mgr); err != nil { + if err := nodecsrapprover.Add(bs.mgr, bs.opt.log); err != nil { return fmt.Errorf("failed to add NodeCSRApprover controller to manager: %w", err) } } - klog.Info("machine controller startup complete") + bs.opt.log.Info("Machine-controller startup complete") return nil } @@ -441,7 +461,7 @@ func parseClusterDNSIPs(s string) ([]net.IP, error) { for _, sip := range sips { ip := net.ParseIP(strings.TrimSpace(sip)) if ip == nil { - return nil, fmt.Errorf("unable to parse ip %s", sip) + return nil, fmt.Errorf("failed to parse IP %q", sip) } ips = append(ips, ip) } diff --git a/cmd/userdata/amzn2/main.go b/cmd/userdata/amzn2/main.go index 60f5afb24..2c317143b 100644 --- a/cmd/userdata/amzn2/main.go +++ b/cmd/userdata/amzn2/main.go @@ -22,25 +22,37 @@ package main import ( "flag" + "log" + "go.uber.org/zap" + + machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" "github.com/kubermatic/machine-controller/pkg/userdata/amzn2" userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" - - "k8s.io/klog" ) func main() { // Parse flags. var debug bool - flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging") + + logFlags := machinecontrollerlog.NewDefaultOptions() + logFlags.AddFlags(flag.CommandLine) + flag.Parse() + if err := logFlags.Validate(); err != nil { + log.Fatalf("Invalid options: %v", err) + } + + rawLog := machinecontrollerlog.New(logFlags.Debug, logFlags.Format) + log := rawLog.Sugar() + // Instantiate provider and start plugin. var provider = &amzn2.Provider{} var p = userdataplugin.New(provider, debug) - if err := p.Run(); err != nil { - klog.Fatalf("error running Amazon Linux 2 plugin: %v", err) + if err := p.Run(log); err != nil { + log.Fatalw("Failed to run Amazon Linux 2 plugin", zap.Error(err)) } } diff --git a/cmd/userdata/centos/main.go b/cmd/userdata/centos/main.go index 5ddd80f73..3369abaac 100644 --- a/cmd/userdata/centos/main.go +++ b/cmd/userdata/centos/main.go @@ -22,25 +22,37 @@ package main import ( "flag" + "log" + "go.uber.org/zap" + + machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" "github.com/kubermatic/machine-controller/pkg/userdata/centos" userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" - - "k8s.io/klog" ) func main() { // Parse flags. var debug bool - flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging") + + logFlags := machinecontrollerlog.NewDefaultOptions() + logFlags.AddFlags(flag.CommandLine) + flag.Parse() + if err := logFlags.Validate(); err != nil { + log.Fatalf("Invalid options: %v", err) + } + + rawLog := machinecontrollerlog.New(logFlags.Debug, logFlags.Format) + log := rawLog.Sugar() + // Instantiate provider and start plugin. var provider = ¢os.Provider{} var p = userdataplugin.New(provider, debug) - if err := p.Run(); err != nil { - klog.Fatalf("error running CentOS plugin: %v", err) + if err := p.Run(log); err != nil { + log.Fatalw("Failed to run CentOS plugin", zap.Error(err)) } } diff --git a/cmd/userdata/flatcar/main.go b/cmd/userdata/flatcar/main.go index 21b397fd2..e1bb145a6 100644 --- a/cmd/userdata/flatcar/main.go +++ b/cmd/userdata/flatcar/main.go @@ -22,24 +22,37 @@ package main import ( "flag" + "log" + "go.uber.org/zap" + + machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" "github.com/kubermatic/machine-controller/pkg/userdata/flatcar" userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" - - "k8s.io/klog" ) func main() { // Parse flags. var debug bool flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging") + + logFlags := machinecontrollerlog.NewDefaultOptions() + logFlags.AddFlags(flag.CommandLine) + flag.Parse() + if err := logFlags.Validate(); err != nil { + log.Fatalf("Invalid options: %v", err) + } + + rawLog := machinecontrollerlog.New(logFlags.Debug, logFlags.Format) + log := rawLog.Sugar() + // Instantiate provider and start plugin. var provider = &flatcar.Provider{} var p = userdataplugin.New(provider, debug) - if err := p.Run(); err != nil { - klog.Fatalf("error running flatcar plugin: %v", err) + if err := p.Run(log); err != nil { + log.Fatalw("Failed to run Flatcar plugin", zap.Error(err)) } } diff --git a/cmd/userdata/rhel/main.go b/cmd/userdata/rhel/main.go index ef14f008b..aaae79699 100644 --- a/cmd/userdata/rhel/main.go +++ b/cmd/userdata/rhel/main.go @@ -22,25 +22,37 @@ package main import ( "flag" + "log" + "go.uber.org/zap" + + machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" "github.com/kubermatic/machine-controller/pkg/userdata/rhel" - - "k8s.io/klog" ) func main() { // Parse flags. var debug bool - flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging") + + logFlags := machinecontrollerlog.NewDefaultOptions() + logFlags.AddFlags(flag.CommandLine) + flag.Parse() + if err := logFlags.Validate(); err != nil { + log.Fatalf("Invalid options: %v", err) + } + + rawLog := machinecontrollerlog.New(logFlags.Debug, logFlags.Format) + log := rawLog.Sugar() + // Instantiate provider and start plugin. var provider = &rhel.Provider{} var p = userdataplugin.New(provider, debug) - if err := p.Run(); err != nil { - klog.Fatalf("error running RHEL plugin: %v", err) + if err := p.Run(log); err != nil { + log.Fatalw("Failed to run RHEL plugin", zap.Error(err)) } } diff --git a/cmd/userdata/rockylinux/main.go b/cmd/userdata/rockylinux/main.go index 204d38eb2..5a81bea0e 100644 --- a/cmd/userdata/rockylinux/main.go +++ b/cmd/userdata/rockylinux/main.go @@ -22,25 +22,37 @@ package main import ( "flag" + "log" + "go.uber.org/zap" + + machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" "github.com/kubermatic/machine-controller/pkg/userdata/rockylinux" - - "k8s.io/klog" ) func main() { // Parse flags. var debug bool - flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging") + + logFlags := machinecontrollerlog.NewDefaultOptions() + logFlags.AddFlags(flag.CommandLine) + flag.Parse() + if err := logFlags.Validate(); err != nil { + log.Fatalf("Invalid options: %v", err) + } + + rawLog := machinecontrollerlog.New(logFlags.Debug, logFlags.Format) + log := rawLog.Sugar() + // Instantiate provider and start plugin. var provider = &rockylinux.Provider{} var p = userdataplugin.New(provider, debug) - if err := p.Run(); err != nil { - klog.Fatalf("error running RockyLinux plugin: %v", err) + if err := p.Run(log); err != nil { + log.Fatalw("Failed to run RockyLinux plugin", zap.Error(err)) } } diff --git a/cmd/userdata/ubuntu/main.go b/cmd/userdata/ubuntu/main.go index df8eb2b3a..67f1b4af0 100644 --- a/cmd/userdata/ubuntu/main.go +++ b/cmd/userdata/ubuntu/main.go @@ -22,25 +22,37 @@ package main import ( "flag" + "log" + "go.uber.org/zap" + + machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" "github.com/kubermatic/machine-controller/pkg/userdata/ubuntu" - - "k8s.io/klog" ) func main() { // Parse flags. var debug bool - flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging") + + logFlags := machinecontrollerlog.NewDefaultOptions() + logFlags.AddFlags(flag.CommandLine) + flag.Parse() + if err := logFlags.Validate(); err != nil { + log.Fatalf("Invalid options: %v", err) + } + + rawLog := machinecontrollerlog.New(logFlags.Debug, logFlags.Format) + log := rawLog.Sugar() + // Instantiate provider and start plugin. var provider = &ubuntu.Provider{} var p = userdataplugin.New(provider, debug) - if err := p.Run(); err != nil { - klog.Fatalf("error running Ubuntu plugin: %v", err) + if err := p.Run(log); err != nil { + log.Fatalw("Failed to run Ubuntu plugin", zap.Error(err)) } } diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index c3d4af796..f9c6d8239 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -18,17 +18,21 @@ package main import ( "flag" + "log" "github.com/Masterminds/semver/v3" + "github.com/go-logr/zapr" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/admission" "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" + machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" "github.com/kubermatic/machine-controller/pkg/node" userdatamanager "github.com/kubermatic/machine-controller/pkg/userdata/manager" "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + ctrlruntimelog "sigs.k8s.io/controller-runtime/pkg/log" ) type options struct { @@ -47,9 +51,11 @@ type options struct { func main() { nodeFlags := node.NewFlags(flag.CommandLine) + logFlags := machinecontrollerlog.NewDefaultOptions() + logFlags.AddFlags(flag.CommandLine) + opt := &options{} - klog.InitFlags(nil) if flag.Lookup("kubeconfig") == nil { flag.StringVar(&opt.kubeconfig, "kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.") } @@ -69,28 +75,39 @@ func main() { flag.BoolVar(&opt.useExternalBootstrap, "use-external-bootstrap", false, "user-data is provided by external bootstrap mechanism (e.g. operating-system-manager, also known as OSM)") flag.Parse() + + if err := logFlags.Validate(); err != nil { + log.Fatalf("Invalid options: %v", err) + } + + rawLog := machinecontrollerlog.New(logFlags.Debug, logFlags.Format) + log := rawLog.Sugar() + + // set the logger used by controller-runtime + ctrlruntimelog.SetLogger(zapr.NewLogger(rawLog.WithOptions(zap.AddCallerSkip(1)))) + opt.kubeconfig = flag.Lookup("kubeconfig").Value.(flag.Getter).Get().(string) opt.masterURL = flag.Lookup("master").Value.(flag.Getter).Get().(string) if opt.caBundleFile != "" { if err := util.SetCABundleFile(opt.caBundleFile); err != nil { - klog.Fatalf("-ca-bundle is invalid: %v", err) + log.Fatalw("-ca-bundle is invalid", zap.Error(err)) } } cfg, err := clientcmd.BuildConfigFromFlags(opt.masterURL, opt.kubeconfig) if err != nil { - klog.Fatalf("error building kubeconfig: %v", err) + log.Fatalw("Failed to build kubeconfig", zap.Error(err)) } client, err := ctrlruntimeclient.New(cfg, ctrlruntimeclient.Options{}) if err != nil { - klog.Fatalf("failed to build client: %v", err) + log.Fatalw("Failed to build client", zap.Error(err)) } constraint, err := semver.NewConstraint(opt.versionConstraint) if err != nil { - klog.Fatalf("failed to validate kubernetes-version-constraints: %v", err) + log.Fatalw("Failed to validate kubernetes-version-constraints", zap.Error(err)) } // Start with assuming that current cluster will be used as worker cluster @@ -101,23 +118,24 @@ func main() { &clientcmd.ClientConfigLoadingRules{ExplicitPath: opt.workerClusterKubeconfig}, &clientcmd.ConfigOverrides{}).ClientConfig() if err != nil { - klog.Fatal(err) + log.Fatalw("Failed to create worker cluster config", zap.Error(err)) } // Build dedicated client for worker cluster workerClient, err = ctrlruntimeclient.New(workerClusterConfig, ctrlruntimeclient.Options{}) if err != nil { - klog.Fatalf("failed to build worker client: %v", err) + log.Fatalw("Failed to build worker client", zap.Error(err)) } } - um, err := userdatamanager.New() + um, err := userdatamanager.New(log) if err != nil { - klog.Fatalf("error initialising userdata plugins: %v", err) + log.Fatalw("Failed to initialise userdata plugins", zap.Error(err)) } srv, err := admission.Builder{ ListenAddress: opt.admissionListenAddress, + Log: log, Client: client, WorkerClient: workerClient, UserdataManager: um, @@ -127,17 +145,17 @@ func main() { VersionConstraints: constraint, }.Build() if err != nil { - klog.Fatalf("failed to create admission hook: %v", err) + log.Fatalw("Failed to create admission hook", zap.Error(err)) } + log.Infow("Listening", "address", opt.admissionListenAddress) + if err := srv.ListenAndServeTLS(opt.admissionTLSCertPath, opt.admissionTLSKeyPath); err != nil { - klog.Fatalf("Failed to start server: %v", err) + log.Fatalw("Failed to start server", zap.Error(err)) } defer func() { if err := srv.Close(); err != nil { - klog.Fatalf("Failed to shutdown server: %v", err) + log.Fatalw("Failed to shutdown server", zap.Error(err)) } }() - klog.Infof("Listening on %s", opt.admissionListenAddress) - select {} } diff --git a/examples/machine-controller.yaml b/examples/machine-controller.yaml index a06dc9c2f..957deca48 100644 --- a/examples/machine-controller.yaml +++ b/examples/machine-controller.yaml @@ -211,8 +211,8 @@ spec: name: machine-controller command: - /usr/local/bin/machine-controller - - -logtostderr - - -v=3 + - -log-debug=false + - -log-format=json # json or console - -worker-count=5 - -node-csr-approver=true - -cluster-dns=10.10.10.10 @@ -259,9 +259,9 @@ spec: name: webhook command: - /usr/local/bin/webhook - - -logtostderr - # Starting with v=6, full Machine objects with inline credentials are logged, beware! - - -v=4 + # on debug level, full Machine objects with inline credentials might be logged, beware! + - -log-debug=false + - -log-format=json # json or console - -use-osm=true - -namespace=kube-system - -listen-address=0.0.0.0:9876 diff --git a/go.mod b/go.mod index 076a82b7a..ee1bae4c6 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,8 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/digitalocean/godo v1.93.0 github.com/ghodss/yaml v1.0.0 + github.com/go-logr/logr v1.2.3 + github.com/go-logr/zapr v1.2.3 github.com/go-test/deep v1.0.8 github.com/google/uuid v1.3.0 github.com/gophercloud/gophercloud v1.1.1 @@ -37,11 +39,13 @@ require ( github.com/prometheus/client_golang v1.14.0 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10 github.com/sethvargo/go-password v0.2.0 + github.com/spf13/pflag v1.0.5 github.com/tinkerbell/tink v0.8.0 github.com/vmware/go-vcloud-director/v2 v2.19.0 github.com/vmware/govmomi v0.30.0 github.com/vultr/govultr/v2 v2.17.2 go.anx.io/go-anxcloud v0.5.0 + go.uber.org/zap v1.24.0 golang.org/x/crypto v0.4.0 golang.org/x/oauth2 v0.3.0 gomodules.xyz/jsonpatch/v2 v2.2.0 @@ -101,9 +105,7 @@ require ( github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-logr/zapr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/swag v0.22.3 // indirect @@ -152,7 +154,6 @@ require ( github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/cobra v1.6.1 // indirect - github.com/spf13/pflag v1.0.5 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 // indirect @@ -161,7 +162,6 @@ require ( go.opentelemetry.io/otel/trace v1.11.2 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect - go.uber.org/zap v1.24.0 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/net v0.7.0 // indirect golang.org/x/sync v0.1.0 // indirect diff --git a/hack/ci/setup-machine-controller-in-kind.sh b/hack/ci/setup-machine-controller-in-kind.sh index 4778e7654..011644a4d 100755 --- a/hack/ci/setup-machine-controller-in-kind.sh +++ b/hack/ci/setup-machine-controller-in-kind.sh @@ -54,6 +54,9 @@ if [ ! -f machine-controller-deployed ]; then sed -i "s;-use-osm=true;-use-osm=false;g" examples/machine-controller.yaml fi + # e2e tests logs are primarily read by humans, if ever + sed -i 's/log-format=json/log-format=console/g' examples/machine-controller.yaml + make deploy touch machine-controller-deployed diff --git a/hack/run-machine-controller.sh b/hack/run-machine-controller.sh index 7718af663..a21eddc05 100755 --- a/hack/run-machine-controller.sh +++ b/hack/run-machine-controller.sh @@ -28,7 +28,7 @@ $(dirname $0)/../machine-controller \ -kubeconfig=$MC_KUBECONFIG \ -worker-count=50 \ -logtostderr \ - -v=6 \ + -log-debug \ -cluster-dns=169.254.20.10 \ -enable-profiling \ -metrics-address=0.0.0.0:8080 \ diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go index beaf4ed90..d972fd9af 100644 --- a/pkg/admission/admission.go +++ b/pkg/admission/admission.go @@ -27,6 +27,7 @@ import ( "time" "github.com/Masterminds/semver/v3" + "go.uber.org/zap" "gomodules.xyz/jsonpatch/v2" machinecontroller "github.com/kubermatic/machine-controller/pkg/controller/machine" @@ -37,11 +38,11 @@ import ( apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/klog" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) type admissionData struct { + log *zap.SugaredLogger client ctrlruntimeclient.Client workerClient ctrlruntimeclient.Client userDataManager *userdatamanager.Manager @@ -55,6 +56,7 @@ var jsonPatch = admissionv1.PatchTypeJSONPatch type Builder struct { ListenAddress string + Log *zap.SugaredLogger Client ctrlruntimeclient.Client WorkerClient ctrlruntimeclient.Client UserdataManager *userdatamanager.Manager @@ -67,6 +69,7 @@ type Builder struct { func (build Builder) Build() (*http.Server, error) { mux := http.NewServeMux() ad := &admissionData{ + log: build.Log, client: build.Client, workerClient: build.WorkerClient, userDataManager: build.UserdataManager, @@ -79,8 +82,8 @@ func (build Builder) Build() (*http.Server, error) { return nil, fmt.Errorf("error updating nodeSettings, %w", err) } - mux.HandleFunc("/machinedeployments", handleFuncFactory(ad.mutateMachineDeployments)) - mux.HandleFunc("/machines", handleFuncFactory(ad.mutateMachines)) + mux.HandleFunc("/machinedeployments", handleFuncFactory(build.Log, ad.mutateMachineDeployments)) + mux.HandleFunc("/machines", handleFuncFactory(build.Log, ad.mutateMachines)) mux.HandleFunc("/healthz", healthZHandler) return &http.Server{ @@ -103,16 +106,14 @@ func newJSONPatch(original, current runtime.Object) ([]jsonpatch.JsonPatchOperat if err != nil { return nil, err } - klog.V(6).Infof("jsonpatch: Marshaled original: %s", string(ori)) cur, err := json.Marshal(current) if err != nil { return nil, err } - klog.V(6).Infof("jsonpatch: Marshaled target: %s", string(cur)) return jsonpatch.CreatePatch(ori, cur) } -func createAdmissionResponse(original, mutated runtime.Object) (*admissionv1.AdmissionResponse, error) { +func createAdmissionResponse(log *zap.SugaredLogger, original, mutated runtime.Object) (*admissionv1.AdmissionResponse, error) { response := &admissionv1.AdmissionResponse{} response.Allowed = true if !apiequality.Semantic.DeepEqual(original, mutated) { @@ -125,7 +126,7 @@ func createAdmissionResponse(original, mutated runtime.Object) (*admissionv1.Adm if err != nil { return nil, fmt.Errorf("failed to marshal json patch: %w", err) } - klog.V(6).Infof("Produced jsonpatch: %s", string(patchRaw)) + log.Debugw("Produced jsonpatch", "patch", string(patchRaw)) response.Patch = patchRaw response.PatchType = &jsonPatch @@ -135,17 +136,17 @@ func createAdmissionResponse(original, mutated runtime.Object) (*admissionv1.Adm type mutator func(context.Context, admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error) -func handleFuncFactory(mutate mutator) func(http.ResponseWriter, *http.Request) { +func handleFuncFactory(log *zap.SugaredLogger, mutate mutator) func(http.ResponseWriter, *http.Request) { return func(w http.ResponseWriter, r *http.Request) { review, err := readReview(r) if err != nil { - klog.Warningf("invalid admission review: %v", err) + log.Errorw("Invalid admission review", zap.Error(err)) // proper AdmissionReview responses require metadata that is not available // in broken requests, so we return a basic failure response w.WriteHeader(http.StatusBadRequest) if _, err := w.Write([]byte(fmt.Sprintf("invalid request: %v", err))); err != nil { - klog.Errorf("failed to write badRequest: %v", err) + log.Errorw("Failed to write badRequest", zap.Error(err)) } return } @@ -166,12 +167,12 @@ func handleFuncFactory(mutate mutator) func(http.ResponseWriter, *http.Request) Response: response, }) if err != nil { - klog.Errorf("failed to marshal admissionResponse: %v", err) + log.Errorw("Failed to marshal admissionResponse", zap.Error(err)) return } if _, err := w.Write(resp); err != nil { - klog.Errorf("failed to write admissionResponse: %v", err) + log.Errorw("Failed to write admissionResponse", zap.Error(err)) } } } diff --git a/pkg/admission/machinedeployments.go b/pkg/admission/machinedeployments.go index a20fa5f53..7b0b0585a 100644 --- a/pkg/admission/machinedeployments.go +++ b/pkg/admission/machinedeployments.go @@ -25,6 +25,7 @@ import ( admissionv1 "k8s.io/api/admission/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) func (ad *admissionData) mutateMachineDeployments(ctx context.Context, ar admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error) { @@ -34,6 +35,9 @@ func (ad *admissionData) mutateMachineDeployments(ctx context.Context, ar admiss } machineDeploymentOriginal := machineDeployment.DeepCopy() + log := ad.log.With("machinedeployment", ctrlruntimeclient.ObjectKeyFromObject(&machineDeployment)) + log.Debug("Defaulting and validating machine deployment") + machineDeploymentDefaultingFunction(&machineDeployment) if err := mutationsForMachineDeployment(&machineDeployment); err != nil { @@ -62,5 +66,5 @@ func (ad *admissionData) mutateMachineDeployments(ctx context.Context, ar admiss } } - return createAdmissionResponse(machineDeploymentOriginal, &machineDeployment) + return createAdmissionResponse(log, machineDeploymentOriginal, &machineDeployment) } diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index 4d7df978f..d96b9e977 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -33,7 +33,7 @@ import ( admissionv1 "k8s.io/api/admission/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/klog" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) // BypassSpecNoModificationRequirementAnnotation is used to bypass the "no machine.spec modification" allowed @@ -48,7 +48,8 @@ func (ad *admissionData) mutateMachines(ctx context.Context, ar admissionv1.Admi } machineOriginal := machine.DeepCopy() - klog.V(3).Infof("Defaulting and validating machine %s/%s", machine.Namespace, machine.Name) + log := ad.log.With("machine", ctrlruntimeclient.ObjectKeyFromObject(&machine)) + log.Debug("Defaulting and validating machine") // Mutating .Spec is never allowed // Only hidden exception: the machine-controller may set the .Spec.Name to .Metadata.Name @@ -109,7 +110,7 @@ func (ad *admissionData) mutateMachines(ctx context.Context, ar admissionv1.Admi machine.Labels[controllerutil.LegacyMachineControllerUserDataLabel] = "true" } - return createAdmissionResponse(machineOriginal, &machine) + return createAdmissionResponse(log, machineOriginal, &machine) } func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec *clusterv1alpha1.MachineSpec) error { @@ -177,13 +178,13 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec return fmt.Errorf("failed to json marshal machine.spec.providerSpec: %w", err) } - defaultedSpec, err := prov.AddDefaults(*spec) + defaultedSpec, err := prov.AddDefaults(ad.log, *spec) if err != nil { return fmt.Errorf("failed to default machineSpec: %w", err) } spec = &defaultedSpec - if err := prov.Validate(ctx, *spec); err != nil { + if err := prov.Validate(ctx, ad.log, *spec); err != nil { return fmt.Errorf("validation failed: %w", err) } diff --git a/pkg/apis/cluster/common/plugins.go b/pkg/apis/cluster/common/plugins.go index a611863a1..9439a5e53 100644 --- a/pkg/apis/cluster/common/plugins.go +++ b/pkg/apis/cluster/common/plugins.go @@ -17,11 +17,10 @@ limitations under the License. package common import ( + "fmt" "sync" "github.com/pkg/errors" - - "k8s.io/klog" ) var ( @@ -35,9 +34,8 @@ func RegisterClusterProvisioner(name string, provisioner interface{}) { providersMutex.Lock() defer providersMutex.Unlock() if _, found := providers[name]; found { - klog.Fatalf("Cluster provisioner %q was registered twice", name) + panic(fmt.Sprintf("Cluster provisioner %q was registered twice", name)) } - klog.V(1).Infof("Registered cluster provisioner %q", name) providers[name] = provisioner } @@ -46,7 +44,7 @@ func ClusterProvisioner(name string) (interface{}, error) { defer providersMutex.Unlock() provisioner, found := providers[name] if !found { - return nil, errors.Errorf("unable to find provisioner for %s", name) + return nil, errors.Errorf("failed to find provisioner for %s", name) } return provisioner, nil } diff --git a/pkg/apis/cluster/v1alpha1/migrations/migrations.go b/pkg/apis/cluster/v1alpha1/migrations/migrations.go index 284bfa18a..84cab3a91 100644 --- a/pkg/apis/cluster/v1alpha1/migrations/migrations.go +++ b/pkg/apis/cluster/v1alpha1/migrations/migrations.go @@ -22,6 +22,8 @@ import ( "fmt" "time" + "go.uber.org/zap" + machinecontrolleradmission "github.com/kubermatic/machine-controller/pkg/admission" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1/conversions" @@ -46,13 +48,12 @@ import ( "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/util/retry" - "k8s.io/klog" "sigs.k8s.io/controller-runtime/pkg/cache" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) -func MigrateProviderConfigToProviderSpecIfNecessary(ctx context.Context, config *restclient.Config, client ctrlruntimeclient.Client) error { - klog.Infof("Starting to migrate providerConfigs to providerSpecs") +func MigrateProviderConfigToProviderSpecIfNecessary(ctx context.Context, log *zap.SugaredLogger, config *restclient.Config, client ctrlruntimeclient.Client) error { + log.Info("Starting to migrate providerConfigs to providerSpecs") dynamicClient, err := dynamicclient.NewForConfig(config) if err != nil { return fmt.Errorf("failed to construct dynamic client: %w", err) @@ -67,6 +68,8 @@ func MigrateProviderConfigToProviderSpecIfNecessary(ctx context.Context, config return fmt.Errorf("failed to list machine objects: %w", err) } for _, machine := range machines.Items { + machineLog := log.With("machine", ctrlruntimeclient.ObjectKeyFromObject(&machine)) + marshalledObject, err := machine.MarshalJSON() if err != nil { return fmt.Errorf("failed to marshal unstructured machine %s: %w", machine.GetName(), err) @@ -76,7 +79,7 @@ func MigrateProviderConfigToProviderSpecIfNecessary(ctx context.Context, config return fmt.Errorf("failed to convert machine: %w", err) } if wasConverted { - klog.Infof("Converted providerConfig -> providerSpec for machine %s/%s, attempting to update", convertedMachine.Namespace, convertedMachine.Name) + machineLog.Info("Converted providerConfig -> providerSpec, attempting to update") if convertedMachine.Annotations == nil { convertedMachine.Annotations = map[string]string{} } @@ -86,7 +89,7 @@ func MigrateProviderConfigToProviderSpecIfNecessary(ctx context.Context, config if err := client.Update(ctx, convertedMachine); err != nil { return fmt.Errorf("failed to update converted machine %s: %w", convertedMachine.Name, err) } - klog.Infof("Successfully updated machine %s/%s after converting providerConfig -> providerSpec", convertedMachine.Namespace, convertedMachine.Name) + machineLog.Info("Successfully updated machine after converting providerConfig -> providerSpec") } } @@ -95,6 +98,8 @@ func MigrateProviderConfigToProviderSpecIfNecessary(ctx context.Context, config return fmt.Errorf("failed to list MachineSets: %w", err) } for _, machineSet := range machineSets.Items { + machineSetLog := log.With("machineset", ctrlruntimeclient.ObjectKeyFromObject(&machineSet)) + marshalledObject, err := machineSet.MarshalJSON() if err != nil { return fmt.Errorf("failed to marshal unstructured MachineSet %s: %w", machineSet.GetName(), err) @@ -104,11 +109,11 @@ func MigrateProviderConfigToProviderSpecIfNecessary(ctx context.Context, config return fmt.Errorf("failed to convert MachineSet %s/%s: %w", machineSet.GetNamespace(), machineSet.GetName(), err) } if machineSetWasConverted { - klog.Infof("Converted providerConfig -> providerSpec for MachineSet %s/%s, attempting to update", convertedMachineSet.Namespace, convertedMachineSet.Name) + machineSetLog.Info("Converted providerConfig -> providerSpec, attempting to update") if err := client.Update(ctx, convertedMachineSet); err != nil { return fmt.Errorf("failed to update MachineSet %s/%s after converting providerConfig -> providerSpec: %w", convertedMachineSet.Namespace, convertedMachineSet.Name, err) } - klog.Infof("Successfully updated MachineSet %s/%s after converting providerConfig -> providerSpec", convertedMachineSet.Namespace, convertedMachineSet.Name) + machineSetLog.Info("Successfully updated MachineSet after converting providerConfig -> providerSpec") } } @@ -117,6 +122,8 @@ func MigrateProviderConfigToProviderSpecIfNecessary(ctx context.Context, config return fmt.Errorf("failed to list MachineDeplyoments: %w", err) } for _, machineDeployment := range machineDeployments.Items { + machineDeploymentLog := log.With("machinedeployment", ctrlruntimeclient.ObjectKeyFromObject(&machineDeployment)) + marshalledObject, err := machineDeployment.MarshalJSON() if err != nil { return fmt.Errorf("failed to marshal unstructured MachineDeployment %s: %w", machineDeployment.GetName(), err) @@ -126,20 +133,21 @@ func MigrateProviderConfigToProviderSpecIfNecessary(ctx context.Context, config return fmt.Errorf("failed to convert MachineDeployment %s/%s: %w", machineDeployment.GetNamespace(), machineDeployment.GetName(), err) } if machineDeploymentWasConverted { - klog.Infof("Converted providerConfig -> providerSpec for MachineDeployment %s/%s, attempting to update", convertedMachineDeployment.Namespace, convertedMachineDeployment.Name) + machineDeploymentLog.Info("Converted providerConfig -> providerSpec, attempting to update") if err := client.Update(ctx, convertedMachineDeployment); err != nil { return fmt.Errorf("failed to update MachineDeployment %s/%s after converting providerConfig -> providerSpec: %w", convertedMachineDeployment.Namespace, convertedMachineDeployment.Name, err) } - klog.Infof("Successfully updated MachineDeployment %s/%s after converting providerConfig -> providerSpec", convertedMachineDeployment.Namespace, convertedMachineDeployment.Name) + machineDeploymentLog.Info("Successfully updated MachineDeployment after converting providerConfig -> providerSpec") } } - klog.Infof("Successfully migrated providerConfigs to providerSpecs") + log.Info("Successfully migrated providerConfigs to providerSpecs") return nil } func MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary( - ctx context.Context, client ctrlruntimeclient.Client, + ctx context.Context, log *zap.SugaredLogger, + client ctrlruntimeclient.Client, kubeClient kubernetes.Interface, providerData *cloudprovidertypes.ProviderData) error { var ( @@ -148,6 +156,8 @@ func MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary( noMigrationNeed = false ) + crdLog := log.With("crd", machines.CRDName) + err := wait.Poll(cachePopulatingInterval, cachePopulatingTimeout, func() (done bool, err error) { err = client.Get(ctx, types.NamespacedName{Name: machines.CRDName}, &apiextensionsv1.CustomResourceDefinition{}) if err != nil { @@ -158,7 +168,7 @@ func MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary( var cerr *cache.ErrCacheNotStarted if errors.As(err, &cerr) { - klog.Info("Cache hasn't started yet, trying in 5 seconds") + log.Info("Cache hasn't started yet, trying in 5 seconds") return false, nil } @@ -168,12 +178,12 @@ func MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary( }) if err != nil { - klog.Errorf("Failed waiting for caches to be populated: %v", err) + crdLog.Errorw("Failed waiting for caches to be populated", zap.Error(err)) return err } if noMigrationNeed { - klog.Infof("CRD %s not present, no migration needed", machines.CRDName) + crdLog.Info("CRD not present, no migration needed") return nil } @@ -182,34 +192,36 @@ func MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary( return fmt.Errorf("error when checking for existence of 'machines.cluster.k8s.io' crd: %w", err) } - if err := migrateMachines(ctx, client, kubeClient, providerData); err != nil { + if err := migrateMachines(ctx, log, client, kubeClient, providerData); err != nil { return fmt.Errorf("failed to migrate machines: %w", err) } - klog.Infof("Attempting to delete CRD %s", machines.CRDName) + crdLog.Info("Attempting to delete CRD") if err := client.Delete(ctx, &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: machines.CRDName}}); err != nil { return fmt.Errorf("failed to delete machinesv1alpha1.machine crd: %w", err) } - klog.Infof("Successfully deleted CRD %s", machines.CRDName) + crdLog.Info("Successfully deleted CRD") return nil } -func migrateMachines(ctx context.Context, client ctrlruntimeclient.Client, kubeClient kubernetes.Interface, providerData *cloudprovidertypes.ProviderData) error { - klog.Infof("Starting migration for machine.machines.k8s.io/v1alpha1 to machine.cluster.k8s.io/v1alpha1") +func migrateMachines(ctx context.Context, log *zap.SugaredLogger, client ctrlruntimeclient.Client, kubeClient kubernetes.Interface, providerData *cloudprovidertypes.ProviderData) error { + log.Info("Starting migration for machine.machines.k8s.io/v1alpha1 to machine.cluster.k8s.io/v1alpha1") // Get machinesv1Alpha1Machines - klog.Infof("Getting existing machine.machines.k8s.io/v1alpha1 to migrate") + log.Info("Getting existing machine.machines.k8s.io/v1alpha1 to migrate") machinesv1Alpha1Machines := &machinesv1alpha1.MachineList{} if err := client.List(ctx, machinesv1Alpha1Machines); err != nil { return fmt.Errorf("failed to list machinesV1Alpha1 machines: %w", err) } - klog.Infof("Found %v machine.machines.k8s.io/v1alpha1", len(machinesv1Alpha1Machines.Items)) + log.Infof("Found %d machine.machines.k8s.io/v1alpha1 resources", len(machinesv1Alpha1Machines.Items)) // Convert the machine, create the new machine, delete the old one, wait for it to be absent // We do this in one loop to avoid ending up having all machines in both the new and the old format if deletion // fails for whatever reason for _, machinesV1Alpha1Machine := range machinesv1Alpha1Machines.Items { - klog.Infof("Starting migration for machine.machines.k8s.io/v1alpha1 %s", machinesV1Alpha1Machine.Name) + machineLog := log.With("machine", machinesV1Alpha1Machine.Name) + machineLog.Info("Starting migration") + convertedClusterv1alpha1Machine := &clusterv1alpha1.Machine{} err := conversions.Convert_MachinesV1alpha1Machine_To_ClusterV1alpha1Machine(&machinesV1Alpha1Machine, convertedClusterv1alpha1Machine) @@ -231,6 +243,8 @@ func migrateMachines(ctx context.Context, client ctrlruntimeclient.Client, kubeC return fmt.Errorf("failed to get cloud provider %q: %w", providerConfig.CloudProvider, err) } + machineLog = machineLog.With("provider", providerConfig.CloudProvider) + // We will set that to what's finally in the apisever, be that a created a clusterv1alpha1machine // or a preexisting one, because the migration got interrupted // It is required to set the ownerRef of the node @@ -239,8 +253,8 @@ func migrateMachines(ctx context.Context, client ctrlruntimeclient.Client, kubeC // Do a get first to cover the case the new machine was already created but then something went wrong // If that is the case and the clusterv1alpha1machine != machinesv1alpha1machine we error out and the operator // has to manually delete either the new or the old machine - klog.Infof("Checking if machine.cluster.k8s.io/v1alpha1 %s/%s already exists", - convertedClusterv1alpha1Machine.Namespace, convertedClusterv1alpha1Machine.Name) + machineLog = machineLog.With("converted", ctrlruntimeclient.ObjectKeyFromObject(convertedClusterv1alpha1Machine)) + machineLog.Info("Checking if converted machine already exists") existingClusterV1alpha1Machine := &clusterv1alpha1.Machine{} err = client.Get(ctx, @@ -253,14 +267,12 @@ func migrateMachines(ctx context.Context, client ctrlruntimeclient.Client, kubeC } // ClusterV1alpha1Machine does not exist yet - klog.Infof("Machine.cluster.k8s.io/v1alpha1 %s/%s does not yet exist, attempting to create it", - convertedClusterv1alpha1Machine.Namespace, convertedClusterv1alpha1Machine.Name) + machineLog.Info("Converted machine does not yet exist, attempting to create it") if err := client.Create(ctx, convertedClusterv1alpha1Machine); err != nil { return fmt.Errorf("failed to create clusterv1alpha1.machine %s: %w", convertedClusterv1alpha1Machine.Name, err) } - klog.Infof("Successfully created machine.cluster.k8s.io/v1alpha1 %s/%s", - convertedClusterv1alpha1Machine.Namespace, convertedClusterv1alpha1Machine.Name) + machineLog.Info("Successfully created converted machine") finalClusterV1Alpha1Machine = convertedClusterv1alpha1Machine } else { // ClusterV1alpha1Machine already exists @@ -272,34 +284,32 @@ func migrateMachines(ctx context.Context, client ctrlruntimeclient.Client, kubeC existingClusterV1alpha1Machine.Annotations = convertedClusterv1alpha1Machine.Annotations existingClusterV1alpha1Machine.Finalizers = convertedClusterv1alpha1Machine.Finalizers - klog.Infof("Updating existing machine.cluster.k8s.io/v1alpha1 %s/%s", - existingClusterV1alpha1Machine.Namespace, existingClusterV1alpha1Machine.Name) + machineLog.Info("Updating existing converted machine") if err := client.Update(ctx, existingClusterV1alpha1Machine); err != nil { return fmt.Errorf("failed to update metadata of existing clusterV1Alpha1 machine: %w", err) } - klog.Infof("Successfully updated existing machine.cluster.k8s.io/v1alpha1 %s/%s", - existingClusterV1alpha1Machine.Namespace, existingClusterV1alpha1Machine.Name) + machineLog.Info("Successfully updated existing converted machine") finalClusterV1Alpha1Machine = existingClusterV1alpha1Machine } // We have to ensure there is an ownerRef to our clusterv1alpha1.Machine on the node if it exists // and that there is no ownerRef to the old machine anymore - if err := ensureClusterV1Alpha1NodeOwnership(ctx, finalClusterV1Alpha1Machine, client); err != nil { + if err := ensureClusterV1Alpha1NodeOwnership(ctx, machineLog, finalClusterV1Alpha1Machine, client); err != nil { return err } if sets.NewString(finalClusterV1Alpha1Machine.Finalizers...).Has(machinecontroller.FinalizerDeleteInstance) { - klog.Infof("Attempting to update the UID at the cloud provider for machine.cluster.k8s.io/v1alpha1 %s", machinesV1Alpha1Machine.Name) + machineLog.Info("Attempting to update the UID at the cloud provider") newMachineWithOldUID := finalClusterV1Alpha1Machine.DeepCopy() newMachineWithOldUID.UID = machinesV1Alpha1Machine.UID - if err := prov.MigrateUID(ctx, newMachineWithOldUID, finalClusterV1Alpha1Machine.UID); err != nil { + if err := prov.MigrateUID(ctx, machineLog, newMachineWithOldUID, finalClusterV1Alpha1Machine.UID); err != nil { return fmt.Errorf("running the provider migration for the UID failed: %w", err) } // Block until we can actually GET the instance with the new UID var isMigrated bool for i := 0; i < 100; i++ { - if _, err := prov.Get(ctx, finalClusterV1Alpha1Machine, providerData); err == nil { + if _, err := prov.Get(ctx, machineLog, finalClusterV1Alpha1Machine, providerData); err == nil { isMigrated = true break } @@ -308,28 +318,27 @@ func migrateMachines(ctx context.Context, client ctrlruntimeclient.Client, kubeC if !isMigrated { return fmt.Errorf("failed to GET instance for machine %s after UID migration", finalClusterV1Alpha1Machine.Name) } - klog.Infof("Successfully updated the UID at the cloud provider for machine.cluster.k8s.io/v1alpha1 %s", machinesV1Alpha1Machine.Name) + machineLog.Info("Successfully updated the UID at the cloud provider") } // All went fine, we only have to clear the old machine now - klog.Infof("Deleting machine.machines.k8s.io/v1alpha1 %s", machinesV1Alpha1Machine.Name) + machineLog.Info("Deleting old machine object") if err := deleteMachinesV1Alpha1Machine(ctx, &machinesV1Alpha1Machine, client); err != nil { return err } - klog.Infof("Successfully deleted machine.machines.k8s.io/v1alpha1 %s", machinesV1Alpha1Machine.Name) - klog.Infof("Successfully finished migration for machine.machines.k8s.io/v1alpha1 %s", machinesV1Alpha1Machine.Name) + machineLog.Info("Successfully deleted old machine object") + machineLog.Info("Successfully finished migration") } - klog.Infof("Successfully finished migration for machine.machines.k8s.io/v1alpha1 to machine.cluster.k8s.io/v1alpha1") + log.Info("Successfully finished migration for machine.machines.k8s.io/v1alpha1 to machine.cluster.k8s.io/v1alpha1") return nil } -func ensureClusterV1Alpha1NodeOwnership(ctx context.Context, machine *clusterv1alpha1.Machine, client ctrlruntimeclient.Client) error { +func ensureClusterV1Alpha1NodeOwnership(ctx context.Context, machineLog *zap.SugaredLogger, machine *clusterv1alpha1.Machine, client ctrlruntimeclient.Client) error { if machine.Spec.Name == "" { machine.Spec.Name = machine.Name } - klog.Infof("Checking if node for machines.cluster.k8s.io/v1alpha1 %s/%s exists", - machine.Namespace, machine.Name) + machineLog.Info("Checking if node for machines exists") nodeNameCandidates := []string{machine.Spec.Name} if machine.Status.NodeRef != nil { if machine.Status.NodeRef.Name != machine.Spec.Name { @@ -341,16 +350,16 @@ func ensureClusterV1Alpha1NodeOwnership(ctx context.Context, machine *clusterv1a node := &corev1.Node{} if err := client.Get(ctx, types.NamespacedName{Name: nodeName}, node); err != nil { if kerrors.IsNotFound(err) { - klog.Infof("No node for machines.cluster.k8s.io/v1alpha1 %s/%s found", - machine.Namespace, machine.Name) + machineLog.Info("No node for machines found") continue } return fmt.Errorf("Failed to get node %s for machine %s: %w", machine.Spec.Name, machine.Name, err) } - klog.Infof("Found node for machines.cluster.k8s.io/v1alpha1 %s/%s: %s, removing its ownerRef and adding NodeOwnerLabel", - node.Name, machine.Namespace, machine.Name) + nodeLog := machineLog.With("node", node.Name) + nodeLog.Info("Found node for machine, removing its ownerRef and adding NodeOwnerLabel") + nodeLabels := node.Labels nodeLabels[machinecontroller.NodeOwnerLabelName] = string(machine.UID) // We retry this because nodes get frequently updated so there is a reasonable chance this may fail @@ -365,8 +374,7 @@ func ensureClusterV1Alpha1NodeOwnership(ctx context.Context, machine *clusterv1a }); err != nil { return fmt.Errorf("failed to update OwnerLabel on node %s: %w", node.Name, err) } - klog.Infof("Successfully removed ownerRef and added NodeOwnerLabelName to node %s for machines.cluster.k8s.io/v1alpha1 %s/%s", - node.Name, machine.Namespace, machine.Name) + nodeLog.Info("Successfully removed ownerRef and added NodeOwnerLabelName to node") } return nil diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index f70d42c45..54ff507ca 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -25,6 +25,7 @@ import ( "github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests" "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -109,11 +110,11 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes return &provider{configVarResolver: configVarResolver} } -func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } -func (p *provider) Validate(_ context.Context, machineSpec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, machineSpec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(machineSpec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -154,7 +155,7 @@ func (p *provider) Validate(_ context.Context, machineSpec clusterv1alpha1.Machi return nil } -func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -203,7 +204,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri return "", "", nil } -func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -263,8 +264,8 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d return &alibabaInstance{instance: foundInstance}, nil } -func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - foundInstance, err := p.Get(ctx, machine, data) +func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { + foundInstance, err := p.Get(ctx, log, machine, data) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return util.RemoveFinalizerOnInstanceNotFound(finalizerInstance, machine, data) @@ -308,7 +309,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } -func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to decode providerconfig: %w", err) diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 47aa46a83..6b51e714a 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -36,6 +36,7 @@ import ( "go.anx.io/go-anxcloud/pkg/vsphere" "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/progress" anxvm "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -51,7 +52,6 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" k8stypes "k8s.io/apimachinery/pkg/types" - "k8s.io/klog" ) const ( @@ -89,16 +89,16 @@ type resolvedConfig struct { Disks []resolvedDisk } -func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance instance.Instance, retErr error) { - status := getProviderStatus(machine) - klog.V(3).Infof(fmt.Sprintf("'%s' has status %#v", machine.Name, status)) +func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance instance.Instance, retErr error) { + status := getProviderStatus(log, machine) + log.Debugw("Machine status", "status", status) // ensure conditions are present on machine ensureConditions(&status) - config, _, err := p.getConfig(ctx, machine.Spec.ProviderSpec) + config, _, err := p.getConfig(ctx, log, machine.Spec.ProviderSpec) if err != nil { - return nil, fmt.Errorf("unable to get provider config: %w", err) + return nil, fmt.Errorf("failed to get provider config: %w", err) } ctx = createReconcileContext(ctx, reconcileContext{ @@ -121,14 +121,14 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, }() // provision machine - err = provisionVM(ctx, client) + err = provisionVM(ctx, log, client) if err != nil { return nil, anexiaErrorToTerminalError(err, "failed waiting for vm provisioning") } - return p.Get(ctx, machine, data) + return p.Get(ctx, log, machine, data) } -func provisionVM(ctx context.Context, client anxclient.Client) error { +func provisionVM(ctx context.Context, log *zap.SugaredLogger, client anxclient.Client) error { reconcileContext := getReconcileContext(ctx) vmAPI := vsphere.NewAPI(client) @@ -137,11 +137,10 @@ func provisionVM(ctx context.Context, client anxclient.Client) error { status := reconcileContext.Status if status.ProvisioningID == "" { - klog.V(2).Info(fmt.Sprintf("Machine '%s' does not contain a provisioningID yet. Starting to provision", - reconcileContext.Machine.Name)) + log.Info("Machine does not contain a provisioningID yet. Starting to provision") config := reconcileContext.Config - reservedIP, err := getIPAddress(ctx, client) + reservedIP, err := getIPAddress(ctx, log, client) if err != nil { return newError(common.CreateMachineError, "failed to reserve IP: %v", err) } @@ -194,8 +193,7 @@ func provisionVM(ctx context.Context, client anxclient.Client) error { } } - klog.V(2).Info(fmt.Sprintf("Using provisionID from machine '%s' to await completion", - reconcileContext.Machine.Name)) + log.Info("Using provisionID from machine to await completion") meta.SetStatusCondition(&status.Conditions, v1.Condition{ Type: ProvisionedType, @@ -209,20 +207,20 @@ func provisionVM(ctx context.Context, client anxclient.Client) error { var _engsup3404mutex sync.Mutex -func getIPAddress(ctx context.Context, client anxclient.Client) (string, error) { +func getIPAddress(ctx context.Context, log *zap.SugaredLogger, client anxclient.Client) (string, error) { reconcileContext := getReconcileContext(ctx) status := reconcileContext.Status // only use IP if it is still unbound if status.ReservedIP != "" && status.IPState == anxtypes.IPStateUnbound { - klog.Infof("reusing already provisioned ip %q", status.ReservedIP) + log.Infow("Re-using already provisioned IP", "ip", status.ReservedIP) return status.ReservedIP, nil } _engsup3404mutex.Lock() defer _engsup3404mutex.Unlock() - klog.Info(fmt.Sprintf("Creating a new IP for machine %q", reconcileContext.Machine.Name)) + log.Info("Creating a new IP for machine") addrAPI := anxaddr.NewAPI(client) config := reconcileContext.Config res, err := addrAPI.ReserveRandom(ctx, anxaddr.ReserveRandom{ @@ -291,7 +289,7 @@ func resolveTemplateID(ctx context.Context, a api.API, config anxtypes.RawConfig return template.Identifier, nil } -func (p *provider) resolveConfig(ctx context.Context, config anxtypes.RawConfig) (*resolvedConfig, error) { +func (p *provider) resolveConfig(ctx context.Context, log *zap.SugaredLogger, config anxtypes.RawConfig) (*resolvedConfig, error) { var err error ret := resolvedConfig{ RawConfig: config, @@ -337,7 +335,7 @@ func (p *provider) resolveConfig(ctx context.Context, config anxtypes.RawConfig) return nil, ErrConfigDiskSizeAndDisks } - klog.Warningf("Configuration uses the deprecated DiskSize attribute, please migrate to the Disks array instead.") + log.Info("Configuration uses the deprecated DiskSize attribute, please migrate to the Disks array instead.") config.Disks = []anxtypes.RawDisk{ { @@ -361,7 +359,7 @@ func (p *provider) resolveConfig(ctx context.Context, config anxtypes.RawConfig) return &ret, nil } -func (p *provider) getConfig(ctx context.Context, provSpec clusterv1alpha1.ProviderSpec) (*resolvedConfig, *providerconfigtypes.Config, error) { +func (p *provider) getConfig(ctx context.Context, log *zap.SugaredLogger, provSpec clusterv1alpha1.ProviderSpec) (*resolvedConfig, *providerconfigtypes.Config, error) { if provSpec.Value == nil { return nil, nil, fmt.Errorf("machine.spec.providerSpec.value is nil") } @@ -379,7 +377,7 @@ func (p *provider) getConfig(ctx context.Context, provSpec clusterv1alpha1.Provi return nil, nil, fmt.Errorf("error parsing provider config: %w", err) } - resolvedConfig, err := p.resolveConfig(ctx, *rawConfig) + resolvedConfig, err := p.resolveConfig(ctx, log, *rawConfig) if err != nil { return nil, nil, fmt.Errorf("error resolving config: %w", err) } @@ -393,13 +391,13 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes } // AddDefaults adds omitted optional values to the given MachineSpec. -func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } // Validate returns success or failure based according to its ProviderSpec. -func (p *provider) Validate(ctx context.Context, machinespec clusterv1alpha1.MachineSpec) error { - config, _, err := p.getConfig(ctx, machinespec.ProviderSpec) +func (p *provider) Validate(ctx context.Context, log *zap.SugaredLogger, machinespec clusterv1alpha1.MachineSpec) error { + config, _, err := p.getConfig(ctx, log, machinespec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) } @@ -445,8 +443,8 @@ func (p *provider) Validate(ctx context.Context, machinespec clusterv1alpha1.Mac return nil } -func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, pd *cloudprovidertypes.ProviderData) (instance.Instance, error) { - config, _, err := p.getConfig(ctx, machine.Spec.ProviderSpec) +func (p *provider) Get(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, pd *cloudprovidertypes.ProviderData) (instance.Instance, error) { + config, _, err := p.getConfig(ctx, log, machine.Spec.ProviderSpec) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, "failed to retrieve config: %v", err) } @@ -457,7 +455,7 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, pd } vsphereAPI := vsphere.NewAPI(cli) - status := getProviderStatus(machine) + status := getProviderStatus(log, machine) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, "failed to get machine status: %v", err) } @@ -512,19 +510,19 @@ func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string return "", "", nil } -func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (isDeleted bool, retErr error) { - if inst, err := p.Get(ctx, machine, data); err != nil { +func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (isDeleted bool, retErr error) { + if inst, err := p.Get(ctx, log, machine, data); err != nil { if cloudprovidererrors.IsNotFound(err) { return true, nil } return false, err } else if inst.Status() == instance.StatusCreating { - klog.Warningf("Unable to cleanup machine %q. Instance is still creating", machine.Name) + log.Error("Failed to cleanup machine: instance is still creating") return false, nil } - status := getProviderStatus(machine) + status := getProviderStatus(log, machine) // make sure status is reflected in Machine Object defer func() { // if error occurs during updating the machine object don't override the original error @@ -532,7 +530,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine }() ensureConditions(&status) - config, _, err := p.getConfig(ctx, machine.Spec.ProviderSpec) + config, _, err := p.getConfig(ctx, log, machine.Spec.ProviderSpec) if err != nil { return false, newError(common.InvalidConfigurationMachineError, "failed to parse MachineSpec: %v", err) } @@ -581,7 +579,7 @@ func isTaskDone(ctx context.Context, cli anxclient.Client, progressIdentifier st return false, nil } -func (p *provider) MigrateUID(_ context.Context, _ *clusterv1alpha1.Machine, _ k8stypes.UID) error { +func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ k8stypes.UID) error { return nil } @@ -610,12 +608,12 @@ func getClient(token string) (api.API, anxclient.Client, error) { return a, legacyClient, nil } -func getProviderStatus(machine *clusterv1alpha1.Machine) anxtypes.ProviderStatus { +func getProviderStatus(log *zap.SugaredLogger, machine *clusterv1alpha1.Machine) anxtypes.ProviderStatus { var providerStatus anxtypes.ProviderStatus status := machine.Status.ProviderStatus if status != nil && status.Raw != nil { if err := json.Unmarshal(status.Raw, &providerStatus); err != nil { - klog.Warningf("Unable to parse status from machine object. status was discarded for machine") + log.Error("Failed to parse status from machine object; status was discarded for machine") return anxtypes.ProviderStatus{} } } diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index 4bda59f26..085aa70d2 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -38,6 +38,7 @@ import ( "go.anx.io/go-anxcloud/pkg/ipam/address" "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/progress" "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -58,6 +59,7 @@ const ( func TestAnexiaProvider(t *testing.T) { testhelper.SetupHTTP() client, server := anxclient.NewTestClient(nil, testhelper.Mux) + log := zap.NewNop().Sugar() a := mock.NewMockAPI() a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID-OLD-BUILD", Name: testTemplateName, Build: "b01"}) @@ -165,7 +167,7 @@ func TestAnexiaProvider(t *testing.T) { }, }) - err := provisionVM(ctx, client) + err := provisionVM(ctx, log, client) testhelper.AssertNoErr(t, err) }) @@ -270,7 +272,7 @@ func TestAnexiaProvider(t *testing.T) { expectedIP := "8.8.8.8" providerStatus.ReservedIP = expectedIP providerStatus.IPState = anxtypes.IPStateUnbound - reservedIP, err := getIPAddress(ctx, client) + reservedIP, err := getIPAddress(ctx, log, client) testhelper.AssertNoErr(t, err) testhelper.AssertEquals(t, expectedIP, reservedIP) }) @@ -354,7 +356,7 @@ func TestValidate(t *testing.T) { provider := New(nil) for _, testCase := range getSpecsForValidationTest(t, configCases) { - err := provider.Validate(context.Background(), testCase.Spec) + err := provider.Validate(context.Background(), zap.NewNop().Sugar(), testCase.Spec) if testCase.ExpectedError != nil { if !errors.Is(err, testCase.ExpectedError) { testhelper.AssertEquals(t, testCase.ExpectedError.Error(), err.Error()) @@ -390,7 +392,7 @@ func TestGetProviderStatus(t *testing.T) { testhelper.AssertNoErr(t, err) machine.Status.ProviderStatus = &runtime.RawExtension{Raw: providerStatusJSON} - returnedStatus := getProviderStatus(machine) + returnedStatus := getProviderStatus(zap.NewNop().Sugar(), machine) testhelper.AssertEquals(t, "InstanceID", returnedStatus.InstanceID) } @@ -409,7 +411,7 @@ func TestUpdateStatus(t *testing.T) { err = updateMachineStatus(machine, providerStatus, func(paramMachine *v1alpha1.Machine, modifier ...cloudprovidertypes.MachineModifier) error { called = true testhelper.AssertEquals(t, machine, paramMachine) - status := getProviderStatus(machine) + status := getProviderStatus(zap.NewNop().Sugar(), machine) testhelper.AssertEquals(t, status.InstanceID, providerStatus.InstanceID) return nil }) diff --git a/pkg/cloudprovider/provider/anexia/types/types.go b/pkg/cloudprovider/provider/anexia/types/types.go index e6d8e9f22..3686defe1 100644 --- a/pkg/cloudprovider/provider/anexia/types/types.go +++ b/pkg/cloudprovider/provider/anexia/types/types.go @@ -43,7 +43,7 @@ const ( var StatusUpdateFailed = cloudprovidererrors.TerminalError{ Reason: common.UpdateMachineError, - Message: "Unable to update the machine status", + Message: "Failed to update the machine status", } // RawDisk specifies a single disk, with some values maybe being fetched from secrets. diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index bf7bb3398..6a61988e4 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -36,6 +36,7 @@ import ( "github.com/aws/smithy-go" gocache "github.com/patrickmn/go-cache" "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -52,7 +53,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog" "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/metrics" ) @@ -216,7 +216,7 @@ type amiFilter struct { productCode string } -func getDefaultAMIID(ctx context.Context, client *ec2.Client, os providerconfigtypes.OperatingSystem, region string, cpuArchitecture awstypes.CPUArchitecture) (string, error) { +func getDefaultAMIID(ctx context.Context, log *zap.SugaredLogger, client *ec2.Client, os providerconfigtypes.OperatingSystem, region string, cpuArchitecture awstypes.CPUArchitecture) (string, error) { cacheLock.Lock() defer cacheLock.Unlock() @@ -233,7 +233,7 @@ func getDefaultAMIID(ctx context.Context, client *ec2.Client, os providerconfigt cacheKey := fmt.Sprintf("ami-id-%s-%s-%s", region, os, cpuArchitecture) amiID, found := cache.Get(cacheKey) if found { - klog.V(3).Info("found AMI-ID in cache!") + log.Debugw("Found AMI-ID in cache", "key", cacheKey, "ami", amiID) return amiID.(string), nil } @@ -514,7 +514,7 @@ func getEC2client(ctx context.Context, id, secret, region, assumeRoleArn, assume return ec2.NewFromConfig(cfg), nil } -func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { _, _, rawConfig, err := p.getConfig(spec.ProviderSpec) if err != nil { return spec, err @@ -535,7 +535,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, err } -func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, log *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { config, pc, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -642,7 +642,7 @@ func getVpc(ctx context.Context, client *ec2.Client, id string) (*ec2types.Vpc, } if len(vpcOut.Vpcs) != 1 { - return nil, fmt.Errorf("unable to find specified vpc with id %q", id) + return nil, fmt.Errorf("failed to find specified vpc with id %q", id) } return &vpcOut.Vpcs[0], nil @@ -665,7 +665,7 @@ func areVpcDNSHostnamesEnabled(ctx context.Context, client *ec2.Client, id strin return *out.EnableDnsHostnames.Value, nil } -func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -696,7 +696,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, } } - if amiID, err = getDefaultAMIID(ctx, ec2Client, pc.OperatingSystem, config.Region, cpuArchitecture); err != nil { + if amiID, err = getDefaultAMIID(ctx, log, ec2Client, pc.OperatingSystem, config.Region, cpuArchitecture); err != nil { return nil, cloudprovidererrors.TerminalError{ Reason: common.InvalidConfigurationMachineError, Message: fmt.Sprintf("Failed to get AMI-ID for operating system %s in region %s: %v", pc.OperatingSystem, config.Region, err), @@ -817,7 +817,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return &awsInstance{instance: &runOut.Instances[0]}, nil } -func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { ec2instance, err := p.get(ctx, machine) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { @@ -852,7 +852,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine } if cOut.CancelledSpotInstanceRequests[0].State == ec2types.CancelSpotInstanceRequestStateCancelled { - klog.V(3).Infof("successfully canceled spot instance request %s at aws", *ec2instance.instance.SpotInstanceRequestId) + log.Infow("Successfully canceled spot instance request", "request", *ec2instance.instance.SpotInstanceRequestId) } } @@ -864,13 +864,13 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine } if tOut.TerminatingInstances[0].PreviousState.Name != tOut.TerminatingInstances[0].CurrentState.Name { - klog.V(3).Infof("successfully triggered termination of instance %s at aws", ec2instance.ID()) + log.Infow("Successfully triggered termination of instance", "instance", ec2instance.ID()) } return false, nil } -func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { return p.get(ctx, machine) } @@ -953,7 +953,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } -func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *provider) MigrateUID(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { machineInstance, err := p.get(ctx, machine) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { diff --git a/pkg/cloudprovider/provider/azure/create_delete_resources.go b/pkg/cloudprovider/provider/azure/create_delete_resources.go index b751736a1..4d0d80129 100644 --- a/pkg/cloudprovider/provider/azure/create_delete_resources.go +++ b/pkg/cloudprovider/provider/azure/create_delete_resources.go @@ -24,11 +24,11 @@ import ( "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" "github.com/Azure/go-autorest/autorest/azure/auth" "github.com/Azure/go-autorest/autorest/to" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "k8s.io/apimachinery/pkg/types" - "k8s.io/klog" ) // deleteInterfacesByMachineUID will remove all network interfaces tagged with the specific machine's UID. @@ -194,8 +194,8 @@ func getDisksByMachineUID(ctx context.Context, disksClient *compute.DisksClient, return matchingDisks, nil } -func createOrUpdatePublicIPAddress(ctx context.Context, ipName string, ipVersion network.IPVersion, sku network.PublicIPAddressSkuName, ipAllocationMethod network.IPAllocationMethod, machineUID types.UID, c *config) (*network.PublicIPAddress, error) { - klog.Infof("Creating public IP %q", ipName) +func createOrUpdatePublicIPAddress(ctx context.Context, log *zap.SugaredLogger, ipName string, ipVersion network.IPVersion, sku network.PublicIPAddressSkuName, ipAllocationMethod network.IPAllocationMethod, machineUID types.UID, c *config) (*network.PublicIPAddress, error) { + log.Infow("Creating public IP", "name", ipName) ipClient, err := getIPClient(c) if err != nil { return nil, err @@ -229,7 +229,7 @@ func createOrUpdatePublicIPAddress(ctx context.Context, ipName string, ipVersion return nil, fmt.Errorf("failed to create public IP address: %w", err) } - klog.Infof("Fetching info for IP address %q", ipName) + log.Infow("Fetching info for IP address", "name", ipName) ip, err := getPublicIPAddress(ctx, ipName, c.ResourceGroup, ipClient) if err != nil { return nil, fmt.Errorf("failed to fetch info about public IP %q: %w", ipName, err) @@ -256,14 +256,14 @@ func getSubnet(ctx context.Context, c *config) (network.Subnet, error) { return subnetsClient.Get(ctx, c.VNetResourceGroup, c.VNetName, c.SubnetName, "") } -func getSKU(ctx context.Context, c *config) (compute.ResourceSku, error) { +func getSKU(ctx context.Context, log *zap.SugaredLogger, c *config) (compute.ResourceSku, error) { cacheLock.Lock() defer cacheLock.Unlock() cacheKey := fmt.Sprintf("%s-%s", c.Location, c.VMSize) cacheSku, found := cache.Get(cacheKey) if found { - klog.V(3).Info("found SKU in cache!") + log.Debugw("Found SKU in cache", "key", cacheKey, "sku", cacheSku) return cacheSku.(compute.ResourceSku), nil } @@ -319,7 +319,7 @@ func getVirtualNetwork(ctx context.Context, c *config) (network.VirtualNetwork, return virtualNetworksClient.Get(ctx, c.VNetResourceGroup, c.VNetName, "") } -func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineUID types.UID, config *config, publicIP, publicIPv6 *network.PublicIPAddress, ipFamily util.IPFamily, enableAcceleratedNetworking *bool) (*network.Interface, error) { +func createOrUpdateNetworkInterface(ctx context.Context, log *zap.SugaredLogger, ifName string, machineUID types.UID, config *config, publicIP, publicIPv6 *network.PublicIPAddress, ipFamily util.IPFamily, enableAcceleratedNetworking *bool) (*network.Interface, error) { ifClient, err := getInterfacesClient(config) if err != nil { return nil, fmt.Errorf("failed to create interfaces client: %w", err) @@ -377,7 +377,7 @@ func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineU } ifSpec.NetworkSecurityGroup = &secGroup } - klog.Infof("Creating/Updating public network interface %q", ifName) + log.Infow("Creating/Updating public network interface", "interface", ifName) future, err := ifClient.CreateOrUpdate(ctx, config.ResourceGroup, ifName, ifSpec) if err != nil { return nil, fmt.Errorf("failed to create interface: %w", err) @@ -393,7 +393,7 @@ func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineU return nil, fmt.Errorf("failed to get interface creation result: %w", err) } - klog.Infof("Fetching info about network interface %q", ifName) + log.Infow("Fetching info about network interface", "interface", ifName) iface, err := ifClient.Get(ctx, config.ResourceGroup, ifName, "") if err != nil { return nil, fmt.Errorf("failed to fetch info about interface %q: %w", ifName, err) diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 85def986b..819f7465a 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -31,6 +31,7 @@ import ( "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" "github.com/Azure/go-autorest/autorest/to" gocache "github.com/patrickmn/go-cache" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -46,7 +47,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/klog" "k8s.io/utils/pointer" ) @@ -388,7 +388,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p return &c, pconfig, nil } -func getVMIPAddresses(ctx context.Context, c *config, vm *compute.VirtualMachine, ipFamily util.IPFamily) (map[string]v1.NodeAddressType, error) { +func getVMIPAddresses(ctx context.Context, log *zap.SugaredLogger, c *config, vm *compute.VirtualMachine, ipFamily util.IPFamily) (map[string]v1.NodeAddressType, error) { var ( ipAddresses = map[string]v1.NodeAddressType{} err error @@ -413,7 +413,7 @@ func getVMIPAddresses(ctx context.Context, c *config, vm *compute.VirtualMachine splitIfaceID := strings.Split(*iface.ID, "/") ifaceName := splitIfaceID[len(splitIfaceID)-1] - ipAddresses, err = getNICIPAddresses(ctx, c, ipFamily, ifaceName) + ipAddresses, err = getNICIPAddresses(ctx, log, c, ipFamily, ifaceName) if err != nil || vm.NetworkProfile.NetworkInterfaces == nil { return nil, fmt.Errorf("failed to get addresses for interface %q: %w", ifaceName, err) } @@ -422,7 +422,7 @@ func getVMIPAddresses(ctx context.Context, c *config, vm *compute.VirtualMachine return ipAddresses, nil } -func getNICIPAddresses(ctx context.Context, c *config, ipFamily util.IPFamily, ifaceName string) (map[string]v1.NodeAddressType, error) { +func getNICIPAddresses(ctx context.Context, log *zap.SugaredLogger, c *config, ipFamily util.IPFamily, ifaceName string) (map[string]v1.NodeAddressType, error) { ifClient, err := getInterfacesClient(c) if err != nil { return nil, fmt.Errorf("failed to create interfaces client: %w", err) @@ -444,7 +444,7 @@ func getNICIPAddresses(ctx context.Context, c *config, ipFamily util.IPFamily, i if conf.Name != nil { name = *conf.Name } else { - klog.Warningf("IP configuration of NIC %q was returned with no name, trying to dissect the ID.", ifaceName) + log.Infow("IP configuration of NIC was returned with no name, trying to dissect the ID.", "interface", ifaceName) if conf.ID == nil || len(*conf.ID) == 0 { return nil, fmt.Errorf("IP configuration of NIC %q was returned with no ID", ifaceName) } @@ -528,7 +528,7 @@ func getInternalIPAddresses(ctx context.Context, c *config, inetface, ipconfigNa return ipAddresses, nil } -func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } @@ -573,7 +573,7 @@ func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) return sp, nil } -func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { config, providerCfg, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -618,13 +618,13 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, }); err != nil { return nil, err } - publicIP, err = createOrUpdatePublicIPAddress(ctx, publicIPName(ifaceName(machine)), network.IPVersionIPv4, sku, network.IPAllocationMethodStatic, machine.UID, config) + publicIP, err = createOrUpdatePublicIPAddress(ctx, log, publicIPName(ifaceName(machine)), network.IPVersionIPv4, sku, network.IPAllocationMethodStatic, machine.UID, config) if err != nil { return nil, fmt.Errorf("failed to create public IP: %w", err) } if ipFamily.IsDualstack() { - publicIPv6, err = createOrUpdatePublicIPAddress(ctx, publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodStatic, machine.UID, config) + publicIPv6, err = createOrUpdatePublicIPAddress(ctx, log, publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodStatic, machine.UID, config) if err != nil { return nil, fmt.Errorf("failed to create public IP: %w", err) } @@ -639,7 +639,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, err } - iface, err := createOrUpdateNetworkInterface(ctx, ifaceName(machine), machine.UID, config, publicIP, publicIPv6, ipFamily, config.EnableAcceleratedNetworking) + iface, err := createOrUpdateNetworkInterface(ctx, log, ifaceName(machine), machine.UID, config, publicIP, publicIPv6, ipFamily, config.EnableAcceleratedNetworking) if err != nil { return nil, fmt.Errorf("failed to generate main network interface: %w", err) } @@ -711,7 +711,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, } } - klog.Infof("Creating machine %q", machine.Name) + log.Info("Creating machine") if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { if !kuberneteshelper.HasFinalizer(updatedMachine, finalizerDisks) { updatedMachine.Finalizers = append(updatedMachine.Finalizers, finalizerDisks) @@ -744,12 +744,12 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, fmt.Errorf("failed to retrieve updated data for VM %q: %w", machine.Name, err) } - ipAddresses, err := getVMIPAddresses(ctx, config, &vm, ipFamily) + ipAddresses, err := getVMIPAddresses(ctx, log, config, &vm, ipFamily) if err != nil { return nil, fmt.Errorf("failed to retrieve IP addresses for VM %q: %w", machine.Name, err) } - status, err := getVMStatus(ctx, config, machine.Name) + status, err := getVMStatus(ctx, log, config, machine.Name) if err != nil { return nil, fmt.Errorf("failed to retrieve status for VM %q: %w", machine.Name, err) } @@ -757,14 +757,14 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return &azureVM{vm: &vm, ipAddresses: ipAddresses, status: status}, nil } -func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, fmt.Errorf("failed to parse MachineSpec: %w", err) } if kuberneteshelper.HasFinalizer(machine, finalizerVM) { - klog.Infof("deleting VM %q", machine.Name) + log.Info("Deleting VM") if err = deleteVMsByMachineUID(ctx, config, machine.UID); err != nil { return false, fmt.Errorf("failed to delete instance for machine %q: %w", machine.Name, err) } @@ -777,7 +777,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine } if kuberneteshelper.HasFinalizer(machine, finalizerDisks) { - klog.Infof("deleting disks of VM %q", machine.Name) + log.Info("Deleting disks") if err := deleteDisksByMachineUID(ctx, config, machine.UID); err != nil { return false, fmt.Errorf("failed to remove disks of machine %q: %w", machine.Name, err) } @@ -789,7 +789,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine } if kuberneteshelper.HasFinalizer(machine, finalizerNIC) { - klog.Infof("deleting network interfaces of VM %q", machine.Name) + log.Info("Deleting network interfaces") if err := deleteInterfacesByMachineUID(ctx, config, machine.UID); err != nil { return false, fmt.Errorf("failed to remove network interfaces of machine %q: %w", machine.Name, err) } @@ -801,7 +801,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine } if kuberneteshelper.HasFinalizer(machine, finalizerPublicIP) { - klog.Infof("deleting public IP addresses of VM %q", machine.Name) + log.Infof("Deleting public IP addresses") if err := deleteIPAddressesByMachineUID(ctx, config, machine.UID); err != nil { return false, fmt.Errorf("failed to remove public IP addresses of machine %q: %w", machine.Name, err) } @@ -844,7 +844,7 @@ func getVMByUID(ctx context.Context, c *config, uid types.UID) (*compute.Virtual return nil, cloudprovidererrors.ErrInstanceNotFound } -func getVMStatus(ctx context.Context, c *config, vmName string) (instance.Status, error) { +func getVMStatus(ctx context.Context, log *zap.SugaredLogger, c *config, vmName string) (instance.Status, error) { vmClient, err := getVMClient(c) if err != nil { return instance.StatusUnknown, err @@ -863,7 +863,7 @@ func getVMStatus(ctx context.Context, c *config, vmName string) (instance.Status if len(*iv.Statuses) < 2 { provisioningStatus := (*iv.Statuses)[0] if provisioningStatus.Code == nil { - klog.Warningf("azure provisioning status has missing code") + log.Info("Azure provisioning status has missing code") return instance.StatusUnknown, nil } @@ -873,7 +873,7 @@ func getVMStatus(ctx context.Context, c *config, vmName string) (instance.Status case "ProvisioningState/deleting": return instance.StatusDeleting, nil default: - klog.Warningf("unknown Azure provisioning status %q", *provisioningStatus.Code) + log.Errorw("Unknown Azure provisioning status", "code", *provisioningStatus.Code, "level", provisioningStatus.Level) return instance.StatusUnknown, nil } } @@ -882,7 +882,7 @@ func getVMStatus(ctx context.Context, c *config, vmName string) (instance.Status // https://docs.microsoft.com/en-us/azure/virtual-machines/windows/tutorial-manage-vm#vm-power-states powerStatus := (*iv.Statuses)[1] if powerStatus.Code == nil { - klog.Warningf("azure power status has missing code") + log.Info("Azure power status has missing code") return instance.StatusUnknown, nil } @@ -894,16 +894,16 @@ func getVMStatus(ctx context.Context, c *config, vmName string) (instance.Status case "PowerState/starting": return instance.StatusCreating, nil default: - klog.Warningf("unknown Azure power status %q", *powerStatus.Code) + log.Errorw("Unknown Azure power status", "code", *powerStatus.Code, "level", powerStatus.Level) return instance.StatusUnknown, nil } } -func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - return p.get(ctx, machine) +func (p *provider) Get(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { + return p.get(ctx, log, machine) } -func (p *provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (*azureVM, error) { +func (p *provider) get(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine) (*azureVM, error) { config, providerCfg, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("failed to parse MachineSpec: %w", err) @@ -919,12 +919,12 @@ func (p *provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (* } ipFamily := providerCfg.Network.GetIPFamily() - ipAddresses, err := getVMIPAddresses(ctx, config, vm, ipFamily) + ipAddresses, err := getVMIPAddresses(ctx, log, config, vm, ipFamily) if err != nil { return nil, fmt.Errorf("failed to retrieve IP addresses for VM %v: %w", vm.Name, err) } - status, err := getVMStatus(ctx, config, machine.Name) + status, err := getVMStatus(ctx, log, config, machine.Name) if err != nil { return nil, fmt.Errorf("failed to retrieve status for VM %v: %w", vm.Name, err) } @@ -1011,7 +1011,7 @@ func validateSKUCapabilities(ctx context.Context, c *config, sku compute.Resourc return nil } -func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, log *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { c, providerConfig, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -1095,7 +1095,7 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return fmt.Errorf("failed to get subnet: %w", err) } - sku, err := getSKU(ctx, c) + sku, err := getSKU(ctx, log, c) if err != nil { return fmt.Errorf("failed to get VM SKU: %w", err) } @@ -1124,7 +1124,7 @@ func publicIPv6Name(ifaceName string) string { return ifaceName + "-pubipv6" } -func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *provider) MigrateUID(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { config, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return cloudprovidererrors.TerminalError{ @@ -1143,21 +1143,21 @@ func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Mach if kuberneteshelper.HasFinalizer(machine, finalizerPublicIPv6) { sku = network.PublicIPAddressSkuNameStandard - _, err = createOrUpdatePublicIPAddress(ctx, publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodDynamic, newUID, config) + _, err = createOrUpdatePublicIPAddress(ctx, log, publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodDynamic, newUID, config) if err != nil { return fmt.Errorf("failed to update UID on public IP: %w", err) } } if kuberneteshelper.HasFinalizer(machine, finalizerPublicIP) { - _, err = createOrUpdatePublicIPAddress(ctx, publicIPName(ifaceName(machine)), network.IPVersionIPv4, sku, network.IPAllocationMethodStatic, newUID, config) + _, err = createOrUpdatePublicIPAddress(ctx, log, publicIPName(ifaceName(machine)), network.IPVersionIPv4, sku, network.IPAllocationMethodStatic, newUID, config) if err != nil { return fmt.Errorf("failed to update UID on public IP: %w", err) } } if kuberneteshelper.HasFinalizer(machine, finalizerNIC) { - _, err = createOrUpdateNetworkInterface(ctx, ifaceName(machine), newUID, config, publicIP, publicIPv6, util.IPFamilyUnspecified, config.EnableAcceleratedNetworking) + _, err = createOrUpdateNetworkInterface(ctx, log, ifaceName(machine), newUID, config, publicIP, publicIPv6, util.IPFamilyUnspecified, config.EnableAcceleratedNetworking) if err != nil { return fmt.Errorf("failed to update UID on main network interface: %w", err) } diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/hardware.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/hardware.go index dd8a006ab..22ecd02a7 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/hardware.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/hardware.go @@ -20,8 +20,6 @@ import ( "encoding/json" "github.com/tinkerbell/tink/pkg" - - "k8s.io/klog" ) type HardwareSpec struct { @@ -59,7 +57,6 @@ func (h *HardwareSpec) GetStatus() string { }{} if err := json.Unmarshal([]byte(h.Hardware.Metadata), &metadata); err != nil { - klog.Errorf("failed to unmarshal hardware metadata: %v", err) return "" } diff --git a/pkg/cloudprovider/provider/baremetal/provider.go b/pkg/cloudprovider/provider/baremetal/provider.go index 1203cb786..fedca37f1 100644 --- a/pkg/cloudprovider/provider/baremetal/provider.go +++ b/pkg/cloudprovider/provider/baremetal/provider.go @@ -22,6 +22,8 @@ import ( "errors" "fmt" + "go.uber.org/zap" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" @@ -166,12 +168,12 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return &c, pconfig, err } -func (p provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { _, _, err := p.getConfig(spec.ProviderSpec) return spec, err } -func (p provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { c, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -188,7 +190,7 @@ func (p provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) return nil } -func (p provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -215,7 +217,7 @@ func (p provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, return "", "", nil } -func (p provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -250,7 +252,7 @@ func (p provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, }, nil } -func (p provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, cloudprovidererrors.TerminalError{ @@ -283,7 +285,7 @@ func (p provider) MachineMetricsLabels(_ *clusterv1alpha1.Machine) (map[string]s return nil, nil } -func (p provider) MigrateUID(_ context.Context, _ *clusterv1alpha1.Machine, _ types.UID) error { +func (p provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ types.UID) error { return nil } diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index 63b5d65ae..d6a2c554a 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -25,6 +25,7 @@ import ( "time" "github.com/digitalocean/godo" + "go.uber.org/zap" "golang.org/x/oauth2" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" @@ -42,7 +43,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog" ) type provider struct { @@ -162,11 +162,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return &c, pconfig, err } -func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } -func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -276,7 +276,7 @@ func uploadRandomSSHPublicKey(ctx context.Context, service godo.KeysService) (st return newDoKey.Fingerprint, nil } -func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -294,7 +294,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, defer func() { _, err := client.Keys.DeleteByFingerprint(ctx, fingerprint) if err != nil { - klog.Errorf("failed to remove a temporary ssh key with fingerprint = %v, due to = %v", fingerprint, err) + log.Errorw("Failed to remove a temporary ssh key", "fingerprint", fingerprint, zap.Error(err)) } }() @@ -324,6 +324,8 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, doStatusAndErrToTerminalError(rsp.StatusCode, err) } + dropletLog := log.With("droplet", droplet.ID) + //We need to wait until the droplet really got created as tags will be only applied when the droplet is running err = wait.Poll(createCheckPeriod, createCheckTimeout, func() (done bool, err error) { newDroplet, rsp, err := client.Droplets.Get(ctx, droplet.ID) @@ -334,20 +336,20 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, } //Well just wait 10 sec and hope the droplet got started by then... time.Sleep(createCheckFailedWaitPeriod) - return false, fmt.Errorf("droplet (id='%d') got created but we failed to fetch its status", droplet.ID) + return false, fmt.Errorf("droplet %q got created but we failed to fetch its status", droplet.ID) } if sets.NewString(newDroplet.Tags...).Has(string(machine.UID)) { - klog.V(6).Infof("droplet (id='%d') got fully created", droplet.ID) + dropletLog.Debug("Droplet got fully created") return true, nil } - klog.V(6).Infof("waiting until droplet (id='%d') got fully created...", droplet.ID) + dropletLog.Debug("Waiting until droplet got fully created...") return false, nil }) return &doInstance{droplet: droplet}, err } -func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { instance, err := p.get(ctx, machine) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { @@ -378,7 +380,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine return false, nil } -func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { return p.get(ctx, machine) } @@ -436,7 +438,7 @@ func (p *provider) listDroplets(ctx context.Context, token string) ([]godo.Dropl return result, nil } -func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *provider) MigrateUID(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to decode providerconfig: %w", err) diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index 2c290450e..2bbcb6aa5 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -25,6 +25,7 @@ import ( "strings" "github.com/packethost/packngo" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -38,7 +39,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/klog" ) const ( @@ -169,7 +169,7 @@ func (p *provider) getMetalDevice(machine *clusterv1alpha1.Machine) (*packngo.De return device, client, nil } -func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { c, _, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -240,7 +240,7 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) return nil } -func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, _, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -281,8 +281,8 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d return &metalDevice{device: device}, nil } -func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - instance, err := p.Get(ctx, machine, data) +func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { + instance, err := p.Get(ctx, log, machine, data) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return true, nil @@ -307,7 +307,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine return false, nil } -func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { _, rawConfig, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return spec, err @@ -320,7 +320,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, nil } -func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { device, _, err := p.getMetalDevice(machine) if err != nil { return nil, err @@ -332,13 +332,13 @@ func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, _ *c return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machine, newID types.UID) error { +func (p *provider) MigrateUID(_ context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newID types.UID) error { device, client, err := p.getMetalDevice(machine) if err != nil { return err } if device == nil { - klog.Infof("No instance exists for machine %s", machine.Name) + log.Info("No instance exists for machine") return nil } @@ -354,7 +354,7 @@ func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machin // create a new UID label tags = append(tags, generateTag(string(newID))) - klog.Infof("Setting UID label for machine %s", machine.Name) + log.Info("Setting UID label for machine") dur := &packngo.DeviceUpdateRequest{ Tags: &tags, } @@ -362,7 +362,7 @@ func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machin if err != nil { return metalErrorToTerminalError(err, response, "failed to update UID label") } - klog.Infof("Successfully set UID label for machine %s", machine.Name) + log.Info("Successfully set UID label for machine") return nil } diff --git a/pkg/cloudprovider/provider/fake/provider.go b/pkg/cloudprovider/provider/fake/provider.go index c1d0c7c56..2a01f5f92 100644 --- a/pkg/cloudprovider/provider/fake/provider.go +++ b/pkg/cloudprovider/provider/fake/provider.go @@ -21,6 +21,8 @@ import ( "encoding/json" "fmt" + "go.uber.org/zap" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" @@ -29,7 +31,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/klog" ) type provider struct{} @@ -65,12 +66,12 @@ func New(_ *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{} } -func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } // Validate returns success or failure based according to its FakeCloudProviderSpec. -func (p *provider) Validate(_ context.Context, machinespec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(_ context.Context, log *zap.SugaredLogger, machinespec clusterv1alpha1.MachineSpec) error { pconfig, err := providerconfigtypes.GetConfig(machinespec.ProviderSpec) if err != nil { return err @@ -82,15 +83,15 @@ func (p *provider) Validate(_ context.Context, machinespec clusterv1alpha1.Machi } if fakeCloudProviderSpec.PassValidation { - klog.V(3).Infof("succeeding validation as requested") + log.Debug("Succeeding validation as requested") return nil } - klog.V(3).Infof("failing validation as requested") + log.Debug("Failing validation as requested") return fmt.Errorf("failing validation as requested") } -func (p *provider) Get(_ context.Context, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { return CloudProviderInstance{}, nil } @@ -99,15 +100,15 @@ func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string } // Create creates a cloud instance according to the given machine. -func (p *provider) Create(_ context.Context, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, _ string) (instance.Instance, error) { +func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, _ string) (instance.Instance, error) { return CloudProviderInstance{}, nil } -func (p *provider) Cleanup(_ context.Context, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { return true, nil } -func (p *provider) MigrateUID(_ context.Context, _ *clusterv1alpha1.Machine, _ types.UID) error { +func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ types.UID) error { return nil } diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index ba4f0e947..62190dbb1 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -29,6 +29,7 @@ import ( "cloud.google.com/go/logging" monitoring "cloud.google.com/go/monitoring/apiv3/v2" + "go.uber.org/zap" compute "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" @@ -84,7 +85,7 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) *Provider { } // AddDefaults reads the MachineSpec and applies defaults for provider specific fields. -func (p *Provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *Provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { // Read cloud provider spec. cpSpec, _, err := newCloudProviderSpec(spec.ProviderSpec) if err != nil { @@ -102,7 +103,7 @@ func (p *Provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha } // Validate checks the given machine's specification. -func (p *Provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *Provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { // Read configuration. cfg, err := newConfig(p.resolver, spec.ProviderSpec) if err != nil { @@ -143,7 +144,7 @@ func (p *Provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) } // Get retrieves a node instance that is associated with the given machine. -func (p *Provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *Provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { return p.get(machine) } @@ -210,7 +211,7 @@ func (p *Provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri } // Create inserts a cloud instance according to the given machine. -func (p *Provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *Provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { // Read configuration. cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) if err != nil { @@ -222,7 +223,7 @@ func (p *Provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, newError(common.InvalidConfigurationMachineError, errConnect, err) } // Create Google compute instance spec and insert it. - networkInterfaces, err := svc.networkInterfaces(cfg) + networkInterfaces, err := svc.networkInterfaces(log, cfg) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, errMachineSpec, err) } @@ -299,11 +300,11 @@ func (p *Provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, newError(common.InvalidConfigurationMachineError, errInsertInstance, err) } // Retrieve it to get a full qualified instance. - return p.Get(ctx, machine, data) + return p.Get(ctx, log, machine, data) } // Cleanup deletes the instance associated with the machine and all associated resources. -func (p *Provider) Cleanup(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *Provider) Cleanup(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { // Read configuration. cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) if err != nil { @@ -354,7 +355,7 @@ func (p *Provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s // MigrateUID updates the UID of an instance after the controller migrates types // and the UID of the machine object changed. -func (p *Provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *Provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { // Read configuration. cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) if err != nil { diff --git a/pkg/cloudprovider/provider/gce/provider_test.go b/pkg/cloudprovider/provider/gce/provider_test.go index d1217a2f8..cf3de6de7 100644 --- a/pkg/cloudprovider/provider/gce/provider_test.go +++ b/pkg/cloudprovider/provider/gce/provider_test.go @@ -24,6 +24,8 @@ import ( "strings" "testing" + "go.uber.org/zap" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/providerconfig" @@ -168,7 +170,7 @@ func TestValidate(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - err := p.Validate(context.Background(), test.mspec) + err := p.Validate(context.Background(), zap.NewNop().Sugar(), test.mspec) if (err != nil) != test.expectErr { t.Fatalf("expectedErr: %t, got: %v", test.expectErr, err) } diff --git a/pkg/cloudprovider/provider/gce/service.go b/pkg/cloudprovider/provider/gce/service.go index 3a18c1b01..2f451d4ab 100644 --- a/pkg/cloudprovider/provider/gce/service.go +++ b/pkg/cloudprovider/provider/gce/service.go @@ -26,12 +26,12 @@ import ( "fmt" "time" + "go.uber.org/zap" "golang.org/x/oauth2" "google.golang.org/api/compute/v1" "google.golang.org/api/option" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog" ) const ( @@ -70,7 +70,7 @@ func connectComputeService(cfg *config) (*service, error) { } // networkInterfaces returns the configured network interfaces for an instance creation. -func (svc *service) networkInterfaces(cfg *config) ([]*compute.NetworkInterface, error) { +func (svc *service) networkInterfaces(log *zap.SugaredLogger, cfg *config) ([]*compute.NetworkInterface, error) { network := cfg.network if cfg.network == "" && cfg.subnetwork == "" { @@ -82,7 +82,7 @@ func (svc *service) networkInterfaces(cfg *config) ([]*compute.NetworkInterface, Subnetwork: cfg.subnetwork, } - klog.Infof("using network:%s subnetwork: %s", cfg.network, cfg.subnetwork) + log.Infow("Network configuration", "network", cfg.network, "subnetwork", cfg.subnetwork) if cfg.assignPublicIPAddress { ifc.AccessConfigs = []*compute.AccessConfig{ @@ -109,7 +109,7 @@ func (svc *service) networkInterfaces(cfg *config) ([]*compute.NetworkInterface, }, } } else { - klog.Infof("IP family doesn't specify dual stack: %s", cfg.providerConfig.Network.GetIPFamily()) + log.Infow("IP family doesn't specify dual stack", "family", cfg.providerConfig.Network.GetIPFamily()) } } return []*compute.NetworkInterface{ifc}, nil diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 7af032611..83521dccf 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -25,6 +25,7 @@ import ( "strings" "github.com/hetznercloud/hcloud-go/hcloud" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -39,7 +40,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" - "k8s.io/klog" ) const ( @@ -194,7 +194,7 @@ func (p *provider) getServerPlacementGroup(ctx context.Context, client *hcloud.C return createdPg.PlacementGroup, nil } -func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -260,7 +260,7 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return nil } -func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -389,7 +389,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, defer func() { _, err := client.SSHKey.Delete(ctx, hkey) if err != nil { - klog.Errorf("Failed to delete temporary ssh key: %v", err) + log.Errorw("Failed to delete temporary ssh key", zap.Error(err)) } }() serverCreateOpts.SSHKeys = []*hcloud.SSHKey{hkey} @@ -405,8 +405,8 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return &hetznerServer{server: serverCreateRes.Server}, nil } -func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - instance, err := p.Get(ctx, machine, data) +func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { + instance, err := p.Get(ctx, log, machine, data) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return true, nil @@ -455,11 +455,11 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine return false, nil } -func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } -func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -486,7 +486,7 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *provider) MigrateUID(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return cloudprovidererrors.TerminalError{ @@ -502,11 +502,11 @@ func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Mach return fmt.Errorf("failed to get server: %w", err) } if server == nil { - klog.Infof("No instance exists for machine %s", machine.Name) + log.Info("No instance exists for machine") return nil } - klog.Infof("Setting UID label for machine %s", machine.Name) + log.Info("Setting UID label for machine") _, response, err := client.Server.Update(ctx, server, hcloud.ServerUpdateOpts{ Labels: map[string]string{machineUIDLabelKey: string(newUID)}, }) @@ -518,7 +518,7 @@ func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Mach } // This succeeds, but does not result in a label on the server, seems to be a bug // on Hetzner side - klog.Infof("Successfully set UID label for machine %s", machine.Name) + log.Info("Successfully set UID label for machine") return nil } diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 044439b1b..e5aba1041 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -27,6 +27,7 @@ import ( "strings" "time" + "go.uber.org/zap" kubevirtv1 "kubevirt.io/api/core/v1" cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" @@ -49,17 +50,16 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog" utilpointer "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" ) func init() { if err := kubevirtv1.AddToScheme(scheme.Scheme); err != nil { - klog.Fatalf("failed to add kubevirtv1 to scheme: %v", err) + panic(fmt.Sprintf("failed to add kubevirtv1 to scheme: %v", err)) } if err := cdiv1beta1.AddToScheme(scheme.Scheme); err != nil { - klog.Fatalf("failed to add cdiv1beta1 to scheme: %v", err) + panic(fmt.Sprintf("failed to add cdiv1beta1 to scheme: %v", err)) } } @@ -407,7 +407,7 @@ func getNamespace() string { return ns } -func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -463,11 +463,11 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ // We don't use the UID for kubevirt because the name of a VMI must stay stable // in order for the node name to stay stable. The operator is responsible for ensuring // there are no conflicts, e.G. by using one Namespace per Kubevirt user cluster. -func (p *provider) MigrateUID(_ context.Context, _ *clusterv1alpha1.Machine, _ types.UID) error { +func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ types.UID) error { return nil } -func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -501,7 +501,7 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return nil } -func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } @@ -545,7 +545,7 @@ func machineDeploymentNameAndRevisionForMachineGetter(ctx context.Context, machi } } -func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -674,7 +674,7 @@ func (p *provider) newVirtualMachine(ctx context.Context, c *Config, pc *provide return virtualMachine, nil } -func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index 67b200588..c3ef9ebe3 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -29,6 +29,7 @@ import ( "time" "github.com/linode/linodego" + "go.uber.org/zap" "golang.org/x/oauth2" common "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" @@ -159,11 +160,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return &c, pconfig, err } -func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } -func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -211,7 +212,7 @@ func createRandomPassword() (string, error) { return rootPass, nil } -func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -272,8 +273,8 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return &linodeInstance{linode: linode}, err } -func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - instance, err := p.Get(ctx, machine, data) +func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { + instance, err := p.Get(ctx, log, machine, data) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return true, nil @@ -312,7 +313,7 @@ func getListOptions(name string) *linodego.ListOptions { return listOptions } -func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -339,7 +340,7 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *provider) MigrateUID(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to decode providerconfig: %w", err) diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 0020be4c3..c93816d58 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -23,6 +23,8 @@ import ( "strconv" "time" + "go.uber.org/zap" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" @@ -199,11 +201,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return &c, pconfig, rawConfig, nil } -func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } -func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { config, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse machineSpec: %w", err) @@ -255,10 +257,10 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return nil } -func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { vm, err := p.create(ctx, machine, userdata) if err != nil { - _, cleanupErr := p.Cleanup(ctx, machine, data) + _, cleanupErr := p.Cleanup(ctx, log, machine, data) if cleanupErr != nil { return nil, fmt.Errorf("cleaning up failed with err %w after creation failed with err %w", cleanupErr, err) } @@ -287,7 +289,7 @@ func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, return createVM(ctx, client, machine.Name, *config, pc.OperatingSystem, userdata) } -func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { return p.cleanup(ctx, machine, data) } @@ -352,7 +354,7 @@ func (p *provider) cleanup(ctx context.Context, machine *clusterv1alpha1.Machine return true, nil } -func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -417,7 +419,7 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, da }, nil } -func (p *provider) MigrateUID(_ context.Context, _ *clusterv1alpha1.Machine, _ ktypes.UID) error { +func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ ktypes.UID) error { return nil } diff --git a/pkg/cloudprovider/provider/openstack/helper.go b/pkg/cloudprovider/provider/openstack/helper.go index b7475fd2d..d6bb0b8ab 100644 --- a/pkg/cloudprovider/provider/openstack/helper.go +++ b/pkg/cloudprovider/provider/openstack/helper.go @@ -36,6 +36,7 @@ import ( osports "github.com/gophercloud/gophercloud/openstack/networking/v2/ports" ossubnets "github.com/gophercloud/gophercloud/openstack/networking/v2/subnets" "github.com/gophercloud/gophercloud/pagination" + "go.uber.org/zap" ) var ( @@ -263,7 +264,7 @@ func getSubnet(netClient *gophercloud.ServiceClient, nameOrID string) (*ossubnet return nil, errNotFound } -func ensureKubernetesSecurityGroupExist(client *gophercloud.ProviderClient, region, name string) error { +func ensureKubernetesSecurityGroupExist(log *zap.SugaredLogger, client *gophercloud.ProviderClient, region, name string) error { // We need a mutex here because otherwise if more than one machine gets created at roughly the same time // we will create two security groups and subsequently not be able anymore to identify our security group // by name @@ -272,7 +273,7 @@ func ensureKubernetesSecurityGroupExist(client *gophercloud.ProviderClient, regi netClient, err := goopenstack.NewNetworkV2(client, gophercloud.EndpointOpts{Region: region}) if err != nil { - return osErrorToTerminalError(err, "failed to get network client") + return osErrorToTerminalError(log, err, "failed to get network client") } _, err = getSecurityGroup(client, region, name) @@ -280,7 +281,7 @@ func ensureKubernetesSecurityGroupExist(client *gophercloud.ProviderClient, regi if errors.Is(err, errNotFound) { sg, err := ossecuritygroups.Create(netClient, ossecuritygroups.CreateOpts{Name: name}).Extract() if err != nil { - return osErrorToTerminalError(err, fmt.Sprintf("failed to create security group %s", name)) + return osErrorToTerminalError(log, err, fmt.Sprintf("failed to create security group %s", name)) } rules := []osecruritygrouprules.CreateOpts{ @@ -302,7 +303,7 @@ func ensureKubernetesSecurityGroupExist(client *gophercloud.ProviderClient, regi for _, opts := range rules { if _, err := osecruritygrouprules.Create(netClient, opts).Extract(); err != nil { - return osErrorToTerminalError(err, "failed to create security group rule") + return osErrorToTerminalError(log, err, "failed to create security group rule") } } } diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 90acf3423..65a6096dc 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -34,6 +34,7 @@ import ( osfloatingips "github.com/gophercloud/gophercloud/openstack/networking/v2/extensions/layer3/floatingips" osnetworks "github.com/gophercloud/gophercloud/openstack/networking/v2/networks" "github.com/gophercloud/gophercloud/pagination" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -51,7 +52,6 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/klog" ) const ( @@ -64,7 +64,7 @@ const ( type clientGetterFunc func(c *Config) (*gophercloud.ProviderClient, error) // portReadinessWaiterFunc waits for the port with the given ID to be available. -type portReadinessWaiterFunc func(netClient *gophercloud.ServiceClient, serverID string, networkID string, instanceReadyCheckPeriod time.Duration, instanceReadyCheckTimeout time.Duration) error +type portReadinessWaiterFunc func(instanceLog *zap.SugaredLogger, netClient *gophercloud.ServiceClient, serverID string, networkID string, instanceReadyCheckPeriod time.Duration, instanceReadyCheckTimeout time.Duration) error type provider struct { configVarResolver *providerconfig.ConfigVarResolver @@ -152,7 +152,6 @@ func (p *provider) getConfigAuth(c *Config, rawConfig *openstacktypes.RawConfig) return fmt.Errorf("failed to get the value of \"applicationCredentialID\" field, error = %w", err) } if c.ApplicationCredentialID != "" { - klog.V(6).Infof("applicationCredentialID from configuration or environment was found.") c.ApplicationCredentialSecret, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ApplicationCredentialSecret, "OS_APPLICATION_CREDENTIAL_SECRET") if err != nil { return fmt.Errorf("failed to get the value of \"applicationCredentialSecret\" field, error = %w", err) @@ -210,10 +209,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } // Ignore Region not found as Region might not be found and we can default it later. - cfg.Region, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Region, "OS_REGION_NAME") - if err != nil { - klog.V(6).Infof("Region from configuration or environment variable not found") - } + cfg.Region, _ = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Region, "OS_REGION_NAME") cfg.InstanceReadyCheckPeriod, err = p.configVarResolver.GetConfigVarDurationValueOrDefault(rawConfig.InstanceReadyCheckPeriod, 5*time.Second) if err != nil { @@ -355,7 +351,7 @@ func getClient(c *Config) (*gophercloud.ProviderClient, error) { return pc, err } -func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(log *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { c, _, rawConfig, err := p.getConfig(spec.ProviderSpec) if err != nil { return spec, cloudprovidererrors.TerminalError{ @@ -366,17 +362,17 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha client, err := p.clientGetter(c) if err != nil { - return spec, osErrorToTerminalError(err, "failed to get a openstack client") + return spec, osErrorToTerminalError(log, err, "failed to get a openstack client") } if c.Region == "" { - klog.V(3).Infof("Trying to default region for machine '%s'...", spec.Name) + log.Debug("Trying to default region for machine...") regions, err := getRegions(client) if err != nil { - return spec, osErrorToTerminalError(err, "failed to get regions") + return spec, osErrorToTerminalError(log, err, "failed to get regions") } if len(regions) == 1 { - klog.V(3).Infof("Defaulted region for machine '%s' to '%s'", spec.Name, regions[0].ID) + log.Debugw("Defaulted region for machine", "region", regions[0].ID) rawConfig.Region.Value = regions[0].ID } else { return spec, fmt.Errorf("could not default region because got '%v' results", len(regions)) @@ -385,17 +381,17 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha computeClient, err := getNewComputeV2(client, c) if err != nil { - return spec, osErrorToTerminalError(err, "failed to get computeClient") + return spec, osErrorToTerminalError(log, err, "failed to get computeClient") } if c.AvailabilityZone == "" { - klog.V(3).Infof("Trying to default availability zone for machine '%s'...", spec.Name) + log.Debug("Trying to default availability zone for machine...") availabilityZones, err := getAvailabilityZones(computeClient, c) if err != nil { - return spec, osErrorToTerminalError(err, "failed to get availability zones") + return spec, osErrorToTerminalError(log, err, "failed to get availability zones") } if len(availabilityZones) == 1 { - klog.V(3).Infof("Defaulted availability zone for machine '%s' to '%s'", spec.Name, availabilityZones[0].ZoneName) + log.Debugw("Defaulted availability zone for machine", "zone", availabilityZones[0].ZoneName) rawConfig.AvailabilityZone.Value = availabilityZones[0].ZoneName } } @@ -406,19 +402,21 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha } if c.Network == "" { - klog.V(3).Infof("Trying to default network for machine '%s'...", spec.Name) + log.Debug("Trying to default network for machine...") net, err := getDefaultNetwork(netClient) if err != nil { - return spec, osErrorToTerminalError(err, "failed to default network") + return spec, osErrorToTerminalError(log, err, "failed to default network") } if net != nil { - klog.V(3).Infof("Defaulted network for machine '%s' to '%s'", spec.Name, net.Name) + log.Debugw("Defaulted network for machine ", "network", net.Name) // Use the id as the name may not be unique rawConfig.Network.Value = net.ID } } if c.Subnet == "" { + log.Debug("Trying to default subnet for machine...") + networkID := c.Network if rawConfig.Network.Value != "" { networkID = rawConfig.Network.Value @@ -426,26 +424,26 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha net, err := getNetwork(netClient, networkID) if err != nil { - return spec, osErrorToTerminalError(err, fmt.Sprintf("failed to get network for subnet defaulting '%s", networkID)) + return spec, osErrorToTerminalError(log, err, fmt.Sprintf("failed to get network for subnet defaulting '%s", networkID)) } subnet, err := getDefaultSubnet(netClient, net) if err != nil { - return spec, osErrorToTerminalError(err, "error defaulting subnet") + return spec, osErrorToTerminalError(log, err, "error defaulting subnet") } if subnet != nil { - klog.V(3).Infof("Defaulted subnet for machine '%s' to '%s'", spec.Name, *subnet) + log.Debugw("Defaulted subnet for machine", "subnet", *subnet) rawConfig.Subnet.Value = *subnet } } spec.ProviderSpec.Value, err = setProviderSpec(*rawConfig, spec.ProviderSpec) if err != nil { - return spec, osErrorToTerminalError(err, "error marshaling providerconfig") + return spec, osErrorToTerminalError(log, err, "error marshaling providerconfig") } return spec, nil } -func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { c, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -558,7 +556,7 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) return nil } -func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(_ context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { cfg, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -569,28 +567,28 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d client, err := p.clientGetter(cfg) if err != nil { - return nil, osErrorToTerminalError(err, "failed to get a openstack client") + return nil, osErrorToTerminalError(log, err, "failed to get a openstack client") } computeClient, err := getNewComputeV2(client, cfg) if err != nil { - return nil, osErrorToTerminalError(err, "failed to get a openstack client") + return nil, osErrorToTerminalError(log, err, "failed to get a openstack client") } flavor, err := getFlavor(computeClient, cfg) if err != nil { - return nil, osErrorToTerminalError(err, fmt.Sprintf("failed to get flavor %s", cfg.Flavor)) + return nil, osErrorToTerminalError(log, err, fmt.Sprintf("failed to get flavor %s", cfg.Flavor)) } // Get OS Image Client. imageClient, err := goopenstack.NewImageServiceV2(client, gophercloud.EndpointOpts{Region: cfg.Region}) if err != nil { - return nil, osErrorToTerminalError(err, "failed to get a image client") + return nil, osErrorToTerminalError(log, err, "failed to get a image client") } image, err := getImageByName(imageClient, cfg) if err != nil { - return nil, osErrorToTerminalError(err, fmt.Sprintf("failed to get image %s", cfg.Image)) + return nil, osErrorToTerminalError(log, err, fmt.Sprintf("failed to get image %s", cfg.Image)) } netClient, err := goopenstack.NewNetworkV2(client, gophercloud.EndpointOpts{Region: cfg.Region}) @@ -600,13 +598,13 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d network, err := getNetwork(netClient, cfg.Network) if err != nil { - return nil, osErrorToTerminalError(err, fmt.Sprintf("failed to get network %s", cfg.Network)) + return nil, osErrorToTerminalError(log, err, fmt.Sprintf("failed to get network %s", cfg.Network)) } securityGroups := cfg.SecurityGroups if len(securityGroups) == 0 { - klog.V(2).Infof("creating security group %s for worker nodes", securityGroupName) - err = ensureKubernetesSecurityGroupExist(client, cfg.Region, securityGroupName) + log.Infow("Creating security group for worker nodes", "group", securityGroupName) + err = ensureKubernetesSecurityGroupExist(log, client, cfg.Region, securityGroupName) if err != nil { return nil, fmt.Errorf("Error occurred creating security groups: %w", err) } @@ -658,7 +656,7 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d } if err := bootfromvolume.Create(computeClient, createOpts).ExtractInto(&server); err != nil { - return nil, osErrorToTerminalError(err, "failed to create server with volume") + return nil, osErrorToTerminalError(log, err, "failed to create server with volume") } } else { // Image ID should only be set in server options when block device @@ -667,18 +665,20 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d serverOpts.ImageRef = image.ID if err := osservers.Create(computeClient, createOpts).ExtractInto(&server); err != nil { - return nil, osErrorToTerminalError(err, "failed to create server") + return nil, osErrorToTerminalError(log, err, "failed to create server") } } if cfg.FloatingIPPool != "" { - if err := p.portReadinessWaiter(netClient, server.ID, network.ID, cfg.InstanceReadyCheckPeriod, cfg.InstanceReadyCheckTimeout); err != nil { - klog.V(2).Infof("port for instance %q did not became active due to: %v", server.ID, err) + instanceLog := log.With("instance", server.ID) + + if err := p.portReadinessWaiter(instanceLog, netClient, server.ID, network.ID, cfg.InstanceReadyCheckPeriod, cfg.InstanceReadyCheckTimeout); err != nil { + instanceLog.Infow("Port for instance did not became active", zap.Error(err)) } // Find a free FloatingIP or allocate a new one. - if err := assignFloatingIPToInstance(data.Update, machine, netClient, server.ID, cfg.FloatingIPPool, cfg.Region, network); err != nil { - defer deleteInstanceDueToFatalLogged(computeClient, server.ID) + if err := assignFloatingIPToInstance(instanceLog, data.Update, machine, netClient, server.ID, cfg.FloatingIPPool, cfg.Region, network); err != nil { + defer deleteInstanceDueToFatalLogged(instanceLog, computeClient, server.ID) return nil, fmt.Errorf("failed to assign a floating ip to instance %s: %w", server.ID, err) } } @@ -686,19 +686,19 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d return &osInstance{server: &server}, nil } -func waitForPort(netClient *gophercloud.ServiceClient, serverID string, networkID string, checkPeriod time.Duration, checkTimeout time.Duration) error { +func waitForPort(instanceLog *zap.SugaredLogger, netClient *gophercloud.ServiceClient, serverID string, networkID string, checkPeriod time.Duration, checkTimeout time.Duration) error { started := time.Now() - klog.V(2).Infof("Waiting for the port of instance %s to become active...", serverID) + instanceLog.Info("Waiting for the port to become active...") portIsReady := func() (bool, error) { port, err := getInstancePort(netClient, serverID, networkID) if err != nil { - tErr := osErrorToTerminalError(err, fmt.Sprintf("failed to get current instance port %s", serverID)) + tErr := osErrorToTerminalError(instanceLog, err, fmt.Sprintf("failed to get current instance port %s", serverID)) if isTerminalErr, _, _ := cloudprovidererrors.IsTerminalError(tErr); isTerminalErr { return true, tErr } // Only log the error but don't exit. in case of a network failure we want to retry. - klog.V(2).Infof("failed to get current instance port %s: %v", serverID, err) + instanceLog.Errorw("Failed to get current instance port", zap.Error(err)) return false, nil } @@ -714,30 +714,30 @@ func waitForPort(netClient *gophercloud.ServiceClient, serverID string, networkI return fmt.Errorf("failed to wait for instance port to become active: %w", err) } - klog.V(2).Infof("Instance %q port became active after %f seconds", serverID, time.Since(started).Seconds()) + instanceLog.Infow("Instance port became active", "elapsed", time.Since(started).Round(time.Second)) return nil } -func deleteInstanceDueToFatalLogged(computeClient *gophercloud.ServiceClient, serverID string) { - klog.V(0).Infof("Deleting instance %s due to fatal error during machine creation...", serverID) +func deleteInstanceDueToFatalLogged(instanceLog *zap.SugaredLogger, computeClient *gophercloud.ServiceClient, serverID string) { + instanceLog.Info("Deleting instance due to fatal error during machine creation...") if err := osservers.Delete(computeClient, serverID).ExtractErr(); err != nil { utilruntime.HandleError(fmt.Errorf("failed to delete the instance %s. Please take care of manually deleting the instance: %w", serverID, err)) return } - klog.V(0).Infof("Instance %s got deleted", serverID) + instanceLog.Info("Instance got deleted") } -func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { var hasFloatingIPReleaseFinalizer bool if finalizers := sets.NewString(machine.Finalizers...); finalizers.Has(floatingIPReleaseFinalizer) { hasFloatingIPReleaseFinalizer = true } - instance, err := p.Get(ctx, machine, data) + instance, err := p.Get(ctx, log, machine, data) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { if hasFloatingIPReleaseFinalizer { - if err := p.cleanupFloatingIP(machine, data.Update); err != nil { + if err := p.cleanupFloatingIP(log, machine, data.Update); err != nil { return false, fmt.Errorf("failed to clean up floating ip: %w", err) } } @@ -756,26 +756,26 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine client, err := p.clientGetter(c) if err != nil { - return false, osErrorToTerminalError(err, "failed to get a openstack client") + return false, osErrorToTerminalError(log, err, "failed to get a openstack client") } computeClient, err := getNewComputeV2(client, c) if err != nil { - return false, osErrorToTerminalError(err, "failed to get compute client") + return false, osErrorToTerminalError(log, err, "failed to get compute client") } if err := osservers.Delete(computeClient, instance.ID()).ExtractErr(); err != nil && !errors.As(err, &gophercloud.ErrDefault404{}) { - return false, osErrorToTerminalError(err, "failed to delete instance") + return false, osErrorToTerminalError(log, err, "failed to delete instance") } if hasFloatingIPReleaseFinalizer { - return false, p.cleanupFloatingIP(machine, data.Update) + return false, p.cleanupFloatingIP(log, machine, data.Update) } return false, nil } -func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(_ context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -786,12 +786,12 @@ func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, _ *c client, err := p.clientGetter(c) if err != nil { - return nil, osErrorToTerminalError(err, "failed to get a openstack client") + return nil, osErrorToTerminalError(log, err, "failed to get a openstack client") } computeClient, err := getNewComputeV2(client, c) if err != nil { - return nil, osErrorToTerminalError(err, "failed to get compute client") + return nil, osErrorToTerminalError(log, err, "failed to get compute client") } var allServers []serverWithExt @@ -800,13 +800,13 @@ func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, _ *c var servers []serverWithExt err = osservers.ExtractServersInto(page, &servers) if err != nil { - return false, osErrorToTerminalError(err, "failed to extract instance info") + return false, osErrorToTerminalError(log, err, "failed to extract instance info") } allServers = append(allServers, servers...) return true, nil }) if err != nil { - return nil, osErrorToTerminalError(err, "failed to list instances") + return nil, osErrorToTerminalError(log, err, "failed to list instances") } for i, s := range allServers { @@ -818,7 +818,7 @@ func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, _ *c return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *provider) MigrateUID(_ context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return cloudprovidererrors.TerminalError{ @@ -829,12 +829,12 @@ func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machin client, err := p.clientGetter(c) if err != nil { - return osErrorToTerminalError(err, "failed to get a openstack client") + return osErrorToTerminalError(log, err, "failed to get a openstack client") } computeClient, err := getNewComputeV2(client, c) if err != nil { - return osErrorToTerminalError(err, "failed to get compute client") + return osErrorToTerminalError(log, err, "failed to get compute client") } var allServers []serverWithExt @@ -843,13 +843,13 @@ func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machin var servers []serverWithExt err = osservers.ExtractServersInto(page, &servers) if err != nil { - return false, osErrorToTerminalError(err, "failed to extract instance info") + return false, osErrorToTerminalError(log, err, "failed to extract instance info") } allServers = append(allServers, servers...) return true, nil }) if err != nil { - return osErrorToTerminalError(err, "failed to list instances") + return osErrorToTerminalError(log, err, "failed to list instances") } for _, s := range allServers { @@ -966,7 +966,7 @@ func (d *osInstance) Status() instance.Status { // can be qualified as a "terminal" error, for more info see v1alpha1.MachineStatus // // if the given error doesn't qualify the error passed as an argument will be returned. -func osErrorToTerminalError(err error, msg string) error { +func osErrorToTerminalError(log *zap.SugaredLogger, err error, msg string) error { var errUnauthorized gophercloud.ErrDefault401 if errors.As(err, &errUnauthorized) { return cloudprovidererrors.TerminalError{ @@ -986,7 +986,7 @@ func osErrorToTerminalError(err error, msg string) error { info := &forbiddenResponse{} if err := json.Unmarshal(errForbidden.Body, info); err != nil { // We just log here as we just do this to make the response more pretty - klog.V(0).Infof("failed to unmarshal response body from 403 response from OpenStack API: %v\n%s", err, errForbidden.Body) + log.Errorw("Failed to unmarshal response body from 403 response from OpenStack API", "body", errForbidden.Body, zap.Error(err)) return terr } @@ -1001,7 +1001,7 @@ func osErrorToTerminalError(err error, msg string) error { return terr } - return fmt.Errorf("%s, due to %w", msg, err) + return fmt.Errorf("%s: %w", msg, err) } // forbiddenResponse is a potential response body from the OpenStack API when the request is forbidden (code: 403). @@ -1012,10 +1012,10 @@ type forbiddenResponse struct { } `json:"forbidden"` } -func (p *provider) cleanupFloatingIP(machine *clusterv1alpha1.Machine, updater cloudprovidertypes.MachineUpdater) error { +func (p *provider) cleanupFloatingIP(log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, updater cloudprovidertypes.MachineUpdater) error { floatingIPID, exists := machine.Annotations[floatingIPIDAnnotationKey] if !exists { - return osErrorToTerminalError(fmt.Errorf("failed to release floating ip"), + return osErrorToTerminalError(log, fmt.Errorf("failed to release floating ip"), fmt.Sprintf("%s finalizer exists but %s annotation does not", floatingIPReleaseFinalizer, floatingIPIDAnnotationKey)) } @@ -1029,7 +1029,7 @@ func (p *provider) cleanupFloatingIP(machine *clusterv1alpha1.Machine, updater c client, err := p.clientGetter(c) if err != nil { - return osErrorToTerminalError(err, "failed to get a openstack client") + return osErrorToTerminalError(log, err, "failed to get a openstack client") } netClient, err := goopenstack.NewNetworkV2(client, gophercloud.EndpointOpts{Region: c.Region}) if err != nil { @@ -1049,7 +1049,7 @@ func (p *provider) cleanupFloatingIP(machine *clusterv1alpha1.Machine, updater c return nil } -func assignFloatingIPToInstance(machineUpdater cloudprovidertypes.MachineUpdater, machine *clusterv1alpha1.Machine, netClient *gophercloud.ServiceClient, instanceID, floatingIPPoolName, region string, network *osnetworks.Network) error { +func assignFloatingIPToInstance(instanceLog *zap.SugaredLogger, machineUpdater cloudprovidertypes.MachineUpdater, machine *clusterv1alpha1.Machine, netClient *gophercloud.ServiceClient, instanceID, floatingIPPoolName, region string, network *osnetworks.Network) error { port, err := getInstancePort(netClient, instanceID, network.ID) if err != nil { return fmt.Errorf("failed to get instance port for network %s in region %s: %w", network.ID, region, err) @@ -1057,25 +1057,23 @@ func assignFloatingIPToInstance(machineUpdater cloudprovidertypes.MachineUpdater floatingIPPool, err := getNetwork(netClient, floatingIPPoolName) if err != nil { - return osErrorToTerminalError(err, fmt.Sprintf("failed to get floating ip pool %q", floatingIPPoolName)) + return osErrorToTerminalError(instanceLog, err, fmt.Sprintf("failed to get floating IP pool %q", floatingIPPoolName)) } - // We're only interested in the part which is vulnerable to concurrent access - started := time.Now() - klog.V(2).Infof("Assigning a floating IP to instance %s", instanceID) + instanceLog.Info("Assigning a floating IP to instance") floatingIPAssignLock.Lock() defer floatingIPAssignLock.Unlock() freeFloatingIps, err := getFreeFloatingIPs(netClient, floatingIPPool) if err != nil { - return osErrorToTerminalError(err, "failed to get free floating ips") + return osErrorToTerminalError(instanceLog, err, "failed to get free floating ips") } var ip *osfloatingips.FloatingIP if len(freeFloatingIps) < 1 { if ip, err = createFloatingIP(netClient, port.ID, floatingIPPool); err != nil { - return osErrorToTerminalError(err, "failed to allocate a floating ip") + return osErrorToTerminalError(instanceLog, err, "failed to allocate a floating ip") } if err := machineUpdater(machine, func(m *clusterv1alpha1.Machine) { m.Finalizers = append(m.Finalizers, floatingIPReleaseFinalizer) @@ -1092,7 +1090,7 @@ func assignFloatingIPToInstance(machineUpdater cloudprovidertypes.MachineUpdater PortID: &port.ID, }).Extract() if err != nil { - return fmt.Errorf("failed to update FloatingIP %s(%s): %w", freeIP.ID, freeIP.FloatingIP, err) + return fmt.Errorf("failed to update floating IP %s(%s): %w", freeIP.ID, freeIP.FloatingIP, err) } // We're now going to wait 3 seconds and check if the IP is still ours. If not, we're going to fail @@ -1100,16 +1098,15 @@ func assignFloatingIPToInstance(machineUpdater cloudprovidertypes.MachineUpdater time.Sleep(floatingReassignIPCheckPeriod) currentIP, err := osfloatingips.Get(netClient, ip.ID).Extract() if err != nil { - return fmt.Errorf("failed to load FloatingIP %s after assignment has been done: %w", ip.FloatingIP, err) + return fmt.Errorf("failed to load floating IP %s after assignment has been done: %w", ip.FloatingIP, err) } // Verify if the port is still the one we set it to if currentIP.PortID != port.ID { return fmt.Errorf("floatingIP %s got reassigned", currentIP.FloatingIP) } } - secondsTook := time.Since(started).Seconds() - klog.V(2).Infof("Successfully assigned the FloatingIP %s to instance %s. Took %f seconds(without the recheck wait period %f seconds). ", ip.FloatingIP, instanceID, secondsTook, floatingReassignIPCheckPeriod.Seconds()) + instanceLog.Infow("Successfully assigned the floating IP to instance", "ip", ip.FloatingIP) return nil } diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index 5b4583daa..4b7b056fd 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -30,6 +30,7 @@ import ( "github.com/gophercloud/gophercloud/openstack/compute/v2/servers" th "github.com/gophercloud/gophercloud/testhelper" "github.com/gophercloud/gophercloud/testhelper/client" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidertesting "github.com/kubermatic/machine-controller/pkg/cloudprovider/testing" @@ -281,7 +282,7 @@ func TestCreateServer(t *testing.T) { return pc.ProviderClient, nil }, // mock server readiness checker - portReadinessWaiter: func(*gophercloud.ServiceClient, string, string, time.Duration, time.Duration) error { + portReadinessWaiter: func(*zap.SugaredLogger, *gophercloud.ServiceClient, string, string, time.Duration, time.Duration) error { return nil }, } @@ -295,7 +296,7 @@ func TestCreateServer(t *testing.T) { // It only verifies that the content of the create request matches // the expectation // TODO(irozzo) check the returned instance too - _, err := p.Create(context.Background(), m, tt.data, tt.userdata) + _, err := p.Create(context.Background(), zap.NewNop().Sugar(), m, tt.data, tt.userdata) if (err != nil) != tt.wantErr { t.Errorf("provider.Create() or = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go index 6a2aa4887..2916b671e 100644 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ b/pkg/cloudprovider/provider/scaleway/provider.go @@ -25,6 +25,7 @@ import ( "github.com/scaleway/scaleway-sdk-go/api/instance/v1" "github.com/scaleway/scaleway-sdk-go/scw" "github.com/scaleway/scaleway-sdk-go/validation" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -38,7 +39,6 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/klog" ) type provider struct { @@ -135,11 +135,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return &c, pconfig, err } -func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } -func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -172,7 +172,7 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return nil } -func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (cloudInstance.Instance, error) { +func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (cloudInstance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -215,12 +215,12 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, scalewayErrToTerminalError(err) } - klog.V(6).Infof("Scaleway server (id='%s') got fully created", serverResp.Server.ID) + log.Debugw("Scaleway server got fully created", "server", serverResp.Server.ID) return &scwServer{server: serverResp.Server}, err } -func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { i, err := p.get(machine) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { @@ -252,7 +252,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine return false, nil } -func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (cloudInstance.Instance, error) { +func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (cloudInstance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -317,7 +317,7 @@ func (p *provider) get(machine *clusterv1alpha1.Machine) (*scwServer, error) { return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to decode providerconfig: %w", err) diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/client.go b/pkg/cloudprovider/provider/vmwareclouddirector/client.go index a714abda2..c7f6e7878 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/client.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/client.go @@ -82,7 +82,7 @@ func (c *Client) GetAuthenticatedClient() (*govcd.VCDClient, error) { // Ensure that `/api` suffix exists in the cloud director URL. apiEndpoint, err := url.Parse(c.Auth.URL) if err != nil { - return nil, fmt.Errorf("unable to parse url '%s': %w", c.Auth.URL, err) + return nil, fmt.Errorf("failed to parse url '%s': %w", c.Auth.URL, err) } if !strings.HasSuffix(c.Auth.URL, "/api") { apiEndpoint.Path = path.Join(apiEndpoint.Path, "api") diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go index 1b21f0ef9..99b5064a6 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go @@ -177,12 +177,12 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * task, err := client.VCDClient.Client.ExecuteTaskRequest(apiEndpoint.String(), http.MethodPost, types.MimeRecomposeVappParams, "error instantiating a new VM: %s", vAppRecomposition) if err != nil { - return fmt.Errorf("unable to execute API call to create VM: %w", err) + return fmt.Errorf("failed to execute API call to create VM: %w", err) } // Wait for VM to be created this should take around 1-3 minutes if err = task.WaitTaskCompletion(); err != nil { - return fmt.Errorf("error waiting for VM creation task to complete: %w", err) + return fmt.Errorf("failed to wait for VM creation task to complete: %w", err) } return nil } diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go index 0e568548b..5da253d8c 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -24,6 +24,7 @@ import ( "net/url" "github.com/vmware/go-vcloud-director/v2/govcd" + "go.uber.org/zap" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" @@ -133,7 +134,7 @@ func (s Server) Status() instance.Status { return s.status } -func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { _, _, rawConfig, err := p.getConfig(spec.ProviderSpec) if err != nil { return spec, err @@ -155,7 +156,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, err } -func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, fmt.Errorf("failed to parse config: %w", err) @@ -196,10 +197,10 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine return true, nil } -func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { vm, err := p.create(ctx, machine, userdata) if err != nil { - _, cleanupErr := p.Cleanup(ctx, machine, data) + _, cleanupErr := p.Cleanup(ctx, log, machine, data) if cleanupErr != nil { return nil, fmt.Errorf("cleaning up failed with err %w after creation failed with err %w", cleanupErr, err) } @@ -286,7 +287,7 @@ func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, return p.getInstance(vm) } -func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("failed to parse config: %w", err) @@ -467,7 +468,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } -func (p *provider) MigrateUID(_ context.Context, _ *clusterv1alpha1.Machine, _ types.UID) error { +func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ types.UID) error { return nil } @@ -475,7 +476,7 @@ func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) e return nil } -func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { c, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go index db08f9dbc..a3d5925c5 100644 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ b/pkg/cloudprovider/provider/vsphere/helper.go @@ -32,11 +32,11 @@ import ( "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/soap" "github.com/vmware/govmomi/vim25/types" + "go.uber.org/zap" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/klog" ) const ( @@ -45,7 +45,7 @@ const ( local-hostname: {{ .Hostname }}` ) -func createClonedVM(ctx context.Context, vmName string, config *Config, session *Session, os providerconfigtypes.OperatingSystem, containerLinuxUserdata string) (*object.VirtualMachine, error) { +func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, config *Config, session *Session, os providerconfigtypes.OperatingSystem, containerLinuxUserdata string) (*object.VirtualMachine, error) { tpl, err := session.Finder.VirtualMachine(ctx, config.TemplateVMName) if err != nil { return nil, fmt.Errorf("failed to get template vm: %w", err) @@ -82,7 +82,7 @@ func createClonedVM(ctx context.Context, vmName string, config *Config, session Template: false, Location: relocateSpec, } - datastoreref, err := resolveDatastoreRef(ctx, config, session, tpl, targetVMFolder, &cloneSpec) + datastoreref, err := resolveDatastoreRef(ctx, log, config, session, tpl, targetVMFolder, &cloneSpec) if err != nil { return nil, fmt.Errorf("failed to resolve datastore: %w", err) } @@ -185,7 +185,7 @@ func createClonedVM(ctx context.Context, vmName string, config *Config, session return nil, err } - klog.V(4).Infof("Increasing disk size to %d GB", *config.DiskSizeGB) + log.Debugw("Increasing disk size", "targetgb", *config.DiskSizeGB) disk := disks[0] disk.CapacityInBytes = *config.DiskSizeGB * int64(math.Pow(1024, 3)) diskspec := &types.VirtualDeviceConfigSpec{Operation: types.VirtualDeviceConfigSpecOperationEdit, Device: disk} @@ -227,10 +227,12 @@ func createClonedVM(ctx context.Context, vmName string, config *Config, session return virtualMachine, nil } -func resolveDatastoreRef(ctx context.Context, config *Config, session *Session, vm *object.VirtualMachine, folder *object.Folder, cloneSpec *types.VirtualMachineCloneSpec) (*types.ManagedObjectReference, error) { +func resolveDatastoreRef(ctx context.Context, log *zap.SugaredLogger, config *Config, session *Session, vm *object.VirtualMachine, folder *object.Folder, cloneSpec *types.VirtualMachineCloneSpec) (*types.ManagedObjectReference, error) { // Based on https://github.com/vmware/govmomi/blob/v0.22.1/govc/vm/clone.go#L358 if config.DatastoreCluster != "" && config.Datastore == "" { - klog.Infof("Choosing initial datastore placement for vm %s from datastore cluster %s", vm.Name(), config.DatastoreCluster) + vmLog := log.With("vm", vm.Name(), "datastorecluster", config.DatastoreCluster) + vmLog.Infow("Choosing initial datastore placement for vm from datastore cluster") + storagePod, err := session.Finder.DatastoreCluster(ctx, config.DatastoreCluster) if err != nil { return nil, fmt.Errorf("failed to get datastore cluster: %w", err) @@ -273,7 +275,8 @@ func resolveDatastoreRef(ctx context.Context, config *Config, session *Session, // Get the first recommendation ds := recommendations[0].Action[0].(*types.StoragePlacementAction).Destination.Reference() - klog.Infof("The selected datastore from datastore cluster %s is: %v", config.DatastoreCluster, ds) + vmLog.Infow("Selected datastore from datastore cluster", "datastore", ds) + return &ds, nil } else if config.DatastoreCluster == "" && config.Datastore != "" { datastore, err := session.Finder.Datastore(ctx, config.Datastore) @@ -286,7 +289,7 @@ func resolveDatastoreRef(ctx context.Context, config *Config, session *Session, } } -func uploadAndAttachISO(ctx context.Context, session *Session, vmRef *object.VirtualMachine, localIsoFilePath string) error { +func uploadAndAttachISO(ctx context.Context, log *zap.SugaredLogger, session *Session, vmRef *object.VirtualMachine, localIsoFilePath string) error { p := soap.DefaultUpload remoteIsoFilePath := fmt.Sprintf("%s/%s", vmRef.Name(), "cloud-init.iso") // Get the datastore where VM files are located @@ -294,11 +297,12 @@ func uploadAndAttachISO(ctx context.Context, session *Session, vmRef *object.Vir if err != nil { return fmt.Errorf("error getting datastore from VM %s: %w", vmRef.Name(), err) } - klog.V(3).Infof("Uploading userdata ISO to datastore %+v, destination iso is %s\n", datastore, remoteIsoFilePath) + uploadLog := log.With("datastore", datastore, "source", localIsoFilePath, "destination", remoteIsoFilePath) + uploadLog.Debug("Uploading userdata ISO to datastore") if err := datastore.UploadFile(ctx, localIsoFilePath, remoteIsoFilePath, &p); err != nil { return fmt.Errorf("failed to upload iso: %w", err) } - klog.V(3).Infof("Uploaded ISO file %s", localIsoFilePath) + uploadLog.Debug("Uploaded ISO file") // Find the cd-rom device and insert the cloud init iso file into it. devices, err := vmRef.Device(ctx) @@ -456,14 +460,14 @@ func resolveResourcePoolRef(ctx context.Context, config *Config, session *Sessio return nil, nil } -func attachTags(ctx context.Context, config *Config, vm *object.VirtualMachine) error { +func attachTags(ctx context.Context, log *zap.SugaredLogger, config *Config, vm *object.VirtualMachine) error { restAPISession, err := NewRESTSession(ctx, config) if err != nil { return fmt.Errorf("failed to create REST API session: %w", err) } defer restAPISession.Logout(ctx) tagManager := tags.NewManager(restAPISession.Client) - klog.V(3).Info("Attaching tags") + log.Debug("Attaching tags") for _, tag := range config.Tags { tagID, err := determineTagID(ctx, tagManager, tag) if err != nil { @@ -471,14 +475,14 @@ func attachTags(ctx context.Context, config *Config, vm *object.VirtualMachine) } if err := tagManager.AttachTag(ctx, tagID, vm.Reference()); err != nil { - klog.V(3).Infof("Failed to attach tag %v. The tag was successfully deleted", tag) + log.Debugw("Failed to attach tag; it was successfully deleted", "tag", tag) return fmt.Errorf("failed to attach tag to VM: %v %w", tag.Name, err) } } return nil } -func detachTags(ctx context.Context, config *Config, vm *object.VirtualMachine) error { +func detachTags(ctx context.Context, log *zap.SugaredLogger, config *Config, vm *object.VirtualMachine) error { restAPISession, err := NewRESTSession(ctx, config) if err != nil { return fmt.Errorf("failed to create REST API session: %w", err) @@ -490,7 +494,7 @@ func detachTags(ctx context.Context, config *Config, vm *object.VirtualMachine) if err != nil { return fmt.Errorf("failed to get attached tags for the VM: %s, %w", vm.Name(), err) } - klog.V(3).Info("Deleting tags") + log.Debug("Deleting tags") for _, tag := range attachedTags { tagID, err := determineTagID(ctx, tagManager, tag) if err != nil { diff --git a/pkg/cloudprovider/provider/vsphere/helper_test.go b/pkg/cloudprovider/provider/vsphere/helper_test.go index 359d867e7..783e92d24 100644 --- a/pkg/cloudprovider/provider/vsphere/helper_test.go +++ b/pkg/cloudprovider/provider/vsphere/helper_test.go @@ -28,6 +28,7 @@ import ( "github.com/vmware/govmomi/vim25/methods" "github.com/vmware/govmomi/vim25/soap" "github.com/vmware/govmomi/vim25/types" + "go.uber.org/zap" ) func TestResolveDatastoreRef(t *testing.T) { @@ -112,7 +113,7 @@ func TestResolveDatastoreRef(t *testing.T) { t.Fatalf("error getting virtual machines: %v", err) } - got, err := resolveDatastoreRef(ctx, tt.config, session, vms[2], vmFolder, &types.VirtualMachineCloneSpec{}) + got, err := resolveDatastoreRef(ctx, zap.NewNop().Sugar(), tt.config, session, vms[2], vmFolder, &types.VirtualMachineCloneSpec{}) if (err != nil) != tt.wantErr { t.Errorf("resolveDatastoreRef() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index f5cee0dc9..62efd7871 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -30,6 +30,7 @@ import ( "github.com/vmware/govmomi/vapi/tags" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -43,7 +44,6 @@ import ( corev1 "k8s.io/api/core/v1" ktypes "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" - "k8s.io/klog" ) type provider struct { @@ -110,7 +110,7 @@ func (vsphereServer Server) Status() instance.Status { // Ensures that provider implements Provider interface. var _ cloudprovidertypes.Provider = &provider{} -func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } @@ -205,7 +205,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return &c, pconfig, rawConfig, nil } -func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, log *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { config, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to get config: %w", err) @@ -224,7 +224,7 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe } defer restAPISession.Logout(ctx) tagManager := tags.NewManager(restAPISession.Client) - klog.V(3).Info("Found tags") + log.Debug("Found tags") for _, tag := range config.Tags { if tag.ID == "" && tag.Name == "" { return fmt.Errorf("either tag id or name must be specified") @@ -236,7 +236,7 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return fmt.Errorf("can't get the category with ID %s, %w", tag.CategoryID, err) } } - klog.V(3).Info("Tag validation passed") + log.Debug("Tag validation passed") } // Only and only one between datastore and datastre cluster should be @@ -295,10 +295,10 @@ func machineInvalidConfigurationTerminalError(err error) error { } } -func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - vm, err := p.create(ctx, machine, userdata) +func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { + vm, err := p.create(ctx, log, machine, userdata) if err != nil { - _, cleanupErr := p.Cleanup(ctx, machine, data) + _, cleanupErr := p.Cleanup(ctx, log, machine, data) if cleanupErr != nil { return nil, fmt.Errorf("cleaning up failed with err %w after creation failed with err %w", cleanupErr, err) } @@ -307,7 +307,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return vm, nil } -func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { +func (p *provider) create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("failed to parse config: %w", err) @@ -324,7 +324,9 @@ func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, containerLinuxUserdata = userdata } - virtualMachine, err := createClonedVM(ctx, + virtualMachine, err := createClonedVM( + ctx, + log, machine.Spec.Name, config, session, @@ -335,7 +337,7 @@ func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, machineInvalidConfigurationTerminalError(fmt.Errorf("failed to create cloned vm: '%w'", err)) } - if err := attachTags(ctx, config, virtualMachine); err != nil { + if err := attachTags(ctx, log, config, virtualMachine); err != nil { return nil, fmt.Errorf("failed to attach tags: %w", err) } @@ -352,7 +354,7 @@ func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, } }() - if err := uploadAndAttachISO(ctx, session, virtualMachine, localUserdataIsoFilePath); err != nil { + if err := uploadAndAttachISO(ctx, log, session, virtualMachine, localUserdataIsoFilePath); err != nil { // Destroy VM to avoid a leftover. destroyTask, vmErr := virtualMachine.Destroy(ctx) if vmErr != nil { @@ -377,7 +379,7 @@ func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, return Server{name: virtualMachine.Name(), status: instance.StatusRunning, id: virtualMachine.Reference().Value, uuid: virtualMachine.UUID(ctx)}, nil } -func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, fmt.Errorf("failed to parse config: %w", err) @@ -397,7 +399,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine return false, fmt.Errorf("failed to get instance from vSphere: %w", err) } - if err := detachTags(ctx, config, virtualMachine); err != nil { + if err := detachTags(ctx, log, config, virtualMachine); err != nil { return false, fmt.Errorf("failed to delete tags: %w", err) } @@ -467,11 +469,11 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine } } - klog.V(2).Infof("Successfully destroyed vm %s", virtualMachine.Name()) + log.Infow("Successfully destroyed vm", "vm", virtualMachine.Name()) return true, nil } -func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("failed to parse config: %w", err) @@ -529,13 +531,13 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, da } } } else { - klog.V(3).Infof("Can't fetch the IP addresses for machine %s, the VMware guest utils are not running yet. This might take a few minutes", machine.Spec.Name) + log.Debug("Can't fetch the IP addresses for machine, the VMware guest utils are not running yet. This might take a few minutes") } return Server{name: virtualMachine.Name(), status: instance.StatusRunning, addresses: addresses, id: virtualMachine.Reference().Value, uuid: virtualMachine.UUID(ctx)}, nil } -func (p *provider) MigrateUID(_ context.Context, _ *clusterv1alpha1.Machine, _ ktypes.UID) error { +func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ ktypes.UID) error { return nil } diff --git a/pkg/cloudprovider/provider/vsphere/provider_test.go b/pkg/cloudprovider/provider/vsphere/provider_test.go index 3c28e66bf..e457a5506 100644 --- a/pkg/cloudprovider/provider/vsphere/provider_test.go +++ b/pkg/cloudprovider/provider/vsphere/provider_test.go @@ -25,6 +25,7 @@ import ( "text/template" "github.com/vmware/govmomi/simulator" + "go.uber.org/zap" cloudprovidertesting "github.com/kubermatic/machine-controller/pkg/cloudprovider/testing" "github.com/kubermatic/machine-controller/pkg/providerconfig" @@ -178,7 +179,7 @@ func TestValidate(t *testing.T) { tt.args.URL = vSphereURL m := cloudprovidertesting.Creator{Name: "test", Namespace: "vsphere", ProviderSpecGetter: tt.args.rawProviderSpec}. CreateMachine(t) - if err := p.Validate(context.Background(), m.Spec); (err != nil) != tt.wantErr { + if err := p.Validate(context.Background(), zap.NewNop().Sugar(), m.Spec); (err != nil) != tt.wantErr { t.Errorf("provider.Validate() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/pkg/cloudprovider/provider/vultr/provider.go b/pkg/cloudprovider/provider/vultr/provider.go index 739036c01..3f4e63d5a 100644 --- a/pkg/cloudprovider/provider/vultr/provider.go +++ b/pkg/cloudprovider/provider/vultr/provider.go @@ -23,12 +23,12 @@ import ( "strconv" "github.com/vultr/govultr/v2" + "go.uber.org/zap" "golang.org/x/oauth2" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" vultrtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vultr/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" @@ -122,11 +122,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return &c, pconfig, err } -func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return spec, nil } -func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -205,7 +205,7 @@ func (p *provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (* return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { return p.get(ctx, machine) } @@ -213,7 +213,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri return "", "", nil } -func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -263,8 +263,8 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return &vultrInstance{instance: res}, nil } -func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { - instance, err := p.Get(ctx, machine, data) +func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { + instance, err := p.Get(ctx, log, machine, data) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return true, nil @@ -300,7 +300,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } -func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *provider) MigrateUID(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to decode providerconfig: %w", err) diff --git a/pkg/cloudprovider/types/types.go b/pkg/cloudprovider/types/types.go index adc9010b1..e64a6bf33 100644 --- a/pkg/cloudprovider/types/types.go +++ b/pkg/cloudprovider/types/types.go @@ -18,6 +18,9 @@ package types import ( "context" + "fmt" + + "go.uber.org/zap" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" @@ -31,13 +34,13 @@ import ( // Provider exposed all required functions to interact with a cloud provider. type Provider interface { // AddDefaults will read the MachineSpec and apply defaults for provider specific fields - AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) + AddDefaults(log *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) // Validate validates the given machine's specification. // // In case of any error a "terminal" error should be set, // See v1alpha1.MachineStatus for more info - Validate(ctx context.Context, machinespec clusterv1alpha1.MachineSpec) error + Validate(ctx context.Context, log *zap.SugaredLogger, machinespec clusterv1alpha1.MachineSpec) error // Get gets a node that is associated with the given machine. // @@ -46,19 +49,19 @@ type Provider interface { // See v1alpha1.MachineStatus for more info and TerminalError type // // In case the instance cannot be found, github.com/kubermatic/machine-controller/pkg/cloudprovider/errors/ErrInstanceNotFound will be returned - Get(ctx context.Context, machine *clusterv1alpha1.Machine, data *ProviderData) (instance.Instance, error) + Get(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *ProviderData) (instance.Instance, error) // GetCloudConfig will return the cloud provider specific cloud-config, which gets consumed by the kubelet GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) // Create creates a cloud instance according to the given machine - Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *ProviderData, userdata string) (instance.Instance, error) + Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *ProviderData, userdata string) (instance.Instance, error) // Cleanup will delete the instance associated with the machine and all associated resources. // If all resources have been cleaned up, true will be returned. // In case the cleanup involves asynchronous deletion of resources & those resources are not gone yet, // false should be returned. This is to indicate that the cleanup is not done, but needs to be called again at a later point - Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *ProviderData) (bool, error) + Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *ProviderData) (bool, error) // MachineMetricsLabels returns labels used for the Prometheus metrics // about created machines, e.g. instance type, instance size, region @@ -69,7 +72,7 @@ type Provider interface { // MigrateUID is called when the controller migrates types and the UID of the machine object changes // All cloud providers that use Machine.UID to uniquely identify resources must implement this - MigrateUID(ctx context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error + MigrateUID(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error // SetMetricsForMachines allows providers to provide provider-specific metrics. This may be implemented // as no-op @@ -100,7 +103,7 @@ func GetMachineUpdater(ctx context.Context, client ctrlruntimeclient.Client) Mac namespacedName := types.NamespacedName{Namespace: machine.Namespace, Name: machine.Name} return retry.RetryOnConflict(retry.DefaultBackoff, func() error { if err := client.Get(ctx, namespacedName, machine); err != nil { - return err + return fmt.Errorf("failed to get machine: %w", err) } // Check if we actually change something and only update if that is the case. diff --git a/pkg/cloudprovider/validationwrapper.go b/pkg/cloudprovider/validationwrapper.go index 77302b075..1b60044aa 100644 --- a/pkg/cloudprovider/validationwrapper.go +++ b/pkg/cloudprovider/validationwrapper.go @@ -21,12 +21,13 @@ import ( "errors" "fmt" + "go.uber.org/zap" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "k8s.io/apimachinery/pkg/types" - "k8s.io/klog" ) type cachingValidationWrapper struct { @@ -39,24 +40,24 @@ func NewValidationCacheWrappingCloudProvider(actualProvider cloudprovidertypes.P } // AddDefaults just calls the underlying cloudproviders AddDefaults. -func (w *cachingValidationWrapper) AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { - return w.actualProvider.AddDefaults(spec) +func (w *cachingValidationWrapper) AddDefaults(log *zap.SugaredLogger, spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { + return w.actualProvider.AddDefaults(log, spec) } // Validate tries to get the validation result from the cache and if not found, calls the // cloudproviders Validate and saves that to the cache. -func (w *cachingValidationWrapper) Validate(ctx context.Context, spec v1alpha1.MachineSpec) error { +func (w *cachingValidationWrapper) Validate(ctx context.Context, log *zap.SugaredLogger, spec v1alpha1.MachineSpec) error { result, exists, err := cache.Get(spec) if err != nil { return fmt.Errorf("error getting validation result from cache: %w", err) } if exists { - klog.V(6).Infof("Got cache hit for validation") + log.Debug("Got cache hit for validation") return result } - klog.V(6).Infof("Got cache miss for validation") - err = w.actualProvider.Validate(ctx, spec) + log.Debug("Got cache miss for validation") + err = w.actualProvider.Validate(ctx, log, spec) // do not cache canceled contexts (e.g. the validation request was canceled client-side) // and timeouts (assumed to be temporary) @@ -70,8 +71,8 @@ func (w *cachingValidationWrapper) Validate(ctx context.Context, spec v1alpha1.M } // Get just calls the underlying cloudproviders Get. -func (w *cachingValidationWrapper) Get(ctx context.Context, machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { - return w.actualProvider.Get(ctx, machine, data) +func (w *cachingValidationWrapper) Get(ctx context.Context, log *zap.SugaredLogger, machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { + return w.actualProvider.Get(ctx, log, machine, data) } // GetCloudConfig just calls the underlying cloudproviders GetCloudConfig. @@ -80,18 +81,18 @@ func (w *cachingValidationWrapper) GetCloudConfig(spec v1alpha1.MachineSpec) (st } // Create just calls the underlying cloudproviders Create. -func (w *cachingValidationWrapper) Create(ctx context.Context, machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { - return w.actualProvider.Create(ctx, machine, data, userdata) +func (w *cachingValidationWrapper) Create(ctx context.Context, log *zap.SugaredLogger, machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { + return w.actualProvider.Create(ctx, log, machine, data, userdata) } // Cleanup just calls the underlying cloudproviders Cleanup. -func (w *cachingValidationWrapper) Cleanup(ctx context.Context, m *v1alpha1.Machine, mcd *cloudprovidertypes.ProviderData) (bool, error) { - return w.actualProvider.Cleanup(ctx, m, mcd) +func (w *cachingValidationWrapper) Cleanup(ctx context.Context, log *zap.SugaredLogger, m *v1alpha1.Machine, mcd *cloudprovidertypes.ProviderData) (bool, error) { + return w.actualProvider.Cleanup(ctx, log, m, mcd) } // MigrateUID just calls the underlying cloudproviders MigrateUID. -func (w *cachingValidationWrapper) MigrateUID(ctx context.Context, m *v1alpha1.Machine, newUID types.UID) error { - return w.actualProvider.MigrateUID(ctx, m, newUID) +func (w *cachingValidationWrapper) MigrateUID(ctx context.Context, log *zap.SugaredLogger, m *v1alpha1.Machine, newUID types.UID) error { + return w.actualProvider.MigrateUID(ctx, log, m, newUID) } // MachineMetricsLabels just calls the underlying cloudproviders MachineMetricsLabels. diff --git a/pkg/clusterinfo/configmap.go b/pkg/clusterinfo/configmap.go index 399de944f..27bb5bef7 100644 --- a/pkg/clusterinfo/configmap.go +++ b/pkg/clusterinfo/configmap.go @@ -24,13 +24,14 @@ import ( "os" "strconv" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - "k8s.io/klog" ) const ( @@ -54,11 +55,10 @@ type KubeconfigProvider struct { kubeClient kubernetes.Interface } -func (p *KubeconfigProvider) GetKubeconfig(ctx context.Context) (*clientcmdapi.Config, error) { +func (p *KubeconfigProvider) GetKubeconfig(ctx context.Context, log *zap.SugaredLogger) (*clientcmdapi.Config, error) { cm, err := p.getKubeconfigFromConfigMap(ctx) if err != nil { - klog.V(6).Infof("could not get cluster-info kubeconfig from configmap: %v", err) - klog.V(6).Info("falling back to retrieval via endpoint") + log.Debugw("Failed to get cluster-info kubeconfig from configmap; falling back to retrieval via endpoint", zap.Error(err)) return p.buildKubeconfigFromEndpoint(ctx) } return cm, nil diff --git a/pkg/clusterinfo/configmap_test.go b/pkg/clusterinfo/configmap_test.go index ef45b3a8f..c3a902423 100644 --- a/pkg/clusterinfo/configmap_test.go +++ b/pkg/clusterinfo/configmap_test.go @@ -22,6 +22,7 @@ import ( "github.com/go-test/deep" "github.com/pmezard/go-difflib/difflib" + "go.uber.org/zap" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -124,7 +125,7 @@ func TestKubeconfigProvider_GetKubeconfig(t *testing.T) { kubeClient: client, } - resConfig, err := provider.GetKubeconfig(ctx) + resConfig, err := provider.GetKubeconfig(ctx, zap.NewNop().Sugar()) if diff := deep.Equal(err, test.err); diff != nil { t.Error(diff) } diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/controller.go similarity index 83% rename from pkg/controller/machine/machine_controller.go rename to pkg/controller/machine/controller.go index c4da1a05c..67a537497 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/controller.go @@ -21,12 +21,16 @@ import ( "errors" "fmt" "net" + "sort" "strconv" "strings" "time" + "github.com/go-logr/logr" + "github.com/go-logr/zapr" "github.com/heptiolabs/healthcheck" "github.com/prometheus/client_golang/prometheus" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -63,7 +67,6 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/tools/reference" "k8s.io/client-go/util/retry" - "k8s.io/klog" "sigs.k8s.io/controller-runtime/pkg/client" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -79,7 +82,7 @@ const ( FinalizerDeleteInstance = "machine-delete-finalizer" FinalizerDeleteNode = "machine-node-delete-finalizer" - ControllerName = "machine_controller" + ControllerName = "machine-controller" // AnnotationMachineUninitialized indicates that a machine is not yet // ready to be worked on by the machine-controller. The machine-controller @@ -95,12 +98,11 @@ const ( // AnnotationAutoscalerIdentifier is used by the cluster-autoscaler // cluster-api provider to match Nodes to Machines. AnnotationAutoscalerIdentifier = "cluster.k8s.io/machine" - - CloudInitNotReadyError = "cloud-init configuration to %s machine: %v is not ready yet" ) // Reconciler is the controller implementation for machine resources. type Reconciler struct { + log *zap.SugaredLogger kubeClient kubernetes.Interface client ctrlruntimeclient.Client @@ -148,7 +150,7 @@ type NodeSettings struct { } type KubeconfigProvider interface { - GetKubeconfig(context.Context) (*clientcmdapi.Config, error) + GetKubeconfig(context.Context, *zap.SugaredLogger) (*clientcmdapi.Config, error) GetBearerToken() string } @@ -172,6 +174,7 @@ func (mc *MetricsCollection) MustRegister(registerer prometheus.Registerer) { func Add( ctx context.Context, + log *zap.SugaredLogger, mgr manager.Manager, kubeClient kubernetes.Interface, numWorkers int, @@ -188,6 +191,7 @@ func Add( overrideBootstrapKubeletAPIServer string, ) error { reconciler := &Reconciler{ + log: log.Named(ControllerName), kubeClient: kubeClient, client: mgr.GetClient(), recorder: mgr.GetEventRecorderFor(ControllerName), @@ -199,14 +203,14 @@ func Add( bootstrapTokenServiceAccountName: bootstrapTokenServiceAccountName, skipEvictionAfter: skipEvictionAfter, nodeSettings: nodeSettings, - redhatSubscriptionManager: rhsm.NewRedHatSubscriptionManager(), - satelliteSubscriptionManager: rhsm.NewSatelliteSubscriptionManager(), + redhatSubscriptionManager: rhsm.NewRedHatSubscriptionManager(log), + satelliteSubscriptionManager: rhsm.NewSatelliteSubscriptionManager(log), useExternalBootstrap: useExternalBootstrap, nodePortRange: nodePortRange, overrideBootstrapKubeletAPIServer: overrideBootstrapKubeletAPIServer, } - m, err := userdatamanager.New() + m, err := userdatamanager.New(log) if err != nil { return fmt.Errorf("failed to create userdatamanager: %w", err) } @@ -216,8 +220,14 @@ func Add( reconciler.metrics.Errors.Add(1) }) - c, err := controller.New(ControllerName, mgr, - controller.Options{Reconciler: reconciler, MaxConcurrentReconciles: numWorkers}) + c, err := controller.New(ControllerName, mgr, controller.Options{ + Reconciler: reconciler, + MaxConcurrentReconciles: numWorkers, + LogConstructor: func(request *reconcile.Request) logr.Logger { + // we log ourselves + return zapr.NewLogger(zap.NewNop()) + }, + }) if err != nil { return err } @@ -257,7 +267,7 @@ func Add( for _, machine := range machinesList.Items { if string(machine.UID) == ownerUIDString { - klog.V(6).Infof("Processing node: %s (machine=%s)", node.GetName(), machine.Name) + log.Debugw("Processing node", "node", node.GetName(), "machine", ctrlruntimeclient.ObjectKeyFromObject(&machine)) return []reconcile.Request{{NamespacedName: types.NamespacedName{ Namespace: machine.Namespace, Name: machine.Name, @@ -349,13 +359,13 @@ func (r *Reconciler) updateMachineErrorIfTerminalError(machine *clusterv1alpha1. return fmt.Errorf("%s, due to %w", errMsg, err) } -func (r *Reconciler) createProviderInstance(ctx context.Context, prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { +func (r *Reconciler) createProviderInstance(ctx context.Context, log *zap.SugaredLogger, prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { // Ensure finalizer is there. - _, err := r.ensureDeleteFinalizerExists(machine) + _, err := r.ensureDeleteFinalizerExists(log, machine) if err != nil { return nil, fmt.Errorf("failed to add %q finalizer: %w", FinalizerDeleteInstance, err) } - i, err := prov.Create(ctx, machine, r.providerData, userdata) + i, err := prov.Create(ctx, log, machine, r.providerData, userdata) if err != nil { return nil, err } @@ -363,41 +373,45 @@ func (r *Reconciler) createProviderInstance(ctx context.Context, prov cloudprovi } func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + log := r.log.With("machine", request.NamespacedName) + log.Debug("Reconciling") + machine := &clusterv1alpha1.Machine{} if err := r.client.Get(ctx, request.NamespacedName, machine); err != nil { if kerrors.IsNotFound(err) { - klog.V(2).Infof("machine %q in work queue no longer exists", request.NamespacedName.String()) return reconcile.Result{}, nil } + log.Errorw("Failed to get Machine", zap.Error(err)) return reconcile.Result{}, err } if machine.Labels[controllerNameLabelKey] != r.name { - klog.V(3).Infof("Ignoring machine %q because its worker-name doesn't match", request.NamespacedName.String()) + log.Debug("Ignoring machine because its worker-name doesn't match") return reconcile.Result{}, nil } if machine.Annotations[AnnotationMachineUninitialized] != "" { - klog.V(3).Infof("Ignoring machine %q because it has a non-empty %q annotation", machine.Name, AnnotationMachineUninitialized) + log.Debugf("Ignoring machine because it has a non-empty %q annotation", AnnotationMachineUninitialized) return reconcile.Result{}, nil } recorderMachine := machine.DeepCopy() - result, err := r.reconcile(ctx, machine) + result, err := r.reconcile(ctx, log, machine) if err != nil { // We have no guarantee that machine is non-nil after reconciliation - klog.Errorf("Failed to reconcile machine %q: %v", recorderMachine.Name, err) + log.Errorw("Reconciling failed", zap.Error(err)) r.recorder.Eventf(recorderMachine, corev1.EventTypeWarning, "ReconcilingError", "%v", err) } else { r.clearMachineError(machine) } + if result == nil { result = &reconcile.Result{} } return *result, err } -func (r *Reconciler) reconcile(ctx context.Context, machine *clusterv1alpha1.Machine) (*reconcile.Result, error) { +func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine) (*reconcile.Result, error) { // This must stay in the controller, it can not be moved into the webhook // as the webhook does not get the name of machineset controller generated // machines on the CREATE request, because they only have `GenerateName` set, @@ -416,9 +430,13 @@ func (r *Reconciler) reconcile(ctx context.Context, machine *clusterv1alpha1.Mac return nil, fmt.Errorf("failed to get cloud provider %q: %w", providerConfig.CloudProvider, err) } + log = log.With("provider", providerConfig.CloudProvider) + // step 2: check if a user requested to delete the machine if machine.DeletionTimestamp != nil { - return r.deleteMachine(ctx, prov, providerConfig.CloudProvider, machine) + return r.deleteMachine(ctx, log, prov, providerConfig.CloudProvider, machine) + // deleteResult, deleteErr := r.deleteMachine(ctx, log, prov, providerConfig.CloudProvider, machine) + // if client.IgnoreNotFound(deleteErr) } // Step 3: Essentially creates an instance for the given machine. @@ -429,14 +447,14 @@ func (r *Reconciler) reconcile(ctx context.Context, machine *clusterv1alpha1.Mac // case 3.2: creates an instance if there is no node associated with the given machine if machine.Status.NodeRef == nil { - return r.ensureInstanceExistsForMachine(ctx, prov, machine, userdataPlugin, providerConfig) + return r.ensureInstanceExistsForMachine(ctx, log, prov, machine, userdataPlugin, providerConfig) } node, err := r.getNodeByNodeRef(ctx, machine.Status.NodeRef) if err != nil { // In case we cannot find a node for the NodeRef we must remove the NodeRef & recreate an instance on the next sync if kerrors.IsNotFound(err) { - klog.V(3).Infof("found invalid NodeRef on machine %s. Deleting reference...", machine.Name) + log.Info("Found invalid NodeRef on machine; deleting reference...") return nil, r.updateMachine(machine, func(m *clusterv1alpha1.Machine) { m.Status.NodeRef = nil }) @@ -444,6 +462,8 @@ func (r *Reconciler) reconcile(ctx context.Context, machine *clusterv1alpha1.Mac return nil, fmt.Errorf("failed to check if node for machine exists: '%w'", err) } + nodeLog := log.With("node", node.Name) + if nodeIsReady(node) { // We must do this to ensure the informers in the machineSet and machineDeployment controller // get triggered as soon as a ready node exists for a machine @@ -452,11 +472,11 @@ func (r *Reconciler) reconcile(ctx context.Context, machine *clusterv1alpha1.Mac } } else { // Node is not ready anymore? Maybe it got deleted - return r.ensureInstanceExistsForMachine(ctx, prov, machine, userdataPlugin, providerConfig) + return r.ensureInstanceExistsForMachine(ctx, nodeLog, prov, machine, userdataPlugin, providerConfig) } // case 3.3: if the node exists make sure if it has labels and taints attached to it. - return nil, r.ensureNodeLabelsAnnotationsAndTaints(ctx, node, machine) + return nil, r.ensureNodeLabelsAnnotationsAndTaints(ctx, nodeLog, node, machine) } func (r *Reconciler) ensureMachineHasNodeReadyCondition(machine *clusterv1alpha1.Machine) error { @@ -475,53 +495,58 @@ func (r *Reconciler) ensureMachineHasNodeReadyCondition(machine *clusterv1alpha1 }) } -func (r *Reconciler) shouldCleanupVolumes(ctx context.Context, machine *clusterv1alpha1.Machine, providerName providerconfigtypes.CloudProvider) (bool, error) { - // we need to wait for volumeAttachments clean up only for vSphere - if providerName != providerconfigtypes.CloudProviderVsphere { - return false, nil - } - - // No node - No volumeAttachments to be collected +func (r *Reconciler) machineHasValidNode(ctx context.Context, machine *clusterv1alpha1.Machine) (bool, error) { if machine.Status.NodeRef == nil { - klog.V(4).Infof("Skipping eviction for machine %q since it does not have a node", machine.Name) return false, nil } node := &corev1.Node{} if err := r.client.Get(ctx, types.NamespacedName{Name: machine.Status.NodeRef.Name}, node); err != nil { - // Node does not exist - No volumeAttachments to be collected if kerrors.IsNotFound(err) { - klog.V(4).Infof("Skipping eviction for machine %q since it does not have a node", machine.Name) return false, nil } + return false, fmt.Errorf("failed to get node %q", machine.Status.NodeRef.Name) } + return true, nil } +func (r *Reconciler) shouldCleanupVolumes(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, providerName providerconfigtypes.CloudProvider) (bool, error) { + // we need to wait for volumeAttachments clean up only for vSphere + if providerName != providerconfigtypes.CloudProviderVsphere { + return false, nil + } + + hasMachine, err := r.machineHasValidNode(ctx, machine) + if err != nil { + return false, err + } + + if !hasMachine { + log.Debug("Skipping eviction since it does not have a node") + } + + return hasMachine, nil +} + // evictIfNecessary checks if the machine has a node and evicts it if necessary. -func (r *Reconciler) shouldEvict(ctx context.Context, machine *clusterv1alpha1.Machine) (bool, error) { +func (r *Reconciler) shouldEvict(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine) (bool, error) { // If the deletion got triggered a few hours ago, skip eviction. // We assume here that the eviction is blocked by misconfiguration or a misbehaving kubelet and/or controller-runtime if time.Since(machine.DeletionTimestamp.Time) > r.skipEvictionAfter { - klog.V(0).Infof("Skipping eviction for machine %q since the deletion got triggered %.2f minutes ago", machine.Name, r.skipEvictionAfter.Minutes()) + log.Infow("Skipping eviction since the deletion got triggered too long ago", "threshold", r.skipEvictionAfter) return false, nil } - // No node - Nothing to evict - if machine.Status.NodeRef == nil { - klog.V(4).Infof("Skipping eviction for machine %q since it does not have a node", machine.Name) - return false, nil + hasMachine, err := r.machineHasValidNode(ctx, machine) + if err != nil { + return false, err } - node := &corev1.Node{} - if err := r.client.Get(ctx, types.NamespacedName{Name: machine.Status.NodeRef.Name}, node); err != nil { - // Node does not exist - Nothing to evict - if kerrors.IsNotFound(err) { - klog.V(4).Infof("Skipping eviction for machine %q since it does not have a node", machine.Name) - return false, nil - } - return false, fmt.Errorf("failed to get node %q", machine.Status.NodeRef.Name) + if !hasMachine { + log.Debug("Skipping eviction since it does not have a node") + return false, nil } // We must check if an eviction is actually possible and only then return true @@ -553,17 +578,17 @@ func (r *Reconciler) shouldEvict(ctx context.Context, machine *clusterv1alpha1.M // If we arrived here we didn't find any machine without a NodeRef and we didn't // find any node that is schedulable, so eviction can't succeed - klog.V(4).Infof("Skipping eviction for machine %q since there is no possible target for an eviction", machine.Name) + log.Debug("Skipping eviction since there is no possible target for an eviction") return false, nil } // deleteMachine makes sure that an instance has gone in a series of steps. -func (r *Reconciler) deleteMachine(ctx context.Context, prov cloudprovidertypes.Provider, providerName providerconfigtypes.CloudProvider, machine *clusterv1alpha1.Machine) (*reconcile.Result, error) { - shouldEvict, err := r.shouldEvict(ctx, machine) +func (r *Reconciler) deleteMachine(ctx context.Context, log *zap.SugaredLogger, prov cloudprovidertypes.Provider, providerName providerconfigtypes.CloudProvider, machine *clusterv1alpha1.Machine) (*reconcile.Result, error) { + shouldEvict, err := r.shouldEvict(ctx, log, machine) if err != nil { return nil, err } - shouldCleanUpVolumes, err := r.shouldCleanupVolumes(ctx, machine, providerName) + shouldCleanUpVolumes, err := r.shouldCleanupVolumes(ctx, log, machine, providerName) if err != nil { return nil, err } @@ -571,13 +596,13 @@ func (r *Reconciler) deleteMachine(ctx context.Context, prov cloudprovidertypes. var evictedSomething, deletedSomething bool var volumesFree = true if shouldEvict { - evictedSomething, err = eviction.New(ctx, machine.Status.NodeRef.Name, r.client, r.kubeClient).Run() + evictedSomething, err = eviction.New(machine.Status.NodeRef.Name, r.client, r.kubeClient).Run(ctx, log) if err != nil { return nil, fmt.Errorf("failed to evict node %s: %w", machine.Status.NodeRef.Name, err) } } if shouldCleanUpVolumes { - deletedSomething, volumesFree, err = poddeletion.New(ctx, machine.Status.NodeRef.Name, r.client, r.kubeClient).Run() + deletedSomething, volumesFree, err = poddeletion.New(machine.Status.NodeRef.Name, r.client, r.kubeClient).Run(ctx, log) if err != nil { return nil, fmt.Errorf("failed to delete pods bound to volumes running on node %s: %w", machine.Status.NodeRef.Name, err) } @@ -587,7 +612,7 @@ func (r *Reconciler) deleteMachine(ctx context.Context, prov cloudprovidertypes. return &reconcile.Result{RequeueAfter: 10 * time.Second}, nil } - if result, err := r.deleteCloudProviderInstance(ctx, prov, machine); result != nil || err != nil { + if result, err := r.deleteCloudProviderInstance(ctx, log, prov, machine); result != nil || err != nil { return result, err } @@ -600,12 +625,12 @@ func (r *Reconciler) deleteMachine(ctx context.Context, prov cloudprovidertypes. return nil, nil } - nodes, err := r.retrieveNodesRelatedToMachine(ctx, machine) + nodes, err := r.retrieveNodesRelatedToMachine(ctx, log, machine) if err != nil { return nil, err } - if err := r.deleteNodeForMachine(ctx, nodes, machine); err != nil { + if err := r.deleteNodeForMachine(ctx, log, nodes, machine); err != nil { return nil, err } @@ -614,7 +639,7 @@ func (r *Reconciler) deleteMachine(ctx context.Context, prov cloudprovidertypes. return nil, nil } -func (r *Reconciler) retrieveNodesRelatedToMachine(ctx context.Context, machine *clusterv1alpha1.Machine) ([]*corev1.Node, error) { +func (r *Reconciler) retrieveNodesRelatedToMachine(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine) ([]*corev1.Node, error) { nodes := make([]*corev1.Node, 0) // If there's NodeRef on the Machine object, retrieve the node by using the @@ -627,7 +652,7 @@ func (r *Reconciler) retrieveNodesRelatedToMachine(ctx context.Context, machine if !kerrors.IsNotFound(err) { return nil, fmt.Errorf("failed to get node %s: %w", machine.Status.NodeRef.Name, err) } - klog.V(2).Infof("node %q does not longer exist for machine %q", machine.Status.NodeRef.Name, machine.Spec.Name) + log.Debugw("Node does not longer exist for machine", "node", machine.Status.NodeRef.Name) } else { nodes = append(nodes, node) } @@ -643,7 +668,7 @@ func (r *Reconciler) retrieveNodesRelatedToMachine(ctx context.Context, machine } if len(nodeList.Items) == 0 { // We just want log that we didn't found the node. - klog.V(3).Infof("No node found for the machine %s", machine.Spec.Name) + log.Debug("No node found for the machine") } for i := range nodeList.Items { @@ -654,14 +679,14 @@ func (r *Reconciler) retrieveNodesRelatedToMachine(ctx context.Context, machine return nodes, nil } -func (r *Reconciler) deleteCloudProviderInstance(ctx context.Context, prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine) (*reconcile.Result, error) { +func (r *Reconciler) deleteCloudProviderInstance(ctx context.Context, log *zap.SugaredLogger, prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine) (*reconcile.Result, error) { finalizers := sets.NewString(machine.Finalizers...) if !finalizers.Has(FinalizerDeleteInstance) { return nil, nil } // Delete the instance - completelyGone, err := prov.Cleanup(ctx, machine, r.providerData) + completelyGone, err := prov.Cleanup(ctx, log, machine, r.providerData) if err != nil { message := fmt.Sprintf("%v. Please manually delete %s finalizer from the machine object.", err, FinalizerDeleteInstance) return nil, r.updateMachineErrorIfTerminalError(machine, common.DeleteMachineError, message, err, "failed to delete machine at cloud provider") @@ -724,14 +749,14 @@ func (r *Reconciler) deleteCloudProviderInstance(ctx context.Context, prov cloud }) } -func (r *Reconciler) deleteNodeForMachine(ctx context.Context, nodes []*corev1.Node, machine *clusterv1alpha1.Machine) error { +func (r *Reconciler) deleteNodeForMachine(ctx context.Context, log *zap.SugaredLogger, nodes []*corev1.Node, machine *clusterv1alpha1.Machine) error { // iterates on all nodes and delete them. Finally, remove the finalizer on the machine for _, node := range nodes { if err := r.client.Delete(ctx, node); err != nil { if !kerrors.IsNotFound(err) { return err } - klog.V(2).Infof("node %q does not longer exist for machine %q", machine.Status.NodeRef.Name, machine.Spec.Name) + log.Infow("Node does not longer exist for machine", "node", machine.Status.NodeRef.Name) } } @@ -747,26 +772,27 @@ func (r *Reconciler) deleteNodeForMachine(ctx context.Context, nodes []*corev1.N func (r *Reconciler) ensureInstanceExistsForMachine( ctx context.Context, + log *zap.SugaredLogger, prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine, userdataPlugin userdataplugin.Provider, providerConfig *providerconfigtypes.Config, ) (*reconcile.Result, error) { - klog.V(6).Infof("Requesting instance for machine '%s' from cloudprovider because no associated node with status ready found...", machine.Name) + log.Debug("Requesting instance for machine from cloudprovider because no associated node with status ready found...") - providerInstance, err := prov.Get(ctx, machine, r.providerData) + providerInstance, err := prov.Get(ctx, log, machine, r.providerData) // case 2: retrieving instance from provider was not successful if err != nil { // case 2.1: instance was not found and we are going to create one if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { - klog.V(3).Infof("Validated machine spec of %s", machine.Name) + log.Debug("Validated machine spec") var kubeconfig *clientcmdapi.Config // an external provider will take care of the bootstrap kubeconfig and token by itself. if !r.useExternalBootstrap { - kubeconfig, err = r.createBootstrapKubeconfig(ctx, machine.Name) + kubeconfig, err = r.createBootstrapKubeconfig(ctx, log, machine.Name) if err != nil { return nil, fmt.Errorf("failed to create bootstrap kubeconfig: %w", err) } @@ -832,13 +858,13 @@ func (r *Reconciler) ensureInstanceExistsForMachine( if err := r.client.Get(ctx, types.NamespacedName{Name: bootstrapSecretName, Namespace: util.CloudInitNamespace}, bootstrapSecret); err != nil { - klog.Errorf(CloudInitNotReadyError, bootstrap.BootstrapCloudConfig, machine.Name) - return nil, err + log.Errorw("cloud-init configuration: cloud config is not ready yet", "secret", bootstrap.BootstrapCloudConfig) + return &reconcile.Result{RequeueAfter: 3 * time.Second}, nil } bootstrapSecretRevision := bootstrapSecret.Annotations[bootstrap.MachineDeploymentRevision] if bootstrapSecretRevision != machineDeploymentRevision { - return nil, fmt.Errorf(CloudInitNotReadyError, bootstrap.BootstrapCloudConfig, machine.Name) + return nil, fmt.Errorf("cloud-init configuration: cloud config %q is not ready yet", bootstrap.BootstrapCloudConfig) } userdata = getOSMBootstrapUserdata(machine.Spec.Name, *bootstrapSecret) @@ -860,15 +886,15 @@ func (r *Reconciler) ensureInstanceExistsForMachine( NodePortRange: r.nodePortRange, } - userdata, err = userdataPlugin.UserData(req) + userdata, err = userdataPlugin.UserData(log, req) if err != nil { return nil, fmt.Errorf("failed get userdata: %w", err) } } // Create the instance - if _, err = r.createProviderInstance(ctx, prov, machine, userdata); err != nil { - message := fmt.Sprintf("%v. Unable to create a machine.", err) + if _, err = r.createProviderInstance(ctx, log, prov, machine, userdata); err != nil { + message := fmt.Sprintf("%v. Failed to create a machine.", err) return nil, r.updateMachineErrorIfTerminalError(machine, common.CreateMachineError, message, err, "failed to create machine at cloudprovider") } if providerConfig.OperatingSystem == providerconfigtypes.OperatingSystemRHEL { @@ -877,14 +903,14 @@ func (r *Reconciler) ensureInstanceExistsForMachine( } } r.recorder.Event(machine, corev1.EventTypeNormal, "Created", "Successfully created instance") - klog.V(3).Infof("Created machine %s at cloud provider", machine.Name) + log.Info("Created machine at cloud provider") // Reqeue the machine to make sure we notice if creation failed silently return &reconcile.Result{RequeueAfter: 30 * time.Second}, nil } // case 2.2: terminal error was returned and manual interaction is required to recover if ok, _, _ := cloudprovidererrors.IsTerminalError(err); ok { - message := fmt.Sprintf("%v. Unable to create a machine.", err) + message := fmt.Sprintf("%v. Failed to create a machine.", err) return nil, r.updateMachineErrorIfTerminalError(machine, common.CreateMachineError, message, err, "failed to get instance from provider") } @@ -892,9 +918,9 @@ func (r *Reconciler) ensureInstanceExistsForMachine( return nil, fmt.Errorf("failed to get instance from provider: %w", err) } // Instance exists, so ensure finalizer does as well - machine, err = r.ensureDeleteFinalizerExists(machine) + machine, err = r.ensureDeleteFinalizerExists(log, machine) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to add %q finalizer: %w", FinalizerDeleteInstance, err) } // case 3: retrieving the instance from cloudprovider was successful @@ -923,16 +949,30 @@ func (r *Reconciler) ensureInstanceExistsForMachine( for address, addressType := range addresses { machineAddresses = append(machineAddresses, corev1.NodeAddress{Address: address, Type: addressType}) } + + // Addresses from the provider are a map; prevent needless updates by sorting them. + sort.Slice(machineAddresses, func(i, j int) bool { + a := machineAddresses[i] + b := machineAddresses[j] + + if a.Type == b.Type { + return a.Address < b.Address + } + + return a.Type < b.Type + }) + if err := r.updateMachine(machine, func(m *clusterv1alpha1.Machine) { m.Status.Addresses = machineAddresses }); err != nil { return nil, fmt.Errorf("failed to update machine after setting .status.addresses: %w", err) } - return r.ensureNodeOwnerRef(ctx, providerInstance, machine, providerConfig) + + return r.ensureNodeOwnerRef(ctx, log, providerInstance, machine, providerConfig) } -func (r *Reconciler) ensureNodeOwnerRef(ctx context.Context, providerInstance instance.Instance, machine *clusterv1alpha1.Machine, providerConfig *providerconfigtypes.Config) (*reconcile.Result, error) { - node, exists, err := r.getNode(ctx, providerInstance, providerConfig.CloudProvider) +func (r *Reconciler) ensureNodeOwnerRef(ctx context.Context, log *zap.SugaredLogger, providerInstance instance.Instance, machine *clusterv1alpha1.Machine, providerConfig *providerconfigtypes.Config) (*reconcile.Result, error) { + node, exists, err := r.getNode(ctx, log, providerInstance, providerConfig.CloudProvider) if err != nil { return nil, fmt.Errorf("failed to get node for machine %s: %w", machine.Name, err) } @@ -954,7 +994,7 @@ func (r *Reconciler) ensureNodeOwnerRef(ctx context.Context, providerInstance in // Check if the machine is a potential candidate for triggering deletion if r.joinClusterTimeout != nil && ownerReferencesHasMachineSetKind(machine.OwnerReferences) { if time.Since(machine.CreationTimestamp.Time) > *r.joinClusterTimeout { - klog.V(3).Infof("Join cluster timeout expired for machine %s, deleting it", machine.Name) + log.Info("Join cluster timeout expired for machine; deleting it", "timeout", *r.joinClusterTimeout) if err := r.client.Delete(ctx, machine); err != nil { return nil, fmt.Errorf("failed to delete machine %s/%s that didn't join cluster within expected period of %s: %w", machine.Namespace, machine.Name, r.joinClusterTimeout.String(), err) @@ -977,7 +1017,7 @@ func ownerReferencesHasMachineSetKind(ownerReferences []metav1.OwnerReference) b return false } -func (r *Reconciler) ensureNodeLabelsAnnotationsAndTaints(ctx context.Context, node *corev1.Node, machine *clusterv1alpha1.Machine) error { +func (r *Reconciler) ensureNodeLabelsAnnotationsAndTaints(ctx context.Context, nodeLog *zap.SugaredLogger, node *corev1.Node, machine *clusterv1alpha1.Machine) error { var modifiers []func(*corev1.Node) for k, v := range machine.Spec.Labels { @@ -1035,7 +1075,7 @@ func (r *Reconciler) ensureNodeLabelsAnnotationsAndTaints(ctx context.Context, n return fmt.Errorf("failed to update node %s after setting labels/annotations/taints: %w", node.Name, err) } r.recorder.Event(machine, corev1.EventTypeNormal, "LabelsAnnotationsTaintsUpdated", "Successfully updated labels/annotations/taints") - klog.V(3).Infof("Added labels/annotations/taints to node %s (machine %s)", node.Name, machine.Name) + nodeLog.Info("Added labels/annotations/taints") } return nil @@ -1064,7 +1104,7 @@ func (r *Reconciler) updateMachineStatus(machine *clusterv1alpha1.Machine, node return nil } -func (r *Reconciler) getNode(ctx context.Context, instance instance.Instance, provider providerconfigtypes.CloudProvider) (node *corev1.Node, exists bool, err error) { +func (r *Reconciler) getNode(ctx context.Context, log *zap.SugaredLogger, instance instance.Instance, provider providerconfigtypes.CloudProvider) (node *corev1.Node, exists bool, err error) { if instance == nil { return nil, false, fmt.Errorf("getNode called with nil provider instance") } @@ -1076,7 +1116,7 @@ func (r *Reconciler) getNode(ctx context.Context, instance instance.Instance, pr for _, node := range nodes.Items { // Try to find Node by providerID. Should work if CCM is deployed. if node := findNodeByProviderID(instance, provider, nodes.Items); node != nil { - klog.V(4).Infof("Found node %q by providerID", node.Name) + log.Debugw("Found node by providerID", "node", node.Name) return node, true, nil } @@ -1106,7 +1146,7 @@ func (r *Reconciler) getNode(ctx context.Context, instance instance.Instance, pr continue } if nodeAddress.Address == instanceAddress { - klog.V(4).Infof("Found node %q by IP address", node.Name) + log.Debugw("Found node by IP address", "node", node.Name) return node.DeepCopy(), true, nil } } @@ -1144,26 +1184,26 @@ func findNodeByProviderID(instance instance.Instance, provider providerconfigtyp func (r *Reconciler) ReadinessChecks(ctx context.Context) map[string]healthcheck.Check { return map[string]healthcheck.Check{ "valid-info-kubeconfig": func() error { - cm, err := r.kubeconfigProvider.GetKubeconfig(ctx) + cm, err := r.kubeconfigProvider.GetKubeconfig(ctx, r.log) if err != nil { err := fmt.Errorf("failed to get cluster-info configmap: %w", err) - klog.V(2).Info(err) + r.log.Error(err) return err } if len(cm.Clusters) != 1 { err := errors.New("invalid kubeconfig: no clusters found") - klog.V(2).Info(err) + r.log.Error(err) return err } for name, c := range cm.Clusters { if len(c.CertificateAuthorityData) == 0 { err := fmt.Errorf("invalid kubeconfig: no certificate authority data was specified for kuberconfig.clusters.['%s']", name) - klog.V(2).Info(err) + r.log.Error(err) return err } if len(c.Server) == 0 { err := fmt.Errorf("invalid kubeconfig: no server was specified for kuberconfig.clusters.['%s']", name) - klog.V(2).Info(err) + r.log.Error(err) return err } } @@ -1172,18 +1212,20 @@ func (r *Reconciler) ReadinessChecks(ctx context.Context) map[string]healthcheck } } -func (r *Reconciler) ensureDeleteFinalizerExists(machine *clusterv1alpha1.Machine) (*clusterv1alpha1.Machine, error) { - if !sets.NewString(machine.Finalizers...).Has(FinalizerDeleteInstance) { +func (r *Reconciler) ensureDeleteFinalizerExists(log *zap.SugaredLogger, machine *clusterv1alpha1.Machine) (*clusterv1alpha1.Machine, error) { + finalizers := sets.NewString(machine.Finalizers...) + length := finalizers.Len() + + finalizers.Insert(FinalizerDeleteInstance, FinalizerDeleteNode) + + if finalizers.Len() > length { if err := r.updateMachine(machine, func(m *clusterv1alpha1.Machine) { - finalizers := sets.NewString(m.Finalizers...) - finalizers.Insert(FinalizerDeleteInstance) - finalizers.Insert(FinalizerDeleteNode) m.Finalizers = finalizers.List() }); err != nil { - return nil, fmt.Errorf("failed to update machine after adding the delete instance finalizer: %w", err) + return nil, err } - klog.V(3).Infof("Added delete finalizer to machine %s", machine.Name) } + return machine, nil } diff --git a/pkg/controller/machine/machine_test.go b/pkg/controller/machine/controller_test.go similarity index 96% rename from pkg/controller/machine/machine_test.go rename to pkg/controller/machine/controller_test.go index 9d109ffe0..4ccd41762 100644 --- a/pkg/controller/machine/machine_test.go +++ b/pkg/controller/machine/controller_test.go @@ -23,6 +23,7 @@ import ( "time" "github.com/go-test/deep" + "go.uber.org/zap" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" @@ -35,7 +36,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" - "k8s.io/klog" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ctrlruntimefake "sigs.k8s.io/controller-runtime/pkg/client/fake" fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -43,7 +43,7 @@ import ( func init() { if err := clusterv1alpha1.AddToScheme(scheme.Scheme); err != nil { - klog.Fatalf("failed to add clusterv1alpha1 api to scheme: %v", err) + panic(fmt.Sprintf("failed to add clusterv1alpha1 api to scheme: %v", err)) } } @@ -202,7 +202,7 @@ func TestController_GetNode(t *testing.T) { reconciler := Reconciler{client: client} - node, exists, err := reconciler.getNode(ctx, test.instance, test.provider) + node, exists, err := reconciler.getNode(ctx, zap.NewNop().Sugar(), test.instance, test.provider) if diff := deep.Equal(err, test.err); diff != nil { t.Errorf("expected to get %v instead got: %v", test.err, err) } @@ -314,7 +314,7 @@ func TestControllerDeletesMachinesOnJoinTimeout(t *testing.T) { joinClusterTimeout: test.joinTimeoutConfig, } - if _, err := reconciler.ensureNodeOwnerRef(ctx, instance, machine, providerConfig); err != nil { + if _, err := reconciler.ensureNodeOwnerRef(ctx, zap.NewNop().Sugar(), instance, machine, providerConfig); err != nil { t.Fatalf("failed to call ensureNodeOwnerRef: %v", err) } @@ -467,7 +467,7 @@ func TestControllerShouldEvict(t *testing.T) { skipEvictionAfter: 2 * time.Hour, } - shouldEvict, err := reconciler.shouldEvict(ctx, test.machine) + shouldEvict, err := reconciler.shouldEvict(ctx, zap.NewNop().Sugar(), test.machine) if err != nil { t.Fatal(err) } @@ -591,6 +591,8 @@ func TestControllerDeleteNodeForMachine(t *testing.T) { }, } + log := zap.NewNop().Sugar() + for _, test := range tests { t.Run(test.name, func(t *testing.T) { ctx := context.Background() @@ -617,12 +619,12 @@ func TestControllerDeleteNodeForMachine(t *testing.T) { providerData: providerData, } - nodes, err := reconciler.retrieveNodesRelatedToMachine(ctx, test.machine) + nodes, err := reconciler.retrieveNodesRelatedToMachine(ctx, log, test.machine) if err != nil { return } - err = reconciler.deleteNodeForMachine(ctx, nodes, test.machine) + err = reconciler.deleteNodeForMachine(ctx, log, nodes, test.machine) if diff := deep.Equal(err, test.err); diff != nil { t.Errorf("expected to get %v instead got: %v", test.err, err) } diff --git a/pkg/controller/machine/kubeconfig.go b/pkg/controller/machine/kubeconfig.go index d1926fee4..1ae436607 100644 --- a/pkg/controller/machine/kubeconfig.go +++ b/pkg/controller/machine/kubeconfig.go @@ -23,6 +23,8 @@ import ( "fmt" "time" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -47,7 +49,7 @@ const ( contextIdentifier string = "c" ) -func (r *Reconciler) createBootstrapKubeconfig(ctx context.Context, name string) (*clientcmdapi.Config, error) { +func (r *Reconciler) createBootstrapKubeconfig(ctx context.Context, log *zap.SugaredLogger, name string) (*clientcmdapi.Config, error) { var token string var err error @@ -63,7 +65,7 @@ func (r *Reconciler) createBootstrapKubeconfig(ctx context.Context, name string) } } - infoKubeconfig, err := r.kubeconfigProvider.GetKubeconfig(ctx) + infoKubeconfig, err := r.kubeconfigProvider.GetKubeconfig(ctx, log) if err != nil { return nil, err } diff --git a/pkg/controller/machinedeployment/machinedeployment_controller.go b/pkg/controller/machinedeployment/controller.go similarity index 81% rename from pkg/controller/machinedeployment/machinedeployment_controller.go rename to pkg/controller/machinedeployment/controller.go index 37528d471..9f45b1fe0 100644 --- a/pkg/controller/machinedeployment/machinedeployment_controller.go +++ b/pkg/controller/machinedeployment/controller.go @@ -20,7 +20,10 @@ import ( "context" "reflect" + "github.com/go-logr/logr" + "github.com/go-logr/zapr" "github.com/pkg/errors" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -32,7 +35,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" - "k8s.io/klog" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -52,25 +54,37 @@ var ( // ReconcileMachineDeployment reconciles a MachineDeployment object. type ReconcileMachineDeployment struct { client.Client + log *zap.SugaredLogger scheme *runtime.Scheme recorder record.EventRecorder } // newReconciler returns a new reconcile.Reconciler. -func newReconciler(mgr manager.Manager) *ReconcileMachineDeployment { - return &ReconcileMachineDeployment{Client: mgr.GetClient(), scheme: mgr.GetScheme(), recorder: mgr.GetEventRecorderFor(controllerName)} +func newReconciler(mgr manager.Manager, log *zap.SugaredLogger) *ReconcileMachineDeployment { + return &ReconcileMachineDeployment{ + Client: mgr.GetClient(), + log: log.Named(controllerName), + scheme: mgr.GetScheme(), + recorder: mgr.GetEventRecorderFor(controllerName), + } } // Add creates a new MachineDeployment Controller and adds it to the Manager with default RBAC. -func Add(mgr manager.Manager) error { - r := newReconciler(mgr) - return add(mgr, newReconciler(mgr), r.MachineSetToDeployments) +func Add(mgr manager.Manager, log *zap.SugaredLogger) error { + r := newReconciler(mgr, log) + return add(mgr, r, r.MachineSetToDeployments) } // add adds a new Controller to mgr with r as the reconcile.Reconciler. func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) error { // Create a new controller. - c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r}) + c, err := controller.New(controllerName, mgr, controller.Options{ + Reconciler: r, + LogConstructor: func(request *reconcile.Request) logr.Logger { + // we log ourselves + return zapr.NewLogger(zap.NewNop()) + }, + }) if err != nil { return err } @@ -112,34 +126,38 @@ func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) err // // +kubebuilder:rbac:groups=cluster.k8s.io,resources=machinedeployments;machinedeployments/status,verbs=get;list;watch;create;update;patch;delete func (r *ReconcileMachineDeployment) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + log := r.log.With("machinedeployment", request.NamespacedName) + log.Debug("Reconciling") + // Fetch the MachineDeployment instance - d := &v1alpha1.MachineDeployment{} - if err := r.Get(ctx, request.NamespacedName, d); err != nil { + deployment := &v1alpha1.MachineDeployment{} + if err := r.Get(ctx, request.NamespacedName, deployment); err != nil { if apierrors.IsNotFound(err) { // Object not found, return. Created objects are automatically garbage collected. // For additional cleanup logic use finalizers. return reconcile.Result{}, nil } // Error reading the object - requeue the request. + log.Errorw("Failed to get MachineDeployment", zap.Error(err)) return reconcile.Result{}, err } // Ignore deleted MachineDeployments, this can happen when foregroundDeletion // is enabled - if d.DeletionTimestamp != nil { + if deployment.DeletionTimestamp != nil { return reconcile.Result{}, nil } - result, err := r.reconcile(ctx, d) + result, err := r.reconcile(ctx, log, deployment) if err != nil { - klog.Errorf("Failed to reconcile MachineDeployment %q: %v", request.NamespacedName, err) - r.recorder.Eventf(d, corev1.EventTypeWarning, "ReconcileError", "%v", err) + log.Errorw("Reconciling failed", zap.Error(err)) + r.recorder.Eventf(deployment, corev1.EventTypeWarning, "ReconcileError", "%v", err) } return result, err } -func (r *ReconcileMachineDeployment) reconcile(ctx context.Context, d *v1alpha1.MachineDeployment) (reconcile.Result, error) { +func (r *ReconcileMachineDeployment) reconcile(ctx context.Context, log *zap.SugaredLogger, d *v1alpha1.MachineDeployment) (reconcile.Result, error) { v1alpha1.PopulateDefaultsMachineDeployment(d) everything := metav1.LabelSelector{} @@ -147,7 +165,6 @@ func (r *ReconcileMachineDeployment) reconcile(ctx context.Context, d *v1alpha1. if d.Status.ObservedGeneration < d.Generation { d.Status.ObservedGeneration = d.Generation if err := r.Status().Update(ctx, d); err != nil { - klog.Warningf("Failed to update status for MachineDeployment %q: %v", d.Name, err) return reconcile.Result{}, err } } @@ -168,7 +185,6 @@ func (r *ReconcileMachineDeployment) reconcile(ctx context.Context, d *v1alpha1. if !contains(d.Finalizers, metav1.FinalizerDeleteDependents) { d.Finalizers = append(d.ObjectMeta.Finalizers, metav1.FinalizerDeleteDependents) if err := r.Client.Update(ctx, d); err != nil { - klog.Infof("Failed to add finalizers to MachineSet %q: %v", d.Name, err) return reconcile.Result{}, err } @@ -176,7 +192,7 @@ func (r *ReconcileMachineDeployment) reconcile(ctx context.Context, d *v1alpha1. return reconcile.Result{Requeue: true}, nil } - msList, err := r.getMachineSetsForDeployment(ctx, d) + msList, err := r.getMachineSetsForDeployment(ctx, log, d) if err != nil { return reconcile.Result{}, err } @@ -187,23 +203,23 @@ func (r *ReconcileMachineDeployment) reconcile(ctx context.Context, d *v1alpha1. } if d.DeletionTimestamp != nil { - return reconcile.Result{}, r.sync(ctx, d, msList, machineMap) + return reconcile.Result{}, r.sync(ctx, log, d, msList, machineMap) } if d.Spec.Paused { - return reconcile.Result{}, r.sync(ctx, d, msList, machineMap) + return reconcile.Result{}, r.sync(ctx, log, d, msList, machineMap) } switch d.Spec.Strategy.Type { case common.RollingUpdateMachineDeploymentStrategyType: - return reconcile.Result{}, r.rolloutRolling(ctx, d, msList, machineMap) + return reconcile.Result{}, r.rolloutRolling(ctx, log, d, msList, machineMap) } return reconcile.Result{}, errors.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type) } // getMachineSetsForDeployment returns a list of MachineSets associated with a MachineDeployment. -func (r *ReconcileMachineDeployment) getMachineSetsForDeployment(ctx context.Context, d *v1alpha1.MachineDeployment) ([]*v1alpha1.MachineSet, error) { +func (r *ReconcileMachineDeployment) getMachineSetsForDeployment(ctx context.Context, log *zap.SugaredLogger, d *v1alpha1.MachineDeployment) ([]*v1alpha1.MachineSet, error) { // List all MachineSets to find those we own but that no longer match our selector. machineSets := &v1alpha1.MachineSetList{} listOptions := &client.ListOptions{Namespace: d.Namespace} @@ -214,28 +230,29 @@ func (r *ReconcileMachineDeployment) getMachineSetsForDeployment(ctx context.Con filtered := make([]*v1alpha1.MachineSet, 0, len(machineSets.Items)) for idx := range machineSets.Items { ms := &machineSets.Items[idx] + msLog := log.With("machineset", client.ObjectKeyFromObject(ms)) selector, err := metav1.LabelSelectorAsSelector(&d.Spec.Selector) if err != nil { - klog.Errorf("Skipping MachineSet %q, failed to get label selector from spec selector: %v", ms.Name, err) + msLog.Errorw("Skipping MachineSet, failed to get label selector from spec selector", zap.Error(err)) continue } // If a MachineDeployment with a nil or empty selector creeps in, it should match nothing, not everything. if selector.Empty() { - klog.Warningf("Skipping MachineSet %q as the selector is empty", ms.Name) + msLog.Info("Skipping MachineSet as the selector is empty") continue } if !selector.Matches(labels.Set(ms.Labels)) { - klog.V(4).Infof("Skipping MachineSet %v, label mismatch", ms.Name) + msLog.Debug("Skipping MachineSet, label mismatch") continue } // Attempt to adopt machine if it meets previous conditions and it has no controller references. if metav1.GetControllerOf(ms) == nil { if err := r.adoptOrphan(ctx, d, ms); err != nil { - klog.Warningf("Failed to adopt MachineSet %q into MachineDeployment %q: %v", ms.Name, d.Name, err) + msLog.Infow("Failed to adopt MachineSet into MachineDeployment", zap.Error(err)) continue } } @@ -303,16 +320,16 @@ func (r *ReconcileMachineDeployment) getMachineMapForDeployment(ctx context.Cont } // getMachineDeploymentsForMachineSet returns a list of MachineDeployments that could potentially match a MachineSet. -func (r *ReconcileMachineDeployment) getMachineDeploymentsForMachineSet(ctx context.Context, ms *v1alpha1.MachineSet) []*v1alpha1.MachineDeployment { +func (r *ReconcileMachineDeployment) getMachineDeploymentsForMachineSet(ctx context.Context, log *zap.SugaredLogger, ms *v1alpha1.MachineSet) []*v1alpha1.MachineDeployment { if len(ms.Labels) == 0 { - klog.Warningf("No machine deployments found for MachineSet %q because it has no labels", ms.Name) + log.Info("No MachineDeployments found for MachineSet because it has no labels") return nil } dList := &v1alpha1.MachineDeploymentList{} listOptions := &client.ListOptions{Namespace: ms.Namespace} if err := r.Client.List(ctx, dList, listOptions); err != nil { - klog.Warningf("Failed to list machine deployments: %v", err) + log.Errorw("Failed to list MachineDeployments", zap.Error(err)) return nil } @@ -344,7 +361,7 @@ func (r *ReconcileMachineDeployment) MachineSetToDeployments(o client.Object) [] key := client.ObjectKey{Namespace: o.GetNamespace(), Name: o.GetName()} if err := r.Client.Get(ctx, key, ms); err != nil { if !apierrors.IsNotFound(err) { - klog.Errorf("Unable to retrieve MachineSet %q for possible MachineDeployment adoption: %v", key, err) + r.log.Errorw("Failed to retrieve MachineSet for possible MachineDeployment adoption", "machineset", key, zap.Error(err)) } return nil } @@ -357,9 +374,9 @@ func (r *ReconcileMachineDeployment) MachineSetToDeployments(o client.Object) [] } } - mds := r.getMachineDeploymentsForMachineSet(ctx, ms) + mds := r.getMachineDeploymentsForMachineSet(ctx, r.log.With("machineset", key), ms) if len(mds) == 0 { - klog.V(4).Infof("Found no machine set for machine: %v", ms.Name) + r.log.Debugw("Found no MachineDeployments for MachineSet", "machineset", key) return nil } diff --git a/pkg/controller/machinedeployment/rolling.go b/pkg/controller/machinedeployment/rolling.go index 3267d9487..11eb84d08 100644 --- a/pkg/controller/machinedeployment/rolling.go +++ b/pkg/controller/machinedeployment/rolling.go @@ -21,18 +21,19 @@ import ( "sort" "github.com/pkg/errors" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" dutil "github.com/kubermatic/machine-controller/pkg/controller/util" "k8s.io/apimachinery/pkg/types" - "k8s.io/klog" "k8s.io/utils/integer" + "sigs.k8s.io/controller-runtime/pkg/client" ) // rolloutRolling implements the logic for rolling a new machine set. -func (r *ReconcileMachineDeployment) rolloutRolling(ctx context.Context, d *v1alpha1.MachineDeployment, msList []*v1alpha1.MachineSet, machineMap map[types.UID]*v1alpha1.MachineList) error { - newMS, oldMSs, err := r.getAllMachineSetsAndSyncRevision(ctx, d, msList, machineMap, true) +func (r *ReconcileMachineDeployment) rolloutRolling(ctx context.Context, log *zap.SugaredLogger, d *v1alpha1.MachineDeployment, msList []*v1alpha1.MachineSet, machineMap map[types.UID]*v1alpha1.MachineList) error { + newMS, oldMSs, err := r.getAllMachineSetsAndSyncRevision(ctx, log, d, msList, machineMap, true) if err != nil { return err } @@ -56,7 +57,7 @@ func (r *ReconcileMachineDeployment) rolloutRolling(ctx context.Context, d *v1al } // Scale down, if we can. - if err := r.reconcileOldMachineSets(ctx, allMSs, oldMSs, newMS, d); err != nil { + if err := r.reconcileOldMachineSets(ctx, log.With("newmachineset", client.ObjectKeyFromObject(newMS)), allMSs, oldMSs, newMS, d); err != nil { return err } @@ -65,7 +66,7 @@ func (r *ReconcileMachineDeployment) rolloutRolling(ctx context.Context, d *v1al } if dutil.DeploymentComplete(d, &d.Status) { - if err := r.cleanupDeployment(ctx, oldMSs, d); err != nil { + if err := r.cleanupDeployment(ctx, log, oldMSs, d); err != nil { return err } } @@ -101,7 +102,7 @@ func (r *ReconcileMachineDeployment) reconcileNewMachineSet(ctx context.Context, return err } -func (r *ReconcileMachineDeployment) reconcileOldMachineSets(ctx context.Context, allMSs []*v1alpha1.MachineSet, oldMSs []*v1alpha1.MachineSet, newMS *v1alpha1.MachineSet, deployment *v1alpha1.MachineDeployment) error { +func (r *ReconcileMachineDeployment) reconcileOldMachineSets(ctx context.Context, log *zap.SugaredLogger, allMSs []*v1alpha1.MachineSet, oldMSs []*v1alpha1.MachineSet, newMS *v1alpha1.MachineSet, deployment *v1alpha1.MachineDeployment) error { if deployment.Spec.Replicas == nil { return errors.Errorf("spec replicas for deployment set %v is nil, this is unexpected", deployment.Name) } @@ -117,7 +118,7 @@ func (r *ReconcileMachineDeployment) reconcileOldMachineSets(ctx context.Context } allMachinesCount := dutil.GetReplicaCountForMachineSets(allMSs) - klog.V(4).Infof("New machine set %s/%s has %d available machines.", newMS.Namespace, newMS.Name, newMS.Status.AvailableReplicas) + log.Debugw("New machine set status", "replicas", newMS.Status.AvailableReplicas) maxUnavailable := dutil.MaxUnavailable(*deployment) // Check if we can scale down. We can scale down in the following 2 cases: @@ -159,26 +160,26 @@ func (r *ReconcileMachineDeployment) reconcileOldMachineSets(ctx context.Context // Clean up unhealthy replicas first, otherwise unhealthy replicas will block deployment // and cause timeout. See https://github.com/kubernetes/kubernetes/issues/16737 - oldMSs, cleanupCount, err := r.cleanupUnhealthyReplicas(ctx, oldMSs, deployment, maxScaledDown) + oldMSs, cleanupCount, err := r.cleanupUnhealthyReplicas(ctx, log, oldMSs, deployment, maxScaledDown) if err != nil { return nil } - klog.V(4).Infof("Cleaned up unhealthy replicas from old MachineSets by %d", cleanupCount) + log.Debugw("Cleaned up unhealthy replicas from old MachineSets", "reduction", cleanupCount) // Scale down old machine sets, need check maxUnavailable to ensure we can scale down allMSs = append(oldMSs, newMS) - scaledDownCount, err := r.scaleDownOldMachineSetsForRollingUpdate(ctx, allMSs, oldMSs, deployment) + scaledDownCount, err := r.scaleDownOldMachineSetsForRollingUpdate(ctx, log, allMSs, oldMSs, deployment) if err != nil { return err } - klog.V(4).Infof("Scaled down old MachineSets of deployment %s by %d", deployment.Name, scaledDownCount) + log.Debugw("Scaled down old MachineSets", "reduction", scaledDownCount) return nil } // cleanupUnhealthyReplicas will scale down old machine sets with unhealthy replicas, so that all unhealthy replicas will be deleted. -func (r *ReconcileMachineDeployment) cleanupUnhealthyReplicas(ctx context.Context, oldMSs []*v1alpha1.MachineSet, deployment *v1alpha1.MachineDeployment, maxCleanupCount int32) ([]*v1alpha1.MachineSet, int32, error) { +func (r *ReconcileMachineDeployment) cleanupUnhealthyReplicas(ctx context.Context, log *zap.SugaredLogger, oldMSs []*v1alpha1.MachineSet, deployment *v1alpha1.MachineDeployment, maxCleanupCount int32) ([]*v1alpha1.MachineSet, int32, error) { sort.Sort(dutil.MachineSetsByCreationTimestamp(oldMSs)) // Safely scale down all old machine sets with unhealthy replicas. Replica set will sort the machines in the order @@ -202,7 +203,7 @@ func (r *ReconcileMachineDeployment) cleanupUnhealthyReplicas(ctx context.Contex } oldMSAvailableReplicas := targetMS.Status.AvailableReplicas - klog.V(4).Infof("Found %d available machines in old MS %s/%s", oldMSAvailableReplicas, targetMS.Namespace, targetMS.Name) + log.Debugw("Available machines in old MachineSet", "oldmachineset", client.ObjectKeyFromObject(targetMS), "replicas", oldMSAvailableReplicas) if oldMSReplicas == oldMSAvailableReplicas { // no unhealthy replicas found, no scaling required. continue @@ -229,7 +230,7 @@ func (r *ReconcileMachineDeployment) cleanupUnhealthyReplicas(ctx context.Contex // scaleDownOldMachineSetsForRollingUpdate scales down old machine sets when deployment strategy is "RollingUpdate". // Need check maxUnavailable to ensure availability. -func (r *ReconcileMachineDeployment) scaleDownOldMachineSetsForRollingUpdate(ctx context.Context, allMSs []*v1alpha1.MachineSet, oldMSs []*v1alpha1.MachineSet, deployment *v1alpha1.MachineDeployment) (int32, error) { +func (r *ReconcileMachineDeployment) scaleDownOldMachineSetsForRollingUpdate(ctx context.Context, log *zap.SugaredLogger, allMSs []*v1alpha1.MachineSet, oldMSs []*v1alpha1.MachineSet, deployment *v1alpha1.MachineDeployment) (int32, error) { if deployment.Spec.Replicas == nil { return 0, errors.Errorf("spec replicas for deployment %v is nil, this is unexpected", deployment.Name) } @@ -246,7 +247,7 @@ func (r *ReconcileMachineDeployment) scaleDownOldMachineSetsForRollingUpdate(ctx return 0, nil } - klog.V(4).Infof("Found %d available machines in deployment %s, scaling down old MSes", availableMachineCount, deployment.Name) + log.Debugw("Found available machines, scaling down old MachineSets", "replicas", availableMachineCount) sort.Sort(dutil.MachineSetsByCreationTimestamp(oldMSs)) diff --git a/pkg/controller/machinedeployment/sync.go b/pkg/controller/machinedeployment/sync.go index f0865ef51..9fef99055 100644 --- a/pkg/controller/machinedeployment/sync.go +++ b/pkg/controller/machinedeployment/sync.go @@ -24,6 +24,7 @@ import ( "strconv" "github.com/pkg/errors" + "go.uber.org/zap" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" dutil "github.com/kubermatic/machine-controller/pkg/controller/util" @@ -35,19 +36,18 @@ import ( apirand "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/retry" - "k8s.io/klog" "sigs.k8s.io/controller-runtime/pkg/client" ) // sync is responsible for reconciling deployments on scaling events or when they // are paused. -func (r *ReconcileMachineDeployment) sync(ctx context.Context, d *clusterv1alpha1.MachineDeployment, msList []*clusterv1alpha1.MachineSet, machineMap map[types.UID]*clusterv1alpha1.MachineList) error { - newMS, oldMSs, err := r.getAllMachineSetsAndSyncRevision(ctx, d, msList, machineMap, false) +func (r *ReconcileMachineDeployment) sync(ctx context.Context, log *zap.SugaredLogger, d *clusterv1alpha1.MachineDeployment, msList []*clusterv1alpha1.MachineSet, machineMap map[types.UID]*clusterv1alpha1.MachineList) error { + newMS, oldMSs, err := r.getAllMachineSetsAndSyncRevision(ctx, log, d, msList, machineMap, false) if err != nil { return err } - if err := r.scale(ctx, d, newMS, oldMSs); err != nil { + if err := r.scale(ctx, log, d, newMS, oldMSs); err != nil { // If we get an error while trying to scale, the deployment will be requeued // so we can abort this resync return err @@ -72,11 +72,11 @@ func (r *ReconcileMachineDeployment) sync(ctx context.Context, d *clusterv1alpha // // Note that currently the deployment controller is using caches to avoid querying the server for reads. // This may lead to stale reads of machine sets, thus incorrect deployment status. -func (r *ReconcileMachineDeployment) getAllMachineSetsAndSyncRevision(ctx context.Context, d *clusterv1alpha1.MachineDeployment, msList []*clusterv1alpha1.MachineSet, machineMap map[types.UID]*clusterv1alpha1.MachineList, createIfNotExisted bool) (*clusterv1alpha1.MachineSet, []*clusterv1alpha1.MachineSet, error) { +func (r *ReconcileMachineDeployment) getAllMachineSetsAndSyncRevision(ctx context.Context, log *zap.SugaredLogger, d *clusterv1alpha1.MachineDeployment, msList []*clusterv1alpha1.MachineSet, machineMap map[types.UID]*clusterv1alpha1.MachineList, createIfNotExisted bool) (*clusterv1alpha1.MachineSet, []*clusterv1alpha1.MachineSet, error) { _, allOldMSs := dutil.FindOldMachineSets(d, msList) // Get new machine set with the updated revision number - newMS, err := r.getNewMachineSet(ctx, d, msList, allOldMSs, createIfNotExisted) + newMS, err := r.getNewMachineSet(ctx, log, d, msList, allOldMSs, createIfNotExisted) if err != nil { return nil, nil, err } @@ -89,11 +89,11 @@ func (r *ReconcileMachineDeployment) getAllMachineSetsAndSyncRevision(ctx contex // 2. If there's existing new MS, update its revision number if it's smaller than (maxOldRevision + 1), where maxOldRevision is the max revision number among all old MSes. // 3. If there's no existing new MS and createIfNotExisted is true, create one with appropriate revision number (maxOldRevision + 1) and replicas. // Note that the machine-template-hash will be added to adopted MSes and machines. -func (r *ReconcileMachineDeployment) getNewMachineSet(ctx context.Context, d *clusterv1alpha1.MachineDeployment, msList, oldMSs []*clusterv1alpha1.MachineSet, createIfNotExisted bool) (*clusterv1alpha1.MachineSet, error) { +func (r *ReconcileMachineDeployment) getNewMachineSet(ctx context.Context, log *zap.SugaredLogger, d *clusterv1alpha1.MachineDeployment, msList, oldMSs []*clusterv1alpha1.MachineSet, createIfNotExisted bool) (*clusterv1alpha1.MachineSet, error) { existingNewMS := dutil.FindNewMachineSet(d, msList) // Calculate the max revision number among all old MSes - maxOldRevision := dutil.MaxRevision(oldMSs) + maxOldRevision := dutil.MaxRevision(log, oldMSs) // Calculate revision number for this new machine set newRevision := strconv.FormatInt(maxOldRevision+1, 10) @@ -106,7 +106,7 @@ func (r *ReconcileMachineDeployment) getNewMachineSet(ctx context.Context, d *cl msCopy := existingNewMS.DeepCopy() // Set existing new machine set's annotation - annotationsUpdated := dutil.SetNewMachineSetAnnotations(d, msCopy, newRevision, true) + annotationsUpdated := dutil.SetNewMachineSetAnnotations(log, d, msCopy, newRevision, true) minReadySecondsNeedsUpdate := msCopy.Spec.MinReadySeconds != *d.Spec.MinReadySeconds if annotationsUpdated || minReadySecondsNeedsUpdate { @@ -171,7 +171,7 @@ func (r *ReconcileMachineDeployment) getNewMachineSet(ctx context.Context, d *cl *(newMS.Spec.Replicas) = newReplicasCount // Set new machine set's annotation - dutil.SetNewMachineSetAnnotations(d, &newMS, newRevision, false) + dutil.SetNewMachineSetAnnotations(log, d, &newMS, newRevision, false) // Create the new MachineSet. If it already exists, then we need to check for possible // hash collisions. If there is any other error, we need to report it in the status of // the Deployment. @@ -201,12 +201,12 @@ func (r *ReconcileMachineDeployment) getNewMachineSet(ctx context.Context, d *cl return nil, err case err != nil: - klog.V(4).Infof("Failed to create new machine set %q: %v", newMS.Name, err) + log.Errorw("Failed to create new MachineSet", "machineset", client.ObjectKeyFromObject(&newMS), zap.Error(err)) return nil, err } if !alreadyExists { - klog.V(4).Infof("Created new machine set %q", createdMS.Name) + log.Debugw("Created new MachineSet", "machineset", client.ObjectKeyFromObject(createdMS)) } err = r.updateMachineDeployment(ctx, d, func(innerDeployment *clusterv1alpha1.MachineDeployment) { @@ -221,7 +221,7 @@ func (r *ReconcileMachineDeployment) getNewMachineSet(ctx context.Context, d *cl // have the effect of hastening the rollout progress, which could produce a higher proportion of unavailable // replicas in the event of a problem with the rolled out template. Should run only on scaling events or // when a deployment is paused and not during the normal rollout process. -func (r *ReconcileMachineDeployment) scale(ctx context.Context, deployment *clusterv1alpha1.MachineDeployment, newMS *clusterv1alpha1.MachineSet, oldMSs []*clusterv1alpha1.MachineSet) error { +func (r *ReconcileMachineDeployment) scale(ctx context.Context, log *zap.SugaredLogger, deployment *clusterv1alpha1.MachineDeployment, newMS *clusterv1alpha1.MachineSet, oldMSs []*clusterv1alpha1.MachineSet) error { if deployment.Spec.Replicas == nil { return errors.Errorf("spec replicas for deployment %v is nil, this is unexpected", deployment.Name) } @@ -292,14 +292,14 @@ func (r *ReconcileMachineDeployment) scale(ctx context.Context, deployment *clus for i := range allMSs { ms := allMSs[i] if ms.Spec.Replicas == nil { - klog.Errorf("spec replicas for machine set %v is nil, this is unexpected.", ms.Name) + log.Errorw("spec.replicas for MachineSet is nil, this is unexpected.", "machineset", client.ObjectKeyFromObject(ms)) continue } // Estimate proportions if we have replicas to add, otherwise simply populate // nameToSize with the current sizes for each machine set. if deploymentReplicasToAdd != 0 { - proportion := dutil.GetProportion(ms, *deployment, deploymentReplicasToAdd, deploymentReplicasAdded) + proportion := dutil.GetProportion(log, ms, *deployment, deploymentReplicasToAdd, deploymentReplicasAdded) nameToSize[ms.Name] = *(ms.Spec.Replicas) + proportion deploymentReplicasAdded += proportion } else { @@ -421,7 +421,7 @@ func (r *ReconcileMachineDeployment) scaleMachineSetOperation(ctx context.Contex // cleanupDeployment is responsible for cleaning up a deployment i.e. retains all but the latest N old machine sets // where N=d.Spec.RevisionHistoryLimit. Old machine sets are older versions of the machinetemplate of a deployment kept // around by default 1) for historical reasons and 2) for the ability to rollback a deployment. -func (r *ReconcileMachineDeployment) cleanupDeployment(ctx context.Context, oldMSs []*clusterv1alpha1.MachineSet, deployment *clusterv1alpha1.MachineDeployment) error { +func (r *ReconcileMachineDeployment) cleanupDeployment(ctx context.Context, log *zap.SugaredLogger, oldMSs []*clusterv1alpha1.MachineSet, deployment *clusterv1alpha1.MachineDeployment) error { if deployment.Spec.RevisionHistoryLimit == nil { return nil } @@ -439,12 +439,12 @@ func (r *ReconcileMachineDeployment) cleanupDeployment(ctx context.Context, oldM } sort.Sort(dutil.MachineSetsByCreationTimestamp(cleanableMSes)) - klog.V(4).Infof("Looking to cleanup old machine sets for deployment %q", deployment.Name) + log.Debug("Looking to cleanup old MachineSets for MachineDeployment") for i := int32(0); i < diff; i++ { ms := cleanableMSes[i] if ms.Spec.Replicas == nil { - return errors.Errorf("spec replicas for machine set %v is nil, this is unexpected", ms.Name) + return errors.Errorf("spec replicas for MachineSets %v is nil, this is unexpected", ms.Name) } // Avoid delete machine set with non-zero replica counts @@ -452,7 +452,7 @@ func (r *ReconcileMachineDeployment) cleanupDeployment(ctx context.Context, oldM continue } - klog.V(4).Infof("Trying to cleanup machine set %q for deployment %q", ms.Name, deployment.Name) + log.Debugw("Trying to cleanup MachineSet for MachineDeployment", "machineset", client.ObjectKeyFromObject(ms)) if err := r.Delete(ctx, ms); err != nil && !apierrors.IsNotFound(err) { // Return error instead of aggregating and continuing DELETEs on the theory // that we may be overloading the api server. diff --git a/pkg/controller/machineset/machineset_controller.go b/pkg/controller/machineset/controller.go similarity index 83% rename from pkg/controller/machineset/machineset_controller.go rename to pkg/controller/machineset/controller.go index e43fe6def..995db4209 100644 --- a/pkg/controller/machineset/machineset_controller.go +++ b/pkg/controller/machineset/controller.go @@ -23,7 +23,10 @@ import ( "sync" "time" + "github.com/go-logr/logr" + "github.com/go-logr/zapr" "github.com/pkg/errors" + "go.uber.org/zap" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -34,7 +37,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/record" - "k8s.io/klog" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -60,20 +62,31 @@ var ( // Add creates a new MachineSet Controller and adds it to the Manager with default RBAC. // The Manager will set fields on the Controller and Start it when the Manager is Started. -func Add(mgr manager.Manager) error { - r := newReconciler(mgr) +func Add(mgr manager.Manager, log *zap.SugaredLogger) error { + r := newReconciler(mgr, log) return add(mgr, r, r.MachineToMachineSets) } // newReconciler returns a new reconcile.Reconciler. -func newReconciler(mgr manager.Manager) *ReconcileMachineSet { - return &ReconcileMachineSet{Client: mgr.GetClient(), scheme: mgr.GetScheme(), recorder: mgr.GetEventRecorderFor(controllerName)} +func newReconciler(mgr manager.Manager, log *zap.SugaredLogger) *ReconcileMachineSet { + return &ReconcileMachineSet{ + Client: mgr.GetClient(), + scheme: mgr.GetScheme(), + log: log.Named(controllerName), + recorder: mgr.GetEventRecorderFor(controllerName), + } } // add adds a new Controller to mgr with r as the reconcile.Reconciler. func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) error { // Create a new controller. - c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r}) + c, err := controller.New(controllerName, mgr, controller.Options{ + Reconciler: r, + LogConstructor: func(request *reconcile.Request) logr.Logger { + // we log ourselves + return zapr.NewLogger(zap.NewNop()) + }, + }) if err != nil { return err } @@ -108,6 +121,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) err // ReconcileMachineSet reconciles a MachineSet object. type ReconcileMachineSet struct { client.Client + log *zap.SugaredLogger scheme *runtime.Scheme recorder record.EventRecorder } @@ -118,6 +132,9 @@ type ReconcileMachineSet struct { // +kubebuilder:rbac:groups=cluster.k8s.io,resources=machinesets;machinesets/status,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=cluster.k8s.io,resources=machines,verbs=get;list;watch;create;update;patch;delete func (r *ReconcileMachineSet) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + log := r.log.With("machineset", request.NamespacedName) + log.Debug("Reconciling") + // Fetch the MachineSet instance machineSet := &clusterv1alpha1.MachineSet{} if err := r.Get(ctx, request.NamespacedName, machineSet); err != nil { @@ -127,6 +144,7 @@ func (r *ReconcileMachineSet) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, nil } // Error reading the object - requeue the request. + log.Errorw("Failed to get MachineSet", zap.Error(err)) return reconcile.Result{}, err } @@ -136,16 +154,16 @@ func (r *ReconcileMachineSet) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, nil } - result, err := r.reconcile(ctx, machineSet) + result, err := r.reconcile(ctx, log, machineSet) if err != nil { - klog.Errorf("Failed to reconcile MachineSet %q: %v", request.NamespacedName, err) + log.Errorw("Reconciling failed", zap.Error(err)) r.recorder.Eventf(machineSet, corev1.EventTypeWarning, "ReconcileError", "%v", err) } return result, err } -func (r *ReconcileMachineSet) reconcile(ctx context.Context, machineSet *clusterv1alpha1.MachineSet) (reconcile.Result, error) { - klog.V(4).Infof("Reconcile machineset %v", machineSet.Name) +func (r *ReconcileMachineSet) reconcile(ctx context.Context, log *zap.SugaredLogger, machineSet *clusterv1alpha1.MachineSet) (reconcile.Result, error) { + log.Debug("Reconcile MachineSet") allMachines := &clusterv1alpha1.MachineList{} if err := r.Client.List(ctx, allMachines, client.InNamespace(machineSet.Namespace)); err != nil { @@ -168,7 +186,6 @@ func (r *ReconcileMachineSet) reconcile(ctx context.Context, machineSet *cluster machineSet.Finalizers = append(machineSet.ObjectMeta.Finalizers, metav1.FinalizerDeleteDependents) if err := r.Client.Update(ctx, machineSet); err != nil { - klog.Infof("Failed to add finalizers to MachineSet %q: %v", machineSet.Name, err) return reconcile.Result{}, err } @@ -185,14 +202,16 @@ func (r *ReconcileMachineSet) reconcile(ctx context.Context, machineSet *cluster filteredMachines := make([]*clusterv1alpha1.Machine, 0, len(allMachines.Items)) for idx := range allMachines.Items { machine := &allMachines.Items[idx] - if shouldExcludeMachine(machineSet, machine) { + machineLog := log.With("machine", client.ObjectKeyFromObject(machine)) + + if shouldExcludeMachine(machineLog, machineSet, machine) { continue } // Attempt to adopt machine if it meets previous conditions and it has no controller references. if metav1.GetControllerOf(machine) == nil { if err := r.adoptOrphan(ctx, machineSet, machine); err != nil { - klog.Warningf("Failed to adopt MachineSet %q into MachineSet %q: %v", machine.Name, machineSet.Name, err) + machineLog.Errorw("Failed to adopt Machine into MachineSet", zap.Error(err)) continue } } @@ -200,13 +219,13 @@ func (r *ReconcileMachineSet) reconcile(ctx context.Context, machineSet *cluster filteredMachines = append(filteredMachines, machine) } - syncErr := r.syncReplicas(ctx, machineSet, filteredMachines) + syncErr := r.syncReplicas(ctx, log, machineSet, filteredMachines) ms := machineSet.DeepCopy() - newStatus := r.calculateStatus(ctx, ms, filteredMachines) + newStatus := r.calculateStatus(ctx, log, ms, filteredMachines) // Always updates status as machines come up or die. - updatedMS, err := updateMachineSetStatus(ctx, r.Client, machineSet, newStatus) + updatedMS, err := updateMachineSetStatus(ctx, log, r.Client, machineSet, newStatus) if err != nil { if syncErr != nil { return reconcile.Result{}, errors.Wrapf(err, "failed to sync machines: %v. failed to update machine set status", syncErr) @@ -239,27 +258,26 @@ func (r *ReconcileMachineSet) reconcile(ctx context.Context, machineSet *cluster } // syncReplicas scales Machine resources up or down. -func (r *ReconcileMachineSet) syncReplicas(ctx context.Context, ms *clusterv1alpha1.MachineSet, machines []*clusterv1alpha1.Machine) error { +func (r *ReconcileMachineSet) syncReplicas(ctx context.Context, log *zap.SugaredLogger, ms *clusterv1alpha1.MachineSet, machines []*clusterv1alpha1.Machine) error { if ms.Spec.Replicas == nil { return errors.Errorf("the Replicas field in Spec for machineset %v is nil, this should not be allowed", ms.Name) } diff := len(machines) - int(*(ms.Spec.Replicas)) + replicasLog := log.With("spec", *(ms.Spec.Replicas), "current", len(machines)) if diff < 0 { diff *= -1 - klog.Infof("Too few replicas for %v %s/%s, need %d, creating %d", - controllerKind, ms.Namespace, ms.Name, *(ms.Spec.Replicas), diff) + replicasLog.Infow("Too few replicas, creating more", "diff", diff) var machineList []*clusterv1alpha1.Machine var errstrings []string for i := 0; i < diff; i++ { - klog.Infof("Creating machine %d of %d, ( spec.replicas(%d) > currentMachineCount(%d) )", - i+1, diff, *(ms.Spec.Replicas), len(machines)) + replicasLog.Infow("Creating new machine", "index", i+1) machine := r.createMachine(ms) if err := r.Client.Create(ctx, machine); err != nil { - klog.Errorf("Unable to create Machine %q: %v", machine.Name, err) + log.Errorw("Failed to create Machine", "machine", client.ObjectKeyFromObject(machine), zap.Error(err)) errstrings = append(errstrings, err.Error()) continue } @@ -271,16 +289,15 @@ func (r *ReconcileMachineSet) syncReplicas(ctx context.Context, ms *clusterv1alp return errors.New(strings.Join(errstrings, "; ")) } - return r.waitForMachineCreation(ctx, machineList) + return r.waitForMachineCreation(ctx, log, machineList) } else if diff > 0 { - klog.Infof("Too many replicas for %v %s/%s, need %d, deleting %d", - controllerKind, ms.Namespace, ms.Name, *(ms.Spec.Replicas), diff) + replicasLog.Infow("Too many replicas, deleting extras", "diff", diff, "deletepolicy", ms.Spec.DeletePolicy) deletePriorityFunc, err := getDeletePriorityFunc(ms) if err != nil { return err } - klog.Infof("Found %s delete policy", ms.Spec.DeletePolicy) + // Choose which Machines to delete. machinesToDelete := getMachinesToDeletePrioritized(machines, diff, deletePriorityFunc) @@ -293,7 +310,7 @@ func (r *ReconcileMachineSet) syncReplicas(ctx context.Context, ms *clusterv1alp defer wg.Done() err := r.Client.Delete(ctx, targetMachine) if err != nil { - klog.Errorf("Unable to delete Machine %s: %v", targetMachine.Name, err) + log.Errorw("Failed to delete Machine", "machine", client.ObjectKeyFromObject(targetMachine), zap.Error(err)) errCh <- err } }(machine) @@ -334,10 +351,10 @@ func (r *ReconcileMachineSet) createMachine(machineSet *clusterv1alpha1.MachineS } // shouldExcludeMachine returns true if the machine should be filtered out, false otherwise. -func shouldExcludeMachine(machineSet *clusterv1alpha1.MachineSet, machine *clusterv1alpha1.Machine) bool { +func shouldExcludeMachine(machineLog *zap.SugaredLogger, machineSet *clusterv1alpha1.MachineSet, machine *clusterv1alpha1.Machine) bool { // Ignore inactive machines. if metav1.GetControllerOf(machine) != nil && !metav1.IsControlledBy(machine, machineSet) { - klog.V(4).Infof("%s not controlled by %v", machine.Name, machineSet.Name) + machineLog.Debug("Machine not controlled by MachineSet") return true } @@ -345,7 +362,7 @@ func shouldExcludeMachine(machineSet *clusterv1alpha1.MachineSet, machine *clust return true } - if !hasMatchingLabels(machineSet, machine) { + if !hasMatchingLabels(machineLog, machineSet, machine) { return true } @@ -359,7 +376,7 @@ func (r *ReconcileMachineSet) adoptOrphan(ctx context.Context, machineSet *clust return r.Client.Update(ctx, machine) } -func (r *ReconcileMachineSet) waitForMachineCreation(ctx context.Context, machineList []*clusterv1alpha1.Machine) error { +func (r *ReconcileMachineSet) waitForMachineCreation(ctx context.Context, log *zap.SugaredLogger, machineList []*clusterv1alpha1.Machine) error { for _, machine := range machineList { pollErr := wait.PollImmediate(stateConfirmationInterval, stateConfirmationTimeout, func() (bool, error) { key := client.ObjectKey{Namespace: machine.Namespace, Name: machine.Name} @@ -368,7 +385,7 @@ func (r *ReconcileMachineSet) waitForMachineCreation(ctx context.Context, machin if apierrors.IsNotFound(err) { return false, nil } - klog.Error(err) + log.Error(err) return false, err } @@ -376,7 +393,6 @@ func (r *ReconcileMachineSet) waitForMachineCreation(ctx context.Context, machin }) if pollErr != nil { - klog.Error(pollErr) return errors.Wrap(pollErr, "failed waiting for machine object to be created") } } @@ -399,7 +415,6 @@ func (r *ReconcileMachineSet) waitForMachineDeletion(ctx context.Context, machin }) if pollErr != nil { - klog.Error(pollErr) return errors.Wrap(pollErr, "failed waiting for machine object to be deleted") } } @@ -414,9 +429,11 @@ func (r *ReconcileMachineSet) MachineToMachineSets(o client.Object) []reconcile. m := &clusterv1alpha1.Machine{} key := client.ObjectKey{Namespace: o.GetNamespace(), Name: o.GetName()} + machineLog := r.log.With("machine", key) + if err := r.Client.Get(ctx, key, m); err != nil { if !apierrors.IsNotFound(err) { - klog.Errorf("Unable to retrieve Machine %q for possible MachineSet adoption: %v", key, err) + machineLog.Errorw("Failed to retrieve Machine for possible MachineSet adoption", zap.Error(err)) } return nil } @@ -429,9 +446,9 @@ func (r *ReconcileMachineSet) MachineToMachineSets(o client.Object) []reconcile. } } - mss := r.getMachineSetsForMachine(ctx, m) + mss := r.getMachineSetsForMachine(ctx, machineLog, m) if len(mss) == 0 { - klog.V(4).Infof("Found no MachineSet for Machine %q", m.Name) + machineLog.Debug("Found no MachineSet for Machine") return nil } diff --git a/pkg/controller/machineset/machine.go b/pkg/controller/machineset/machine.go index c64167c66..8a3cb5d75 100644 --- a/pkg/controller/machineset/machine.go +++ b/pkg/controller/machineset/machine.go @@ -19,17 +19,18 @@ package machineset import ( "context" + "go.uber.org/zap" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/klog" "sigs.k8s.io/controller-runtime/pkg/client" ) -func (c *ReconcileMachineSet) getMachineSetsForMachine(ctx context.Context, m *v1alpha1.Machine) []*v1alpha1.MachineSet { +func (c *ReconcileMachineSet) getMachineSetsForMachine(ctx context.Context, machineLog *zap.SugaredLogger, m *v1alpha1.Machine) []*v1alpha1.MachineSet { if len(m.Labels) == 0 { - klog.Warningf("No machine sets found for Machine %v because it has no labels", m.Name) + machineLog.Infow("No MachineSets found for Machine because it has no labels") return nil } @@ -40,14 +41,14 @@ func (c *ReconcileMachineSet) getMachineSetsForMachine(ctx context.Context, m *v err := c.Client.List(ctx, msList, listOptions) if err != nil { - klog.Errorf("Failed to list machine sets, %v", err) + machineLog.Errorw("Failed to list MachineSets", zap.Error(err)) return nil } var mss []*v1alpha1.MachineSet for idx := range msList.Items { ms := &msList.Items[idx] - if hasMatchingLabels(ms, m) { + if hasMatchingLabels(machineLog, ms, m) { mss = append(mss, ms) } } @@ -55,21 +56,21 @@ func (c *ReconcileMachineSet) getMachineSetsForMachine(ctx context.Context, m *v return mss } -func hasMatchingLabels(machineSet *v1alpha1.MachineSet, machine *v1alpha1.Machine) bool { +func hasMatchingLabels(machineLog *zap.SugaredLogger, machineSet *v1alpha1.MachineSet, machine *v1alpha1.Machine) bool { selector, err := metav1.LabelSelectorAsSelector(&machineSet.Spec.Selector) if err != nil { - klog.Warningf("unable to convert selector: %v", err) + machineLog.Errorw("Failed to convert selector", zap.Error(err)) return false } // If a deployment with a nil or empty selector creeps in, it should match nothing, not everything. if selector.Empty() { - klog.V(2).Infof("%v machineset has empty selector", machineSet.Name) + machineLog.Info("MachineSet has empty selector") return false } if !selector.Matches(labels.Set(machine.Labels)) { - klog.V(4).Infof("%v machine has mismatch labels", machine.Name) + machineLog.Debug("Machine has mismatch labels") return false } diff --git a/pkg/controller/machineset/status.go b/pkg/controller/machineset/status.go index b8829174e..0a207835c 100644 --- a/pkg/controller/machineset/status.go +++ b/pkg/controller/machineset/status.go @@ -18,17 +18,16 @@ package machineset import ( "context" - "fmt" "time" "github.com/pkg/errors" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/klog" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -37,7 +36,7 @@ const ( statusUpdateRetries = 1 ) -func (c *ReconcileMachineSet) calculateStatus(ctx context.Context, ms *v1alpha1.MachineSet, filteredMachines []*v1alpha1.Machine) v1alpha1.MachineSetStatus { +func (c *ReconcileMachineSet) calculateStatus(ctx context.Context, log *zap.SugaredLogger, ms *v1alpha1.MachineSet, filteredMachines []*v1alpha1.Machine) v1alpha1.MachineSetStatus { newStatus := ms.Status // Count the number of machines that have labels matching the labels of the machine // template of the replica set, the matching machines may have more @@ -54,7 +53,7 @@ func (c *ReconcileMachineSet) calculateStatus(ctx context.Context, ms *v1alpha1. } node, err := c.getMachineNode(ctx, machine) if err != nil { - klog.V(4).Infof("Unable to get node for machine %v, %v", machine.Name, err) + log.Debugw("Failed to get node for machine", "machine", client.ObjectKeyFromObject(machine), zap.Error(err)) continue } if isNodeReady(node) { @@ -73,7 +72,7 @@ func (c *ReconcileMachineSet) calculateStatus(ctx context.Context, ms *v1alpha1. } // updateMachineSetStatus attempts to update the Status.Replicas of the given MachineSet, with a single GET/PUT retry. -func updateMachineSetStatus(ctx context.Context, c client.Client, ms *v1alpha1.MachineSet, newStatus v1alpha1.MachineSetStatus) (*v1alpha1.MachineSet, error) { +func updateMachineSetStatus(ctx context.Context, log *zap.SugaredLogger, c client.Client, ms *v1alpha1.MachineSet, newStatus v1alpha1.MachineSetStatus) (*v1alpha1.MachineSet, error) { // This is the steady state. It happens when the MachineSet doesn't have any expectations, since // we do a periodic relist every 30s. If the generations differ but the replicas are // the same, a caller might've resized to the same replica count. @@ -97,12 +96,20 @@ func updateMachineSetStatus(ctx context.Context, c client.Client, ms *v1alpha1.M if ms.Spec.Replicas != nil { replicas = *ms.Spec.Replicas } - klog.V(4).Infof(fmt.Sprintf("Updating status for %v: %s/%s, ", ms.Kind, ms.Namespace, ms.Name) + - fmt.Sprintf("replicas %d->%d (need %d), ", ms.Status.Replicas, newStatus.Replicas, replicas) + - fmt.Sprintf("fullyLabeledReplicas %d->%d, ", ms.Status.FullyLabeledReplicas, newStatus.FullyLabeledReplicas) + - fmt.Sprintf("readyReplicas %d->%d, ", ms.Status.ReadyReplicas, newStatus.ReadyReplicas) + - fmt.Sprintf("availableReplicas %d->%d, ", ms.Status.AvailableReplicas, newStatus.AvailableReplicas) + - fmt.Sprintf("sequence No: %v->%v", ms.Status.ObservedGeneration, newStatus.ObservedGeneration)) + + log.Debugw("Updating status", + "specreplicas", replicas, + "oldreplicas", ms.Status.Replicas, + "newreplicas", newStatus.Replicas, + "oldlabeledreplicas", ms.Status.FullyLabeledReplicas, + "newlabeledreplicas", newStatus.FullyLabeledReplicas, + "oldreadyreplicas", ms.Status.ReadyReplicas, + "newreadyreplicas", newStatus.ReadyReplicas, + "oldavailablereplicas", ms.Status.AvailableReplicas, + "newavailablereplicas", newStatus.AvailableReplicas, + "oldobservedgeneration", ms.Status.ObservedGeneration, + "newobservedgeneration", newStatus.ObservedGeneration, + ) ms.Status = newStatus updateErr = c.Status().Update(ctx, ms) diff --git a/pkg/controller/nodecsrapprover/node_csr_approver.go b/pkg/controller/nodecsrapprover/controller.go similarity index 86% rename from pkg/controller/nodecsrapprover/node_csr_approver.go rename to pkg/controller/nodecsrapprover/controller.go index 5997a105e..de00fc679 100644 --- a/pkg/controller/nodecsrapprover/node_csr_approver.go +++ b/pkg/controller/nodecsrapprover/controller.go @@ -23,6 +23,10 @@ import ( "fmt" "strings" + "github.com/go-logr/logr" + "github.com/go-logr/zapr" + "go.uber.org/zap" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" certificatesv1 "k8s.io/api/certificates/v1" @@ -31,7 +35,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" certificatesv1client "k8s.io/client-go/kubernetes/typed/certificates/v1" - "k8s.io/klog" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -42,7 +45,7 @@ import ( const ( // ControllerName is name of the NodeCSRApprover controller. - ControllerName = "node_csr_autoapprover" + ControllerName = "node-csr-approver-controller" nodeUser = "system:node" nodeUserPrefix = nodeUser + ":" @@ -61,21 +64,32 @@ var ( type reconciler struct { client.Client + log *zap.SugaredLogger // Have to use the typed client because csr approval is a subresource // the dynamic client does not approve certClient certificatesv1client.CertificateSigningRequestInterface } -func Add(mgr manager.Manager) error { +func Add(mgr manager.Manager, log *zap.SugaredLogger) error { certClient, err := certificatesv1client.NewForConfig(mgr.GetConfig()) if err != nil { return fmt.Errorf("failed to create certificate client: %w", err) } - rec := &reconciler{Client: mgr.GetClient(), certClient: certClient.CertificateSigningRequests()} + rec := &reconciler{ + Client: mgr.GetClient(), + log: log.Named(ControllerName), + certClient: certClient.CertificateSigningRequests(), + } watchType := &certificatesv1.CertificateSigningRequest{} - cntrl, err := controller.New(ControllerName, mgr, controller.Options{Reconciler: rec}) + cntrl, err := controller.New(ControllerName, mgr, controller.Options{ + Reconciler: rec, + LogConstructor: func(request *reconcile.Request) logr.Logger { + // we log ourselves + return zapr.NewLogger(zap.NewNop()) + }, + }) if err != nil { return fmt.Errorf("failed to construct controller: %w", err) } @@ -84,28 +98,32 @@ func Add(mgr manager.Manager) error { } func (r *reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { - err := r.reconcile(ctx, request) - if err != nil { - klog.Errorf("Reconciliation of request %s failed: %v", request.NamespacedName.String(), err) - } - return reconcile.Result{}, err -} + log := r.log.With("csr", request.NamespacedName) + log.Debug("Reconciling") -func (r *reconciler) reconcile(ctx context.Context, request reconcile.Request) error { // Get the CSR object csr := &certificatesv1.CertificateSigningRequest{} if err := r.Get(ctx, request.NamespacedName, csr); err != nil { if kerrors.IsNotFound(err) { - return nil + return reconcile.Result{}, nil } - return err + log.Errorw("Failed to get CertificateSigningRequest", zap.Error(err)) + return reconcile.Result{}, err + } + + err := r.reconcile(ctx, log, csr) + if err != nil { + log.Errorw("Reconciling failed", zap.Error(err)) } - klog.V(4).Infof("Reconciling CSR %s", csr.ObjectMeta.Name) + return reconcile.Result{}, err +} + +func (r *reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, csr *certificatesv1.CertificateSigningRequest) error { // If CSR is approved, skip it for _, condition := range csr.Status.Conditions { if condition.Type == certificatesv1.CertificateApproved { - klog.V(4).Infof("CSR %s already approved, skipping reconciling", csr.ObjectMeta.Name) + log.Debug("CSR already approved, skipping reconciling") return nil } } @@ -113,7 +131,7 @@ func (r *reconciler) reconcile(ctx context.Context, request reconcile.Request) e // Validate the CSR object and get the node name nodeName, err := r.validateCSRObject(csr) if err != nil { - klog.V(4).Infof("Skipping reconciling CSR '%s' because CSR object is not valid: %v", csr.ObjectMeta.Name, err) + log.Debugw("Skipping reconciling CSR because object is invalid", zap.Error(err)) return nil } @@ -145,7 +163,8 @@ func (r *reconciler) reconcile(ctx context.Context, request reconcile.Request) e } // Approve CSR - klog.V(4).Infof("Approving CSR %s", csr.ObjectMeta.Name) + nodeLog := log.With("node", nodeName) + nodeLog.Debug("Approving CSR") approvalCondition := certificatesv1.CertificateSigningRequestCondition{ Type: certificatesv1.CertificateApproved, Reason: "machine-controller NodeCSRApprover controller approved node serving cert", @@ -157,7 +176,7 @@ func (r *reconciler) reconcile(ctx context.Context, request reconcile.Request) e return fmt.Errorf("failed to approve CSR %q: %w", csr.Name, err) } - klog.Infof("Successfully approved CSR %s", csr.ObjectMeta.Name) + nodeLog.Info("Successfully approved CSR") return nil } diff --git a/pkg/controller/nodecsrapprover/node_csr_approver_test.go b/pkg/controller/nodecsrapprover/controller_test.go similarity index 100% rename from pkg/controller/nodecsrapprover/node_csr_approver_test.go rename to pkg/controller/nodecsrapprover/controller_test.go diff --git a/pkg/controller/util/machine_deployment.go b/pkg/controller/util/machine_deployment.go index 034510ca7..5b7a764ad 100644 --- a/pkg/controller/util/machine_deployment.go +++ b/pkg/controller/util/machine_deployment.go @@ -25,6 +25,7 @@ import ( "strings" "github.com/davecgh/go-spew/spew" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -35,8 +36,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" intstrutil "k8s.io/apimachinery/pkg/util/intstr" - "k8s.io/klog" "k8s.io/utils/integer" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -130,12 +131,15 @@ func SetDeploymentRevision(deployment *v1alpha1.MachineDeployment, revision stri } // MaxRevision finds the highest revision in the machine sets. -func MaxRevision(allMSs []*v1alpha1.MachineSet) int64 { +func MaxRevision(log *zap.SugaredLogger, allMSs []*v1alpha1.MachineSet) int64 { max := int64(0) for _, ms := range allMSs { if v, err := Revision(ms); err != nil { - // Skip the machine sets when it failed to parse their revision information - klog.V(4).Infof("Error: %v. Couldn't parse revision for machine set %#v, deployment controller will skip it when reconciling revisions.", err, ms) + log.Debugw( + "Failed to parse revision for MachineSet, deployment controller will skip it when reconciling revisions", + "machinset", client.ObjectKeyFromObject(ms), + zap.Error(err), + ) } else if v > max { max = v } @@ -194,22 +198,22 @@ func copyDeploymentAnnotationsToMachineSet(deployment *v1alpha1.MachineDeploymen } // GetDesiredReplicasAnnotation returns the number of desired replicas. -func GetDesiredReplicasAnnotation(ms *v1alpha1.MachineSet) (int32, bool) { - return getIntFromAnnotation(ms, DesiredReplicasAnnotation) +func GetDesiredReplicasAnnotation(log *zap.SugaredLogger, ms *v1alpha1.MachineSet) (int32, bool) { + return getIntFromAnnotation(log, ms, DesiredReplicasAnnotation) } -func getMaxReplicasAnnotation(ms *v1alpha1.MachineSet) (int32, bool) { - return getIntFromAnnotation(ms, MaxReplicasAnnotation) +func getMaxReplicasAnnotation(log *zap.SugaredLogger, ms *v1alpha1.MachineSet) (int32, bool) { + return getIntFromAnnotation(log, ms, MaxReplicasAnnotation) } -func getIntFromAnnotation(ms *v1alpha1.MachineSet, annotationKey string) (int32, bool) { +func getIntFromAnnotation(log *zap.SugaredLogger, ms *v1alpha1.MachineSet, annotationKey string) (int32, bool) { annotationValue, ok := ms.Annotations[annotationKey] if !ok { return int32(0), false } intValue, err := strconv.Atoi(annotationValue) if err != nil { - klog.V(2).Infof("Cannot convert the value %q with annotation key %q for the machine set %q", annotationValue, annotationKey, ms.Name) + log.Infow("Failed to convert annotation to integer", "key", annotationKey, "value", annotationValue, zap.Error(err)) return int32(0), false } return int32(intValue), true @@ -217,7 +221,7 @@ func getIntFromAnnotation(ms *v1alpha1.MachineSet, annotationKey string) (int32, // SetNewMachineSetAnnotations sets new machine set's annotations appropriately by updating its revision and // copying required deployment annotations to it; it returns true if machine set's annotation is changed. -func SetNewMachineSetAnnotations(deployment *v1alpha1.MachineDeployment, newMS *v1alpha1.MachineSet, newRevision string, exists bool) bool { +func SetNewMachineSetAnnotations(mdLog *zap.SugaredLogger, deployment *v1alpha1.MachineDeployment, newMS *v1alpha1.MachineSet, newRevision string, exists bool) bool { // First, copy deployment's annotations (except for apply and revision annotations) annotationChanged := copyDeploymentAnnotationsToMachineSet(deployment, newMS) // Then, update machine set's revision annotation @@ -229,25 +233,31 @@ func SetNewMachineSetAnnotations(deployment *v1alpha1.MachineDeployment, newMS * // of all old MSes + 1). However, it's possible that some of the old MSes are deleted after the newMS revision being updated, and // newRevision becomes smaller than newMS's revision. We should only update newMS revision when it's smaller than newRevision. + msLog := mdLog.With("machineset", client.ObjectKeyFromObject(newMS)) + oldRevisionInt, err := strconv.ParseInt(oldRevision, 10, 64) if err != nil { if oldRevision != "" { - klog.Warningf("Updating machine set revision OldRevision not int %s", err) + msLog.Infow("MachineSet revision annotation is not a valid integer", "value", oldRevision, zap.Error(err)) return false } //If the MS annotation is empty then initialise it to 0 oldRevisionInt = 0 } + newRevisionInt, err := strconv.ParseInt(newRevision, 10, 64) if err != nil { - klog.Warningf("Updating machine set revision NewRevision not int %s", err) + // This should never happen, as newRevision is calculated by the machine-controller itself. + msLog.Errorw("New MachineSet revision annotation is not a valid integer", "value", newRevision, zap.Error(err)) return false } + if oldRevisionInt < newRevisionInt { newMS.Annotations[RevisionAnnotation] = newRevision annotationChanged = true - klog.V(4).Infof("Updating machine set %q revision to %s", newMS.Name, newRevision) + msLog.Debugw("Updating MachineSet revision", "revision", newRevision) } + // If a revision annotation already existed and this machine set was updated with a new revision // then that means we are rolling back to this machine set. We need to preserve the old revisions // for historical information. @@ -354,12 +364,12 @@ func MaxSurge(deployment v1alpha1.MachineDeployment) int32 { // GetProportion will estimate the proportion for the provided machine set using 1. the current size // of the parent deployment, 2. the replica count that needs be added on the machine sets of the // deployment, and 3. the total replicas added in the machine sets of the deployment so far. -func GetProportion(ms *v1alpha1.MachineSet, d v1alpha1.MachineDeployment, deploymentReplicasToAdd, deploymentReplicasAdded int32) int32 { +func GetProportion(log *zap.SugaredLogger, ms *v1alpha1.MachineSet, d v1alpha1.MachineDeployment, deploymentReplicasToAdd, deploymentReplicasAdded int32) int32 { if ms == nil || *(ms.Spec.Replicas) == 0 || deploymentReplicasToAdd == 0 || deploymentReplicasToAdd == deploymentReplicasAdded { return int32(0) } - msFraction := getMachineSetFraction(*ms, d) + msFraction := getMachineSetFraction(log, *ms, d) allowed := deploymentReplicasToAdd - deploymentReplicasAdded if deploymentReplicasToAdd > 0 { @@ -376,14 +386,14 @@ func GetProportion(ms *v1alpha1.MachineSet, d v1alpha1.MachineDeployment, deploy // getMachineSetFraction estimates the fraction of replicas a machine set can have in // 1. a scaling event during a rollout or 2. when scaling a paused deployment. -func getMachineSetFraction(ms v1alpha1.MachineSet, d v1alpha1.MachineDeployment) int32 { +func getMachineSetFraction(log *zap.SugaredLogger, ms v1alpha1.MachineSet, d v1alpha1.MachineDeployment) int32 { // If we are scaling down to zero then the fraction of this machine set is its whole size (negative) if *(d.Spec.Replicas) == int32(0) { return -*(ms.Spec.Replicas) } deploymentReplicas := *(d.Spec.Replicas) + MaxSurge(d) - annotatedReplicas, ok := getMaxReplicasAnnotation(&ms) + annotatedReplicas, ok := getMaxReplicasAnnotation(log, &ms) if !ok { // If we cannot find the annotation then fallback to the current deployment size. Note that this // will not be an accurate proportion estimation in case other machine sets have different values diff --git a/pkg/health/readiness.go b/pkg/health/readiness.go index d79a21c6c..8ec6176b6 100644 --- a/pkg/health/readiness.go +++ b/pkg/health/readiness.go @@ -21,6 +21,8 @@ import ( "fmt" "net/http" + "go.uber.org/zap" + machinecontroller "github.com/kubermatic/machine-controller/pkg/controller/machine" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,18 +34,18 @@ func ApiserverReachable(client kubernetes.Interface) healthz.Checker { return func(req *http.Request) error { _, err := client.CoreV1().Nodes().List(req.Context(), metav1.ListOptions{}) if err != nil { - return fmt.Errorf("unable to list nodes check: %w", err) + return fmt.Errorf("failed to list nodes check: %w", err) } return nil } } -func KubeconfigAvailable(kubeconfigProvider machinecontroller.KubeconfigProvider) healthz.Checker { +func KubeconfigAvailable(kubeconfigProvider machinecontroller.KubeconfigProvider, log *zap.SugaredLogger) healthz.Checker { return func(req *http.Request) error { - cm, err := kubeconfigProvider.GetKubeconfig(req.Context()) + cm, err := kubeconfigProvider.GetKubeconfig(req.Context(), log) if err != nil { - return fmt.Errorf("unable to get kubeconfig: %w", err) + return fmt.Errorf("failed to get kubeconfig: %w", err) } if len(cm.Clusters) != 1 { diff --git a/pkg/log/zap.go b/pkg/log/zap.go new file mode 100644 index 000000000..1d13484ea --- /dev/null +++ b/pkg/log/zap.go @@ -0,0 +1,168 @@ +/* +Copyright 2019 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package log has been graciously copied from KKP (Copyright 2020 The Kubermatic Kubernetes Platform contributors). +package log + +import ( + "flag" + "fmt" + "os" + "strings" + + "github.com/spf13/pflag" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + + ctrlruntimelzap "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +func init() { + Logger = NewDefault().Sugar() +} + +var Logger *zap.SugaredLogger + +// Options exports a options struct to be used by cmd's. +type Options struct { + // Enable debug logs + Debug bool + // Log format (JSON or plain text) + Format Format +} + +func NewDefaultOptions() Options { + return Options{ + Debug: false, + Format: FormatJSON, + } +} + +func (o *Options) AddFlags(fs *flag.FlagSet) { + fs.BoolVar(&o.Debug, "log-debug", o.Debug, "Enables more verbose logging") + fs.Var(&o.Format, "log-format", "Log format, one of "+AvailableFormats.String()) +} + +func (o *Options) AddPFlags(fs *pflag.FlagSet) { + fs.BoolVar(&o.Debug, "log-debug", o.Debug, "Enables more verbose logging") + fs.Var(&o.Format, "log-format", "Log format, one of "+AvailableFormats.String()) +} + +func (o *Options) Validate() error { + if !AvailableFormats.Contains(o.Format) { + return fmt.Errorf("invalid log-format specified %q; available: %s", o.Format, AvailableFormats.String()) + } + return nil +} + +type Format string + +// Type implements the pflag.Value interfaces. +func (f *Format) Type() string { + return "string" +} + +// String implements the cli.Value and flag.Value interfaces. +func (f *Format) String() string { + return string(*f) +} + +// Set implements the cli.Value and flag.Value interfaces. +func (f *Format) Set(s string) error { + switch strings.ToLower(s) { + case "json": + *f = FormatJSON + return nil + case "console": + *f = FormatConsole + return nil + default: + return fmt.Errorf("invalid format '%s'", s) + } +} + +type Formats []Format + +const ( + FormatJSON Format = "JSON" + FormatConsole Format = "Console" +) + +var ( + AvailableFormats = Formats{FormatJSON, FormatConsole} +) + +func (f Formats) String() string { + const separator = ", " + var s string + for _, format := range f { + s = s + separator + string(format) + } + return strings.TrimPrefix(s, separator) +} + +func (f Formats) Contains(s Format) bool { + for _, format := range f { + if s == format { + return true + } + } + return false +} + +func NewFromOptions(o Options) *zap.Logger { + return New(o.Debug, o.Format) +} + +func New(debug bool, format Format) *zap.Logger { + // this basically mimics NewConfig, but with a custom sink + sink := zapcore.AddSync(os.Stderr) + + // Level - We only support setting Info+ or Debug+ + lvl := zap.NewAtomicLevelAt(zap.InfoLevel) + if debug { + lvl = zap.NewAtomicLevelAt(zap.DebugLevel) + } + + encCfg := zap.NewProductionEncoderConfig() + // Having a dateformat makes it more easy to look at logs outside of something like Kibana + encCfg.TimeKey = "time" + encCfg.EncodeTime = zapcore.ISO8601TimeEncoder + + // production config encodes durations as a float of the seconds value, but we want a more + // readable, precise representation + encCfg.EncodeDuration = zapcore.StringDurationEncoder + + var enc zapcore.Encoder + if format == FormatJSON { + enc = zapcore.NewJSONEncoder(encCfg) + } else { + enc = zapcore.NewConsoleEncoder(encCfg) + } + + opts := []zap.Option{ + zap.AddCaller(), + zap.ErrorOutput(sink), + } + + coreLog := zapcore.NewCore(&ctrlruntimelzap.KubeAwareEncoder{Encoder: enc}, sink, lvl) + return zap.New(coreLog, opts...) +} + +// NewDefault creates new default logger. +func NewDefault() *zap.Logger { + return New(false, FormatJSON) +} diff --git a/pkg/node/eviction/eviction.go b/pkg/node/eviction/eviction.go index d22679b84..10d81e139 100644 --- a/pkg/node/eviction/eviction.go +++ b/pkg/node/eviction/eviction.go @@ -21,6 +21,8 @@ import ( "fmt" "sync" + "go.uber.org/zap" + evictiontypes "github.com/kubermatic/machine-controller/pkg/node/eviction/types" "github.com/kubermatic/machine-controller/pkg/node/nodemanager" @@ -30,68 +32,68 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/kubernetes" - "k8s.io/klog" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) type NodeEviction struct { nodeManager *nodemanager.NodeManager - ctx context.Context nodeName string kubeClient kubernetes.Interface } // New returns a new NodeEviction. -func New(ctx context.Context, nodeName string, client ctrlruntimeclient.Client, kubeClient kubernetes.Interface) *NodeEviction { +func New(nodeName string, client ctrlruntimeclient.Client, kubeClient kubernetes.Interface) *NodeEviction { return &NodeEviction{ - nodeManager: nodemanager.New(ctx, client, nodeName), - ctx: ctx, + nodeManager: nodemanager.New(client, nodeName), nodeName: nodeName, kubeClient: kubeClient, } } // Run executes the eviction. -func (ne *NodeEviction) Run() (bool, error) { - node, err := ne.nodeManager.GetNode() +func (ne *NodeEviction) Run(ctx context.Context, log *zap.SugaredLogger) (bool, error) { + nodeLog := log.With("node", ne.nodeName) + + node, err := ne.nodeManager.GetNode(ctx) if err != nil { return false, fmt.Errorf("failed to get node from lister: %w", err) } if _, exists := node.Annotations[evictiontypes.SkipEvictionAnnotationKey]; exists { - klog.V(3).Infof("Skipping eviction for node %s as it has a %s annotation", ne.nodeName, evictiontypes.SkipEvictionAnnotationKey) + nodeLog.Info("Skipping eviction for node as it has a %s annotation", evictiontypes.SkipEvictionAnnotationKey) return false, nil } - klog.V(3).Infof("Starting to evict node %s", ne.nodeName) - if err := ne.nodeManager.CordonNode(node); err != nil { + nodeLog.Info("Starting to evict node") + + if err := ne.nodeManager.CordonNode(ctx, node); err != nil { return false, fmt.Errorf("failed to cordon node %s: %w", ne.nodeName, err) } - klog.V(6).Infof("Successfully cordoned node %s", ne.nodeName) + nodeLog.Debug("Successfully cordoned node") - podsToEvict, err := ne.getFilteredPods() + podsToEvict, err := ne.getFilteredPods(ctx) if err != nil { return false, fmt.Errorf("failed to get Pods to evict for node %s: %w", ne.nodeName, err) } - klog.V(6).Infof("Found %v pods to evict for node %s", len(podsToEvict), ne.nodeName) + nodeLog.Debugf("Found %d pods to evict for node", len(podsToEvict)) if len(podsToEvict) == 0 { return false, nil } // If we arrived here we have pods to evict, so tell the controller to retry later - if errs := ne.evictPods(podsToEvict); len(errs) > 0 { + if errs := ne.evictPods(ctx, nodeLog, podsToEvict); len(errs) > 0 { return true, fmt.Errorf("failed to evict pods, errors encountered: %v", errs) } - klog.V(6).Infof("Successfully created evictions for all pods on node %s!", ne.nodeName) + nodeLog.Debug("Successfully created evictions for all pods on node") return true, nil } -func (ne *NodeEviction) getFilteredPods() ([]corev1.Pod, error) { +func (ne *NodeEviction) getFilteredPods(ctx context.Context) ([]corev1.Pod, error) { // The lister-backed client from the mgr automatically creates a lister for all objects requested through it. // We explicitly do not want that for pods, hence we have to use the kubernetes core client // TODO @alvaroaleman: Add source code ref for this - pods, err := ne.kubeClient.CoreV1().Pods(metav1.NamespaceAll).List(ne.ctx, metav1.ListOptions{ + pods, err := ne.kubeClient.CoreV1().Pods(metav1.NamespaceAll).List(ctx, metav1.ListOptions{ FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": ne.nodeName}).String(), }) if err != nil { @@ -115,7 +117,7 @@ func (ne *NodeEviction) getFilteredPods() ([]corev1.Pod, error) { return filteredPods, nil } -func (ne *NodeEviction) evictPods(pods []corev1.Pod) []error { +func (ne *NodeEviction) evictPods(ctx context.Context, log *zap.SugaredLogger, pods []corev1.Pod) []error { errCh := make(chan error, len(pods)) retErrs := []error{} @@ -131,9 +133,9 @@ func (ne *NodeEviction) evictPods(pods []corev1.Pod) []error { if isDone { return } - err := ne.evictPod(&p) + err := ne.evictPod(ctx, &p) if err == nil || kerrors.IsNotFound(err) { - klog.V(6).Infof("Successfully evicted pod %s/%s on node %s", p.Namespace, p.Name, ne.nodeName) + log.Debugw("Successfully evicted pod on node", "pod", ctrlruntimeclient.ObjectKeyFromObject(&p)) return } else if kerrors.IsTooManyRequests(err) { // PDB prevents eviction, return and make the controller retry later @@ -151,22 +153,22 @@ func (ne *NodeEviction) evictPods(pods []corev1.Pod) []error { select { case <-finished: - klog.V(6).Infof("All goroutines for eviction pods on node %s finished", ne.nodeName) + log.Debug("All goroutines for eviction pods on node finished") break case err := <-errCh: - klog.V(6).Infof("Got an error from eviction goroutine for node %s: %v", ne.nodeName, err) + log.Debugw("Got an error from eviction goroutine for node", zap.Error(err)) retErrs = append(retErrs, err) } return retErrs } -func (ne *NodeEviction) evictPod(pod *corev1.Pod) error { +func (ne *NodeEviction) evictPod(ctx context.Context, pod *corev1.Pod) error { eviction := &policy.Eviction{ ObjectMeta: metav1.ObjectMeta{ Name: pod.Name, Namespace: pod.Namespace, }, } - return ne.kubeClient.PolicyV1beta1().Evictions(eviction.Namespace).Evict(ne.ctx, eviction) + return ne.kubeClient.PolicyV1beta1().Evictions(eviction.Namespace).Evict(ctx, eviction) } diff --git a/pkg/node/eviction/eviction_test.go b/pkg/node/eviction/eviction_test.go index 61ed90a5e..29d1d22ef 100644 --- a/pkg/node/eviction/eviction_test.go +++ b/pkg/node/eviction/eviction_test.go @@ -17,8 +17,11 @@ limitations under the License. package eviction import ( + "context" "testing" + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -56,7 +59,7 @@ func TestEvictPods(t *testing.T) { client := kubefake.NewSimpleClientset(test.Pods...) t.Run(test.Name, func(t *testing.T) { ne := &NodeEviction{kubeClient: client, nodeName: "node1"} - if errs := ne.evictPods(literalPods); len(errs) > 0 { + if errs := ne.evictPods(context.Background(), zap.NewNop().Sugar(), literalPods); len(errs) > 0 { t.Fatalf("Got unexpected errors=%v when running evictPods", errs) } diff --git a/pkg/node/nodemanager/node_manager.go b/pkg/node/nodemanager/node_manager.go index 1c69b5d90..2bea84d6c 100644 --- a/pkg/node/nodemanager/node_manager.go +++ b/pkg/node/nodemanager/node_manager.go @@ -29,30 +29,28 @@ import ( ) type NodeManager struct { - ctx context.Context client ctrlruntimeclient.Client nodeName string } -func New(ctx context.Context, client ctrlruntimeclient.Client, nodeName string) *NodeManager { +func New(client ctrlruntimeclient.Client, nodeName string) *NodeManager { return &NodeManager{ - ctx: ctx, client: client, nodeName: nodeName, } } -func (nm *NodeManager) GetNode() (*corev1.Node, error) { +func (nm *NodeManager) GetNode(ctx context.Context) (*corev1.Node, error) { node := &corev1.Node{} - if err := nm.client.Get(nm.ctx, types.NamespacedName{Name: nm.nodeName}, node); err != nil { + if err := nm.client.Get(ctx, types.NamespacedName{Name: nm.nodeName}, node); err != nil { return nil, fmt.Errorf("failed to get node from lister: %w", err) } return node, nil } -func (nm *NodeManager) CordonNode(node *corev1.Node) error { +func (nm *NodeManager) CordonNode(ctx context.Context, node *corev1.Node) error { if !node.Spec.Unschedulable { - _, err := nm.updateNode(func(n *corev1.Node) { + _, err := nm.updateNode(ctx, func(n *corev1.Node) { n.Spec.Unschedulable = true }) if err != nil { @@ -68,7 +66,7 @@ func (nm *NodeManager) CordonNode(node *corev1.Node) error { // not evicted return wait.Poll(1*time.Second, 10*time.Second, func() (bool, error) { node := &corev1.Node{} - if err := nm.client.Get(nm.ctx, types.NamespacedName{Name: nm.nodeName}, node); err != nil { + if err := nm.client.Get(ctx, types.NamespacedName{Name: nm.nodeName}, node); err != nil { return false, err } if node.Spec.Unschedulable { @@ -78,16 +76,16 @@ func (nm *NodeManager) CordonNode(node *corev1.Node) error { }) } -func (nm *NodeManager) updateNode(modify func(*corev1.Node)) (*corev1.Node, error) { +func (nm *NodeManager) updateNode(ctx context.Context, modify func(*corev1.Node)) (*corev1.Node, error) { node := &corev1.Node{} err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { - if err := nm.client.Get(nm.ctx, types.NamespacedName{Name: nm.nodeName}, node); err != nil { + if err := nm.client.Get(ctx, types.NamespacedName{Name: nm.nodeName}, node); err != nil { return err } // Apply modifications modify(node) // Update the node - return nm.client.Update(nm.ctx, node) + return nm.client.Update(ctx, node) }) return node, err diff --git a/pkg/node/poddeletion/pod_deletion.go b/pkg/node/poddeletion/pod_deletion.go index 6a8ecad62..980cb9b2d 100644 --- a/pkg/node/poddeletion/pod_deletion.go +++ b/pkg/node/poddeletion/pod_deletion.go @@ -21,13 +21,14 @@ import ( "fmt" "sync" + "go.uber.org/zap" + "github.com/kubermatic/machine-controller/pkg/node/nodemanager" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/klog" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -37,31 +38,31 @@ const ( type NodeVolumeAttachmentsCleanup struct { nodeManager *nodemanager.NodeManager - ctx context.Context nodeName string kubeClient kubernetes.Interface } // New returns a new NodeVolumeAttachmentsCleanup. -func New(ctx context.Context, nodeName string, client ctrlruntimeclient.Client, kubeClient kubernetes.Interface) *NodeVolumeAttachmentsCleanup { +func New(nodeName string, client ctrlruntimeclient.Client, kubeClient kubernetes.Interface) *NodeVolumeAttachmentsCleanup { return &NodeVolumeAttachmentsCleanup{ - nodeManager: nodemanager.New(ctx, client, nodeName), - ctx: ctx, + nodeManager: nodemanager.New(client, nodeName), nodeName: nodeName, kubeClient: kubeClient, } } // Run executes the pod deletion. -func (vc *NodeVolumeAttachmentsCleanup) Run() (bool, bool, error) { - node, err := vc.nodeManager.GetNode() +func (vc *NodeVolumeAttachmentsCleanup) Run(ctx context.Context, log *zap.SugaredLogger) (bool, bool, error) { + node, err := vc.nodeManager.GetNode(ctx) if err != nil { return false, false, fmt.Errorf("failed to get node from lister: %w", err) } - klog.V(3).Infof("Starting to cleanup node %s", vc.nodeName) + + nodeLog := log.With("node", vc.nodeName) + nodeLog.Info("Starting to cleanup node...") // if there are no more volumeAttachments related to the node, then it can be deleted. - volumeAttachmentsDeleted, err := vc.nodeCanBeDeleted() + volumeAttachmentsDeleted, err := vc.nodeCanBeDeleted(ctx, nodeLog) if err != nil { return false, false, fmt.Errorf("failed to check volumeAttachments deletion: %w", err) } @@ -70,42 +71,42 @@ func (vc *NodeVolumeAttachmentsCleanup) Run() (bool, bool, error) { } // cordon the node to be sure that the deleted pods are re-scheduled in the same node. - if err := vc.nodeManager.CordonNode(node); err != nil { + if err := vc.nodeManager.CordonNode(ctx, node); err != nil { return false, false, fmt.Errorf("failed to cordon node %s: %w", vc.nodeName, err) } - klog.V(6).Infof("Successfully cordoned node %s", vc.nodeName) + nodeLog.Debug("Successfully cordoned node.") // get all the pods that needs to be deleted (i.e. those mounting volumes attached to the node that is going to be deleted). - podsToDelete, errors := vc.getFilteredPods() + podsToDelete, errors := vc.getFilteredPods(ctx) if len(errors) > 0 { return false, false, fmt.Errorf("failed to get Pods to delete for node %s, errors encountered: %w", vc.nodeName, err) } - klog.V(6).Infof("Found %v pods to delete for node %s", len(podsToDelete), vc.nodeName) + nodeLog.Debugf("Found %d pods to delete for node", len(podsToDelete)) if len(podsToDelete) == 0 { return false, false, nil } // delete the previously filtered pods, then tells the controller to retry later. - if errs := vc.deletePods(podsToDelete); len(errs) > 0 { + if errs := vc.deletePods(ctx, nodeLog, podsToDelete); len(errs) > 0 { return false, false, fmt.Errorf("failed to delete pods, errors encountered: %v", errs) } - klog.V(6).Infof("Successfully deleted all pods mounting persistent volumes attached on node %s", vc.nodeName) + nodeLog.Debug("Successfully deleted all pods mounting persistent volumes attached on node") return true, false, err } -func (vc *NodeVolumeAttachmentsCleanup) getFilteredPods() ([]corev1.Pod, []error) { +func (vc *NodeVolumeAttachmentsCleanup) getFilteredPods(ctx context.Context) ([]corev1.Pod, []error) { filteredPods := []corev1.Pod{} lock := sync.Mutex{} retErrs := []error{} - volumeAttachments, err := vc.kubeClient.StorageV1().VolumeAttachments().List(vc.ctx, metav1.ListOptions{}) + volumeAttachments, err := vc.kubeClient.StorageV1().VolumeAttachments().List(ctx, metav1.ListOptions{}) if err != nil { retErrs = append(retErrs, fmt.Errorf("failed to list pods: %w", err)) return nil, retErrs } - persistentVolumeClaims, err := vc.kubeClient.CoreV1().PersistentVolumeClaims(metav1.NamespaceAll).List(vc.ctx, metav1.ListOptions{}) + persistentVolumeClaims, err := vc.kubeClient.CoreV1().PersistentVolumeClaims(metav1.NamespaceAll).List(ctx, metav1.ListOptions{}) if err != nil { retErrs = append(retErrs, fmt.Errorf("failed to list persistent volumes: %w", err)) return nil, retErrs @@ -120,7 +121,7 @@ func (vc *NodeVolumeAttachmentsCleanup) getFilteredPods() ([]corev1.Pod, []error wg.Add(1) go func(pvc corev1.PersistentVolumeClaim) { defer wg.Done() - pods, err := vc.kubeClient.CoreV1().Pods(pvc.Namespace).List(vc.ctx, metav1.ListOptions{}) + pods, err := vc.kubeClient.CoreV1().Pods(pvc.Namespace).List(ctx, metav1.ListOptions{}) switch { case kerrors.IsTooManyRequests(err): return @@ -151,21 +152,21 @@ func (vc *NodeVolumeAttachmentsCleanup) getFilteredPods() ([]corev1.Pod, []error } // nodeCanBeDeleted checks if all the volumeAttachments related to the node have already been collected by the external CSI driver. -func (vc *NodeVolumeAttachmentsCleanup) nodeCanBeDeleted() (bool, error) { - volumeAttachments, err := vc.kubeClient.StorageV1().VolumeAttachments().List(vc.ctx, metav1.ListOptions{}) +func (vc *NodeVolumeAttachmentsCleanup) nodeCanBeDeleted(ctx context.Context, log *zap.SugaredLogger) (bool, error) { + volumeAttachments, err := vc.kubeClient.StorageV1().VolumeAttachments().List(ctx, metav1.ListOptions{}) if err != nil { return false, fmt.Errorf("error while listing volumeAttachments: %w", err) } for _, va := range volumeAttachments.Items { if va.Spec.NodeName == vc.nodeName { - klog.V(3).Infof("waiting for the volumeAttachment %s to be deleted before deleting node %s", va.Name, vc.nodeName) + log.Infow("Waiting for VolumeAttachment to be deleted before deleting node", "volumeattachment", va.Name) return false, nil } } return true, nil } -func (vc *NodeVolumeAttachmentsCleanup) deletePods(pods []corev1.Pod) []error { +func (vc *NodeVolumeAttachmentsCleanup) deletePods(ctx context.Context, log *zap.SugaredLogger, pods []corev1.Pod) []error { errCh := make(chan error, len(pods)) retErrs := []error{} @@ -181,9 +182,9 @@ func (vc *NodeVolumeAttachmentsCleanup) deletePods(pods []corev1.Pod) []error { if isDone { return } - err := vc.kubeClient.CoreV1().Pods(p.Namespace).Delete(vc.ctx, p.Name, metav1.DeleteOptions{}) + err := vc.kubeClient.CoreV1().Pods(p.Namespace).Delete(ctx, p.Name, metav1.DeleteOptions{}) if err == nil || kerrors.IsNotFound(err) { - klog.V(6).Infof("Successfully deleted pod %s/%s on node %s", p.Namespace, p.Name, vc.nodeName) + log.Debugw("Successfully deleted pod on node", "pod", ctrlruntimeclient.ObjectKeyFromObject(&p)) return } else if kerrors.IsTooManyRequests(err) { // PDB prevents pod deletion, return and make the controller retry later. diff --git a/pkg/rhsm/satellite_subscription_manager.go b/pkg/rhsm/satellite_subscription_manager.go index 396800941..9dab43350 100644 --- a/pkg/rhsm/satellite_subscription_manager.go +++ b/pkg/rhsm/satellite_subscription_manager.go @@ -26,7 +26,7 @@ import ( "path" "time" - "k8s.io/klog" + "go.uber.org/zap" ) // SatelliteSubscriptionManager manages the communications between machine-controller and redhat satellite server. @@ -37,12 +37,13 @@ type SatelliteSubscriptionManager interface { // DefaultSatelliteSubscriptionManager default manager for redhat satellite server. type DefaultSatelliteSubscriptionManager struct { client *http.Client + log *zap.SugaredLogger useHTTP bool } // NewSatelliteSubscriptionManager creates a new Redhat satellite manager. -func NewSatelliteSubscriptionManager() SatelliteSubscriptionManager { +func NewSatelliteSubscriptionManager(log *zap.SugaredLogger) SatelliteSubscriptionManager { client := &http.Client{ Transport: &http.Transport{ TLSClientConfig: &tls.Config{ @@ -55,6 +56,7 @@ func NewSatelliteSubscriptionManager() SatelliteSubscriptionManager { return &DefaultSatelliteSubscriptionManager{ client: client, + log: log, } } @@ -68,15 +70,17 @@ func (s *DefaultSatelliteSubscriptionManager) DeleteSatelliteHost(ctx context.Co maxRetries = 15 ) + machineLog := s.log.With("machine", machineName) + for retries < maxRetries { if err := s.executeDeleteRequest(ctx, machineName, username, password, serverURL); err != nil { - klog.Errorf("failed to execute satellite subscription deletion: %v", err) + machineLog.Errorw("Failed to execute satellite subscription deletion", zap.Error(err)) retries++ time.Sleep(500 * time.Second) continue } - klog.Infof("subscription for machine %s deleted successfully", machineName) + machineLog.Info("Subscription for machine deleted successfully") return nil } @@ -110,7 +114,6 @@ func (s *DefaultSatelliteSubscriptionManager) executeDeleteRequest(ctx context.C return fmt.Errorf("error while executing request with status code: %v", response.StatusCode) } - klog.Infof("host %v has been deleted successfully", machineName) return nil } diff --git a/pkg/rhsm/satellite_subscription_manager_test.go b/pkg/rhsm/satellite_subscription_manager_test.go index 38c3f33c1..debeab623 100644 --- a/pkg/rhsm/satellite_subscription_manager_test.go +++ b/pkg/rhsm/satellite_subscription_manager_test.go @@ -23,6 +23,8 @@ import ( "net/http/httptest" "net/url" "testing" + + "go.uber.org/zap" ) func TestDefaultRedHatSatelliteManager_DeleteSatelliteHost(t *testing.T) { @@ -48,7 +50,7 @@ func TestDefaultRedHatSatelliteManager_DeleteSatelliteHost(t *testing.T) { tt.testingServer.Close() }() - manager := NewSatelliteSubscriptionManager() + manager := NewSatelliteSubscriptionManager(zap.NewNop().Sugar()) manager.(*DefaultSatelliteSubscriptionManager).useHTTP = true parsedURL, err := url.Parse(tt.testingServer.URL) diff --git a/pkg/rhsm/subscription_manager.go b/pkg/rhsm/subscription_manager.go index 9d73bdac0..40873dd3f 100644 --- a/pkg/rhsm/subscription_manager.go +++ b/pkg/rhsm/subscription_manager.go @@ -25,9 +25,8 @@ import ( "net/http" "time" + "go.uber.org/zap" "golang.org/x/oauth2" - - "k8s.io/klog" ) const defaultTimeout = 10 * time.Second @@ -54,6 +53,7 @@ type systemsResponse struct { } type defaultRedHatSubscriptionManager struct { + log *zap.SugaredLogger apiURL string authURL string requestsLimiter int @@ -61,8 +61,9 @@ type defaultRedHatSubscriptionManager struct { var errUnauthenticatedRequest = errors.New("unauthenticated") -func NewRedHatSubscriptionManager() RedHatSubscriptionManager { +func NewRedHatSubscriptionManager(log *zap.SugaredLogger) RedHatSubscriptionManager { return &defaultRedHatSubscriptionManager{ + log: log, apiURL: "https://api.access.redhat.com/management/v1/systems", authURL: "https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token", requestsLimiter: 100, @@ -102,18 +103,20 @@ func (d *defaultRedHatSubscriptionManager) UnregisterInstance(ctx context.Contex return fmt.Errorf("failed to find system profile: %w", err) } + machineLog := d.log.With("uuid", machineUUID) + if machineUUID == "" { - klog.Infof("machine uuid %s is not found", machineUUID) + machineLog.Info("Machine UUID was not found") return nil } err = d.deleteSubscription(ctx, machineUUID, offlineToken) if err == nil { - klog.Infof("subscription for vm %v has been deleted successfully", machineUUID) + machineLog.Info("Subscription for VM has been deleted successfully") return nil } - klog.Errorf("failed to delete subscription for system: %s due to: %v", machineUUID, err) + machineLog.Errorw("Failed to delete subscription for system:", zap.Error(err)) time.Sleep(2 * time.Second) retries++ } @@ -142,7 +145,6 @@ func (d *defaultRedHatSubscriptionManager) findSystemsProfile(ctx context.Contex offset += len(systemsInfo.Body) } - klog.Infof("no machine name %s is found", name) return "", nil } diff --git a/pkg/rhsm/subscription_manager_test.go b/pkg/rhsm/subscription_manager_test.go index e081401fd..088203fe3 100644 --- a/pkg/rhsm/subscription_manager_test.go +++ b/pkg/rhsm/subscription_manager_test.go @@ -22,6 +22,8 @@ import ( "net/http" "net/http/httptest" "testing" + + "go.uber.org/zap" ) var ( @@ -58,7 +60,7 @@ func TestDefaultRedHatSubscriptionManager_UnregisterInstance(t *testing.T) { defer func() { tt.testingServer.Close() }() - manager := NewRedHatSubscriptionManager() + manager := NewRedHatSubscriptionManager(zap.NewNop().Sugar()) manager.(*defaultRedHatSubscriptionManager).apiURL = tt.testingServer.URL + apiPath manager.(*defaultRedHatSubscriptionManager).authURL = tt.testingServer.URL manager.(*defaultRedHatSubscriptionManager).requestsLimiter = tt.requestLimiter diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index 67fb2f115..3b23356c5 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -27,6 +27,7 @@ import ( "text/template" "github.com/Masterminds/semver/v3" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/plugin" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -37,8 +38,8 @@ import ( type Provider struct{} // UserData renders user-data template to string. -func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { - tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(userDataTemplate) +func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) (string, error) { + tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap(log)).Parse(userDataTemplate) if err != nil { return "", fmt.Errorf("failed to parse user-data template: %w", err) } diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index 4f656d1d4..f9c93ae16 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -25,6 +25,8 @@ import ( "net" "testing" + "go.uber.org/zap" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/apis/plugin" "github.com/kubermatic/machine-controller/pkg/containerruntime" @@ -255,7 +257,7 @@ func TestUserDataGeneration(t *testing.T) { ContainerRuntime: containerRuntimeConfig, } - s, err := provider.UserData(req) + s, err := provider.UserData(zap.NewNop().Sugar(), req) if err != nil { t.Errorf("error getting userdata: '%v'", err) } diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index 353e8ed68..b4cdf976b 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -27,6 +27,7 @@ import ( "text/template" "github.com/Masterminds/semver/v3" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/plugin" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -37,8 +38,8 @@ import ( type Provider struct{} // UserData renders user-data template to string. -func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { - tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(userDataTemplate) +func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) (string, error) { + tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap(log)).Parse(userDataTemplate) if err != nil { return "", fmt.Errorf("failed to parse user-data template: %w", err) } diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index d8fa5a6c1..9dec8b334 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -25,6 +25,8 @@ import ( "net" "testing" + "go.uber.org/zap" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/apis/plugin" "github.com/kubermatic/machine-controller/pkg/containerruntime" @@ -256,7 +258,7 @@ func TestUserDataGeneration(t *testing.T) { ContainerRuntime: containerRuntimeConfig, } - s, err := provider.UserData(req) + s, err := provider.UserData(zap.NewNop().Sugar(), req) if err != nil { t.Errorf("error getting userdata: '%v'", err) } diff --git a/pkg/userdata/convert/ignition-converter.go b/pkg/userdata/convert/ignition-converter.go index bb9d4c865..6eddf25ca 100644 --- a/pkg/userdata/convert/ignition-converter.go +++ b/pkg/userdata/convert/ignition-converter.go @@ -21,6 +21,7 @@ import ( "fmt" ctconfig "github.com/coreos/container-linux-config-transpiler/config" + "go.uber.org/zap" pluginapi "github.com/kubermatic/machine-controller/pkg/apis/plugin" "github.com/kubermatic/machine-controller/pkg/userdata/plugin" @@ -34,8 +35,8 @@ type Ignition struct { p plugin.Provider } -func (j *Ignition) UserData(req pluginapi.UserDataRequest) (string, error) { - before, err := j.p.UserData(req) +func (j *Ignition) UserData(log *zap.SugaredLogger, req pluginapi.UserDataRequest) (string, error) { + before, err := j.p.UserData(log, req) if err != nil { return "", err } diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index ec4752723..3c50b1c4c 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -26,6 +26,7 @@ import ( "text/template" "github.com/Masterminds/semver/v3" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/plugin" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -37,7 +38,7 @@ import ( type Provider struct{} // UserData renders user-data template to string. -func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { +func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) (string, error) { pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) if err != nil { return "", fmt.Errorf("failed to get provider config: %w", err) @@ -57,7 +58,7 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { return "", fmt.Errorf("failed to get an appropriate user-data template: %w", err) } - tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(userDataTemplate) + tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap(log)).Parse(userDataTemplate) if err != nil { return "", fmt.Errorf("failed to parse user-data template: %w", err) } diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index 3337fb222..c12fe1aa4 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -26,6 +26,8 @@ import ( "net" "testing" + "go.uber.org/zap" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/apis/plugin" "github.com/kubermatic/machine-controller/pkg/containerruntime" @@ -415,7 +417,7 @@ func TestUserDataGeneration(t *testing.T) { ContainerRuntime: containerRuntimeConfig, } - s, err := provider.UserData(req) + s, err := provider.UserData(zap.NewNop().Sugar(), req) if err != nil { t.Fatal(err) } diff --git a/pkg/userdata/helper/download_binaries_script.go b/pkg/userdata/helper/download_binaries_script.go index ab9e248c5..6a10bfaa2 100644 --- a/pkg/userdata/helper/download_binaries_script.go +++ b/pkg/userdata/helper/download_binaries_script.go @@ -21,6 +21,8 @@ import ( "fmt" "strings" "text/template" + + "go.uber.org/zap" ) const ( @@ -158,8 +160,8 @@ fi // SafeDownloadBinariesScript returns the script which is responsible to // download and check checksums of all required binaries. -func SafeDownloadBinariesScript(kubeVersion string) (string, error) { - tmpl, err := template.New("download-binaries").Funcs(TxtFuncMap()).Parse(safeDownloadBinariesTpl) +func SafeDownloadBinariesScript(log *zap.SugaredLogger, kubeVersion string) (string, error) { + tmpl, err := template.New("download-binaries").Funcs(TxtFuncMap(log)).Parse(safeDownloadBinariesTpl) if err != nil { return "", fmt.Errorf("failed to parse download-binaries template: %w", err) } @@ -195,8 +197,8 @@ func SafeDownloadBinariesScript(kubeVersion string) (string, error) { // DownloadBinariesScript returns the script which is responsible to download // all required binaries. -func DownloadBinariesScript(kubeletVersion string, downloadKubelet bool) (string, error) { - tmpl, err := template.New("download-binaries").Funcs(TxtFuncMap()).Parse(downloadBinariesTpl) +func DownloadBinariesScript(log *zap.SugaredLogger, kubeletVersion string, downloadKubelet bool) (string, error) { + tmpl, err := template.New("download-binaries").Funcs(TxtFuncMap(log)).Parse(downloadBinariesTpl) if err != nil { return "", fmt.Errorf("failed to parse download-binaries template: %w", err) } diff --git a/pkg/userdata/helper/download_binaries_script_test.go b/pkg/userdata/helper/download_binaries_script_test.go index 9ecb063fa..8689cc4fd 100644 --- a/pkg/userdata/helper/download_binaries_script_test.go +++ b/pkg/userdata/helper/download_binaries_script_test.go @@ -20,6 +20,8 @@ import ( "fmt" "testing" + "go.uber.org/zap" + "github.com/kubermatic/machine-controller/pkg/test" ) @@ -27,7 +29,7 @@ func TestDownloadBinariesScript(t *testing.T) { for _, version := range versions { name := fmt.Sprintf("download_binaries_%s", version.Original()) t.Run(name, func(t *testing.T) { - script, err := DownloadBinariesScript(version.String(), true) + script, err := DownloadBinariesScript(zap.NewNop().Sugar(), version.String(), true) if err != nil { t.Error(err) } @@ -40,7 +42,7 @@ func TestDownloadBinariesScript(t *testing.T) { func TestSafeDownloadBinariesScript(t *testing.T) { name := "safe_download_binaries_v1.24.9" t.Run(name, func(t *testing.T) { - script, err := SafeDownloadBinariesScript("v1.24.9") + script, err := SafeDownloadBinariesScript(zap.NewNop().Sugar(), "v1.24.9") if err != nil { t.Error(err) } diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index 7d278637a..9a64e81dc 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -23,12 +23,13 @@ import ( "strings" "text/template" + "go.uber.org/zap" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/klog" kubeletv1b1 "k8s.io/kubelet/config/v1beta1" "k8s.io/utils/pointer" kyaml "sigs.k8s.io/yaml" @@ -158,8 +159,8 @@ func CloudProviderFlags(cpName string, external bool) string { } // KubeletSystemdUnit returns the systemd unit for the kubelet. -func KubeletSystemdUnit(containerRuntime, kubeletVersion, cloudProvider, hostname string, dnsIPs []net.IP, external bool, ipFamily util.IPFamily, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string, disableSwap bool) (string, error) { - tmpl, err := template.New("kubelet-systemd-unit").Funcs(TxtFuncMap()).Parse(kubeletSystemdUnitTpl) +func KubeletSystemdUnit(log *zap.SugaredLogger, containerRuntime, kubeletVersion, cloudProvider, hostname string, dnsIPs []net.IP, external bool, ipFamily util.IPFamily, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string, disableSwap bool) (string, error) { + tmpl, err := template.New("kubelet-systemd-unit").Funcs(TxtFuncMap(log)).Parse(kubeletSystemdUnitTpl) if err != nil { return "", fmt.Errorf("failed to parse kubelet-systemd-unit template: %w", err) } @@ -199,7 +200,7 @@ func KubeletSystemdUnit(containerRuntime, kubeletVersion, cloudProvider, hostnam } // kubeletConfiguration returns marshaled kubelet.config.k8s.io/v1beta1 KubeletConfiguration. -func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGates map[string]bool, kubeletConfigs map[string]string, containerRuntime string) (string, error) { +func kubeletConfiguration(log *zap.SugaredLogger, clusterDomain string, clusterDNS []net.IP, featureGates map[string]bool, kubeletConfigs map[string]string, containerRuntime string) (string, error) { clusterDNSstr := make([]string, 0, len(clusterDNS)) for _, ip := range clusterDNS { clusterDNSstr = append(clusterDNSstr, ip.String()) @@ -275,7 +276,7 @@ func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGate mp, err := strconv.ParseInt(maxPods, 10, 32) if err != nil { // Instead of breaking the workflow, just print a warning and skip the configuration - klog.Warningf("Skipping invalid MaxPods value %v for Kubelet configuration", maxPods) + log.Info("Skipping invalid MaxPods value for Kubelet configuration", "value", maxPods) } else { cfg.MaxPods = int32(mp) } @@ -288,7 +289,7 @@ func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGate maxFiles, err := strconv.Atoi(containerLogMaxFiles) if err != nil || maxFiles < 0 { // Instead of breaking the workflow, just print a warning and skip the configuration - klog.Warningf("Skipping invalid ContainerLogMaxSize value %v for Kubelet configuration", containerLogMaxFiles) + log.Infow("Skipping invalid ContainerLogMaxSize value for Kubelet configuration", "value", containerLogMaxFiles) } else { cfg.ContainerLogMaxFiles = pointer.Int32(int32(maxFiles)) } @@ -308,10 +309,10 @@ func kubeletConfiguration(clusterDomain string, clusterDNS []net.IP, featureGate // as the cloud provider is expected to know the correct IPs to return. // For details read kubernetes/sig-networking channel discussion // https://kubernetes.slack.com/archives/C09QYUH5W/p1654003958331739 -func KubeletFlags(version, cloudProvider, hostname string, dnsIPs []net.IP, external bool, ipFamily util.IPFamily, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string) (string, error) { +func KubeletFlags(log *zap.SugaredLogger, version, cloudProvider, hostname string, dnsIPs []net.IP, external bool, ipFamily util.IPFamily, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string) (string, error) { withNodeIPFlag := withNodeIPFlag(ipFamily, cloudProvider, external) - tmpl, err := template.New("kubelet-flags").Funcs(TxtFuncMap()). + tmpl, err := template.New("kubelet-flags").Funcs(TxtFuncMap(log)). Parse(kubeletFlagsTpl(withNodeIPFlag)) if err != nil { return "", fmt.Errorf("failed to parse kubelet-flags template: %w", err) @@ -370,8 +371,8 @@ WantedBy=multi-user.target } // ContainerRuntimeHealthCheckSystemdUnit container-runtime health checking systemd unit. -func ContainerRuntimeHealthCheckSystemdUnit(containerRuntime string) (string, error) { - tmpl, err := template.New("container-runtime-healthcheck-systemd-unit").Funcs(TxtFuncMap()).Parse(containerRuntimeHealthCheckSystemdUnitTpl) +func ContainerRuntimeHealthCheckSystemdUnit(log *zap.SugaredLogger, containerRuntime string) (string, error) { + tmpl, err := template.New("container-runtime-healthcheck-systemd-unit").Funcs(TxtFuncMap(log)).Parse(containerRuntimeHealthCheckSystemdUnitTpl) if err != nil { return "", fmt.Errorf("failed to parse container-runtime-healthcheck-systemd-unit template: %w", err) } diff --git a/pkg/userdata/helper/kubelet_test.go b/pkg/userdata/helper/kubelet_test.go index f706bc9c6..8bfb522fa 100644 --- a/pkg/userdata/helper/kubelet_test.go +++ b/pkg/userdata/helper/kubelet_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/Masterminds/semver/v3" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" testhelper "github.com/kubermatic/machine-controller/pkg/test" @@ -113,6 +114,7 @@ func TestKubeletSystemdUnit(t *testing.T) { name := fmt.Sprintf("kublet_systemd_unit_%s", test.name) t.Run(name, func(t *testing.T) { out, err := KubeletSystemdUnit( + zap.NewNop().Sugar(), defaultTo(test.containerRuntime, "docker"), test.version.String(), test.cloudProvider, diff --git a/pkg/userdata/helper/template_functions.go b/pkg/userdata/helper/template_functions.go index faeb1ea00..2b7fe2ee9 100644 --- a/pkg/userdata/helper/template_functions.go +++ b/pkg/userdata/helper/template_functions.go @@ -17,27 +17,53 @@ limitations under the License. package helper import ( + "net" "regexp" "text/template" "github.com/Masterminds/sprig/v3" + "go.uber.org/zap" + + "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" + + corev1 "k8s.io/api/core/v1" ) // TxtFuncMap returns an aggregated template function map. Currently (custom functions + sprig). -func TxtFuncMap() template.FuncMap { +func TxtFuncMap(log *zap.SugaredLogger) template.FuncMap { funcMap := sprig.TxtFuncMap() - funcMap["downloadBinariesScript"] = DownloadBinariesScript - funcMap["safeDownloadBinariesScript"] = SafeDownloadBinariesScript - funcMap["kubeletSystemdUnit"] = KubeletSystemdUnit - funcMap["kubeletConfiguration"] = kubeletConfiguration - funcMap["kubeletFlags"] = KubeletFlags + // use inline wrappers to inject the logger without forcing the templates to keep track of it + + funcMap["downloadBinariesScript"] = func(kubeletVersion string, downloadKubelet bool) (string, error) { + return DownloadBinariesScript(log, kubeletVersion, downloadKubelet) + } + + funcMap["safeDownloadBinariesScript"] = func(kubeVersion string) (string, error) { + return SafeDownloadBinariesScript(log, kubeVersion) + } + + funcMap["kubeletSystemdUnit"] = func(containerRuntime, kubeletVersion, cloudProvider, hostname string, dnsIPs []net.IP, external bool, ipFamily util.IPFamily, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string, disableSwap bool) (string, error) { + return KubeletSystemdUnit(log, containerRuntime, kubeletVersion, cloudProvider, hostname, dnsIPs, external, ipFamily, pauseImage, initialTaints, extraKubeletFlags, disableSwap) + } + + funcMap["kubeletConfiguration"] = func(clusterDomain string, clusterDNS []net.IP, featureGates map[string]bool, kubeletConfigs map[string]string, containerRuntime string) (string, error) { + return kubeletConfiguration(log, clusterDomain, clusterDNS, featureGates, kubeletConfigs, containerRuntime) + } + + funcMap["kubeletFlags"] = func(version, cloudProvider, hostname string, dnsIPs []net.IP, external bool, ipFamily util.IPFamily, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string) (string, error) { + return KubeletFlags(log, version, cloudProvider, hostname, dnsIPs, external, ipFamily, pauseImage, initialTaints, extraKubeletFlags) + } + + funcMap["containerRuntimeHealthCheckSystemdUnit"] = func(containerRuntime string) (string, error) { + return ContainerRuntimeHealthCheckSystemdUnit(log, containerRuntime) + } + funcMap["cloudProviderFlags"] = CloudProviderFlags funcMap["kernelModulesScript"] = LoadKernelModulesScript funcMap["kernelSettings"] = KernelSettings funcMap["journalDConfig"] = JournalDConfig funcMap["kubeletHealthCheckSystemdUnit"] = KubeletHealthCheckSystemdUnit - funcMap["containerRuntimeHealthCheckSystemdUnit"] = ContainerRuntimeHealthCheckSystemdUnit funcMap["dockerConfig"] = DockerConfig funcMap["proxyEnvironment"] = ProxyEnvironment funcMap["sshConfigAddendum"] = SSHConfigAddendum diff --git a/pkg/userdata/manager/manager.go b/pkg/userdata/manager/manager.go index 473899333..e53e0cfbf 100644 --- a/pkg/userdata/manager/manager.go +++ b/pkg/userdata/manager/manager.go @@ -26,9 +26,9 @@ import ( "errors" "flag" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "go.uber.org/zap" - "k8s.io/klog" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) var ( @@ -56,12 +56,14 @@ var ( // Manager inits and manages the userdata plugins. type Manager struct { debug bool + log *zap.SugaredLogger plugins map[providerconfigtypes.OperatingSystem]*Plugin } // New returns an initialised plugin manager. -func New() (*Manager, error) { +func New(log *zap.SugaredLogger) (*Manager, error) { m := &Manager{ + log: log, plugins: make(map[providerconfigtypes.OperatingSystem]*Plugin), } flag.BoolVar(&m.debug, "plugin-debug", false, "Switch for enabling the plugin debugging") @@ -84,9 +86,11 @@ func (m *Manager) ForOS(os providerconfigtypes.OperatingSystem) (p *Plugin, err // locatePlugins tries to find the plugins and inits their wrapper. func (m *Manager) locatePlugins() { for _, os := range supportedOS { - plugin, err := newPlugin(os, m.debug) + osLog := m.log.With("os", os) + + plugin, err := newPlugin(osLog, os, m.debug) if err != nil { - klog.Errorf("cannot use plugin '%v': %v", os, err) + osLog.Errorw("Cannot use plugin", zap.Error(err)) continue } m.plugins[os] = plugin diff --git a/pkg/userdata/manager/plugin.go b/pkg/userdata/manager/plugin.go index 69fdc7199..6f7014fd6 100644 --- a/pkg/userdata/manager/plugin.go +++ b/pkg/userdata/manager/plugin.go @@ -28,10 +28,10 @@ import ( "path/filepath" "strings" + "go.uber.org/zap" + "github.com/kubermatic/machine-controller/pkg/apis/plugin" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - - "k8s.io/klog" ) const ( @@ -43,14 +43,16 @@ const ( // each request. type Plugin struct { debug bool + log *zap.SugaredLogger command string } // newPlugin creates a new plugin manager. It starts the named // binary and connects to it via net/rpc. -func newPlugin(os providerconfigtypes.OperatingSystem, debug bool) (*Plugin, error) { +func newPlugin(log *zap.SugaredLogger, os providerconfigtypes.OperatingSystem, debug bool) (*Plugin, error) { p := &Plugin{ debug: debug, + log: log, } if err := p.findPlugin(string(os)); err != nil { return nil, err @@ -60,7 +62,7 @@ func newPlugin(os providerconfigtypes.OperatingSystem, debug bool) (*Plugin, err // UserData retrieves the user data of the given resource via // plugin handling the communication. -func (p *Plugin) UserData(req plugin.UserDataRequest) (string, error) { +func (p *Plugin) UserData(_ *zap.SugaredLogger, req plugin.UserDataRequest) (string, error) { // Prepare command. var argv []string if p.debug { @@ -92,7 +94,8 @@ func (p *Plugin) UserData(req plugin.UserDataRequest) (string, error) { // findPlugin tries to find the executable of the plugin. func (p *Plugin) findPlugin(name string) error { filename := pluginPrefix + name - klog.Infof("looking for plugin %q", filename) + pluginLog := p.log.With("plugin", filename) + pluginLog.Infow("Looking for plugin") // Create list to search in. var dirs []string envDir := os.Getenv(plugin.EnvPluginDir) @@ -120,7 +123,7 @@ func (p *Plugin) findPlugin(name string) error { // Now take a look. for _, dir := range dirs { command := dir + string(os.PathSeparator) + filename - klog.V(3).Infof("checking %q", command) + pluginLog.Debugw("Checking directory", "directory", dir) fi, err := os.Stat(command) if err != nil { if os.IsNotExist(err) { @@ -129,13 +132,13 @@ func (p *Plugin) findPlugin(name string) error { return fmt.Errorf("error when looking for %q: %w", command, err) } if fi.IsDir() || (fi.Mode()&0111 == 0) { - klog.Infof("found '%s', but is no executable", command) + pluginLog.Infow("Found file, but is no executable", "filename", command) continue } p.command = command - klog.Infof("found '%s'", command) + p.log.Infow("Found plugin", "filename", command) return nil } - klog.Errorf("did not find '%s'", filename) + pluginLog.Error("Did not find plugin") return ErrPluginNotFound } diff --git a/pkg/userdata/plugin/plugin.go b/pkg/userdata/plugin/plugin.go index 4ceb3a581..b7471f10d 100644 --- a/pkg/userdata/plugin/plugin.go +++ b/pkg/userdata/plugin/plugin.go @@ -28,13 +28,15 @@ import ( "fmt" "os" + "go.uber.org/zap" + "github.com/kubermatic/machine-controller/pkg/apis/plugin" ) // Provider defines the interface each plugin has to implement // for the retrieval of the userdata based on the given arguments. type Provider interface { - UserData(req plugin.UserDataRequest) (string, error) + UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) (string, error) } // Plugin implements a convenient helper to map the request to the given @@ -53,7 +55,7 @@ func New(provider Provider, debug bool) *Plugin { } // Run looks for the given request and executes it. -func (p *Plugin) Run() error { +func (p *Plugin) Run(log *zap.SugaredLogger) error { reqEnv := os.Getenv(plugin.EnvUserDataRequest) if reqEnv == "" { resp := plugin.ErrorResponse{ @@ -67,7 +69,7 @@ func (p *Plugin) Run() error { if err != nil { return err } - userData, err := p.provider.UserData(req) + userData, err := p.provider.UserData(log, req) var resp plugin.UserDataResponse if err != nil { resp.Err = err.Error() diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index 5eee08984..df5a90b5c 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -27,6 +27,7 @@ import ( "text/template" "github.com/Masterminds/semver/v3" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/plugin" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -37,8 +38,8 @@ import ( type Provider struct{} // UserData renders user-data template to string. -func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { - tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(userDataTemplate) +func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) (string, error) { + tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap(log)).Parse(userDataTemplate) if err != nil { return "", fmt.Errorf("failed to parse user-data template: %w", err) } @@ -237,7 +238,7 @@ write_files: grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - + # Restart NetworkManager to apply for IPv6 configs systemctl restart NetworkManager # Let NetworkManager apply the DHCPv6 configs diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index ee0434e49..36101f8e7 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -25,6 +25,8 @@ import ( "net" "testing" + "go.uber.org/zap" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/apis/plugin" "github.com/kubermatic/machine-controller/pkg/containerruntime" @@ -275,7 +277,7 @@ func TestUserDataGeneration(t *testing.T) { KubeletFeatureGates: kubeletFeatureGates, ContainerRuntime: containerRuntimeConfig, } - s, err := provider.UserData(req) + s, err := provider.UserData(zap.NewNop().Sugar(), req) if err != nil { t.Errorf("error getting userdata: '%v'", err) } diff --git a/pkg/userdata/rockylinux/provider.go b/pkg/userdata/rockylinux/provider.go index c19a5dcd1..f572e70a7 100644 --- a/pkg/userdata/rockylinux/provider.go +++ b/pkg/userdata/rockylinux/provider.go @@ -27,6 +27,7 @@ import ( "text/template" "github.com/Masterminds/semver/v3" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/plugin" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -37,8 +38,8 @@ import ( type Provider struct{} // UserData renders user-data template to string. -func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { - tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(userDataTemplate) +func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) (string, error) { + tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap(log)).Parse(userDataTemplate) if err != nil { return "", fmt.Errorf("failed to parse user-data template: %w", err) } diff --git a/pkg/userdata/rockylinux/provider_test.go b/pkg/userdata/rockylinux/provider_test.go index aacab4e07..64b1ad95b 100644 --- a/pkg/userdata/rockylinux/provider_test.go +++ b/pkg/userdata/rockylinux/provider_test.go @@ -25,6 +25,8 @@ import ( "net" "testing" + "go.uber.org/zap" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/apis/plugin" "github.com/kubermatic/machine-controller/pkg/containerruntime" @@ -256,7 +258,7 @@ func TestUserDataGeneration(t *testing.T) { ContainerRuntime: containerRuntimeConfig, } - s, err := provider.UserData(req) + s, err := provider.UserData(zap.NewNop().Sugar(), req) if err != nil { t.Errorf("error getting userdata: '%v'", err) } diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index 2dfb8d3a7..4f154f2d7 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -27,6 +27,7 @@ import ( "text/template" "github.com/Masterminds/semver/v3" + "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/plugin" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -37,8 +38,8 @@ import ( type Provider struct{} // UserData renders user-data template to string. -func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { - tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(userDataTemplate) +func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) (string, error) { + tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap(log)).Parse(userDataTemplate) if err != nil { return "", fmt.Errorf("failed to parse user-data template: %w", err) } diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index d30f210bb..f3f722a50 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -28,6 +28,7 @@ import ( "testing" "github.com/Masterminds/semver/v3" + "go.uber.org/zap" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/apis/plugin" @@ -670,7 +671,7 @@ func TestUserDataGeneration(t *testing.T) { KubeletFeatureGates: kubeletFeatureGates, ContainerRuntime: containerRuntimeConfig, } - s, err := provider.UserData(req) + s, err := provider.UserData(zap.NewNop().Sugar(), req) if err != nil { t.Fatal(err) } diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index fb021f1db..ba16c682d 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -47,7 +47,7 @@ import ( func init() { klog.InitFlags(nil) if err := clusterv1alpha1.SchemeBuilder.AddToScheme(scheme.Scheme); err != nil { - klog.Fatalf("failed to add clusterv1alpha1 to scheme: %v", err) + klog.Fatalf("Failed to add clusterv1alpha1 to scheme: %v", err) } } @@ -122,7 +122,7 @@ func TestCustomCAsAreApplied(t *testing.T) { osNetwork := os.Getenv("OS_NETWORK_NAME") if osAuthURL == "" || osUsername == "" || osPassword == "" || osDomain == "" || osRegion == "" || osTenant == "" { - t.Fatal("unable to run test suite, all of OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_REGION, and OS_TENANT OS_DOMAIN must be set!") + t.Fatal("Unable to run test suite, all of OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_REGION, and OS_TENANT OS_DOMAIN must be set!") } params := []string{ @@ -292,7 +292,7 @@ func TestKubevirtProvisioningE2E(t *testing.T) { kubevirtKubeconfig := os.Getenv("KUBEVIRT_E2E_TESTS_KUBECONFIG") if kubevirtKubeconfig == "" { - t.Fatalf("Unable to run kubevirt tests, KUBEVIRT_E2E_TESTS_KUBECONFIG must be set") + t.Fatal("Unable to run kubevirt tests, KUBEVIRT_E2E_TESTS_KUBECONFIG must be set") } selector := OsSelector("ubuntu", "centos", "flatcar", "rockylinux") @@ -327,7 +327,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { osNetwork := os.Getenv("OS_NETWORK_NAME") if osAuthURL == "" || osUsername == "" || osPassword == "" || osDomain == "" || osRegion == "" || osTenant == "" { - t.Fatal("unable to run test suite, all of OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_REGION, and OS_TENANT OS_DOMAIN must be set!") + t.Fatal("Unable to run test suite, all of OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_REGION, and OS_TENANT OS_DOMAIN must be set!") } params := []string{ @@ -358,7 +358,7 @@ func TestOpenstackProjectAuthProvisioningE2E(t *testing.T) { osNetwork := os.Getenv("OS_NETWORK_NAME") if osAuthURL == "" || osUsername == "" || osPassword == "" || osDomain == "" || osRegion == "" || osProject == "" { - t.Fatal("unable to run test suite, all of OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_REGION, and OS_TENANT OS_DOMAIN must be set!") + t.Fatal("Unable to run test suite, all of OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_REGION, and OS_TENANT OS_DOMAIN must be set!") } params := []string{ @@ -391,7 +391,7 @@ func TestDigitalOceanProvisioningE2E(t *testing.T) { // test data doToken := os.Getenv("DO_E2E_TESTS_TOKEN") if len(doToken) == 0 { - t.Fatal("unable to run the test suite, DO_E2E_TESTS_TOKEN environment variable cannot be empty") + t.Fatal("Unable to run the test suite, DO_E2E_TESTS_TOKEN environment variable cannot be empty") } selector := OsSelector("ubuntu", "centos", "rockylinux") @@ -416,7 +416,7 @@ func TestAWSProvisioningE2E(t *testing.T) { awsKeyID := os.Getenv("AWS_E2E_TESTS_KEY_ID") awsSecret := os.Getenv("AWS_E2E_TESTS_SECRET") if len(awsKeyID) == 0 || len(awsSecret) == 0 { - t.Fatal("unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") + t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } selector := Not(OsSelector("sles")) @@ -441,7 +441,7 @@ func TestAWSAssumeRoleProvisioningE2E(t *testing.T) { awsAssumeRoleARN := os.Getenv("AWS_ASSUME_ROLE_ARN") awsAssumeRoleExternalID := os.Getenv("AWS_ASSUME_ROLE_EXTERNAL_ID") if len(awsKeyID) == 0 || len(awsSecret) == 0 || len(awsAssumeRoleARN) == 0 || len(awsAssumeRoleExternalID) == 0 { - t.Fatal("unable to run the test suite, environment variables AWS_E2E_TESTS_KEY_ID, AWS_E2E_TESTS_SECRET, AWS_E2E_ASSUME_ROLE_ARN and AWS_E2E_ASSUME_ROLE_EXTERNAL_ID cannot be empty") + t.Fatal("Unable to run the test suite, environment variables AWS_E2E_TESTS_KEY_ID, AWS_E2E_TESTS_SECRET, AWS_E2E_ASSUME_ROLE_ARN and AWS_E2E_ASSUME_ROLE_EXTERNAL_ID cannot be empty") } // act @@ -469,7 +469,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { awsKeyID := os.Getenv("AWS_E2E_TESTS_KEY_ID") awsSecret := os.Getenv("AWS_E2E_TESTS_SECRET") if len(awsKeyID) == 0 || len(awsSecret) == 0 { - t.Fatal("unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") + t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. selector := OsSelector("ubuntu") @@ -490,7 +490,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { awsKeyID := os.Getenv("AWS_E2E_TESTS_KEY_ID") awsSecret := os.Getenv("AWS_E2E_TESTS_SECRET") if len(awsKeyID) == 0 || len(awsSecret) == 0 { - t.Fatal("unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") + t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } selector := OsSelector("ubuntu") // act @@ -508,7 +508,7 @@ func TestAWSFlatcarCoreOSCloudInit8ProvisioningE2E(t *testing.T) { awsKeyID := os.Getenv("AWS_E2E_TESTS_KEY_ID") awsSecret := os.Getenv("AWS_E2E_TESTS_SECRET") if len(awsKeyID) == 0 || len(awsSecret) == 0 { - t.Fatal("unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") + t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } params := []string{ @@ -529,7 +529,7 @@ func TestAWSFlatcarContainerdProvisioningE2E(t *testing.T) { awsKeyID := os.Getenv("AWS_E2E_TESTS_KEY_ID") awsSecret := os.Getenv("AWS_E2E_TESTS_SECRET") if len(awsKeyID) == 0 || len(awsSecret) == 0 { - t.Fatal("unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") + t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } params := []string{ @@ -555,7 +555,7 @@ func TestAWSCentOS8ProvisioningE2E(t *testing.T) { awsKeyID := os.Getenv("AWS_E2E_TESTS_KEY_ID") awsSecret := os.Getenv("AWS_E2E_TESTS_SECRET") if len(awsKeyID) == 0 || len(awsSecret) == 0 { - t.Fatal("unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") + t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } amiID := "ami-032025b3afcbb6b34" // official "CentOS 8.2.2004 x86_64" @@ -580,7 +580,7 @@ func TestAWSEbsEncryptionEnabledProvisioningE2E(t *testing.T) { awsKeyID := os.Getenv("AWS_E2E_TESTS_KEY_ID") awsSecret := os.Getenv("AWS_E2E_TESTS_SECRET") if len(awsKeyID) == 0 || len(awsSecret) == 0 { - t.Fatal("unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") + t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // act @@ -609,7 +609,7 @@ func TestAzureProvisioningE2E(t *testing.T) { azureClientID := os.Getenv("AZURE_E2E_TESTS_CLIENT_ID") azureClientSecret := os.Getenv("AZURE_E2E_TESTS_CLIENT_SECRET") if len(azureTenantID) == 0 || len(azureSubscriptionID) == 0 || len(azureClientID) == 0 || len(azureClientSecret) == 0 { - t.Fatal("unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") + t.Fatal("Unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") } selector := Not(OsSelector("amzn2")) @@ -637,7 +637,7 @@ func TestAzureCustomImageReferenceProvisioningE2E(t *testing.T) { azureClientID := os.Getenv("AZURE_E2E_TESTS_CLIENT_ID") azureClientSecret := os.Getenv("AZURE_E2E_TESTS_CLIENT_SECRET") if len(azureTenantID) == 0 || len(azureSubscriptionID) == 0 || len(azureClientID) == 0 || len(azureClientSecret) == 0 { - t.Fatal("unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") + t.Fatal("Unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") } selector := OsSelector("ubuntu") @@ -665,7 +665,7 @@ func TestAzureRedhatSatelliteProvisioningE2E(t *testing.T) { azureClientID := os.Getenv("AZURE_E2E_TESTS_CLIENT_ID") azureClientSecret := os.Getenv("AZURE_E2E_TESTS_CLIENT_SECRET") if len(azureTenantID) == 0 || len(azureSubscriptionID) == 0 || len(azureClientID) == 0 || len(azureClientSecret) == 0 { - t.Fatal("unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") + t.Fatal("Unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") } // act @@ -698,7 +698,7 @@ func TestGCEProvisioningE2E(t *testing.T) { // Test data. googleServiceAccount := os.Getenv("GOOGLE_SERVICE_ACCOUNT") if len(googleServiceAccount) == 0 { - t.Fatal("unable to run the test suite, GOOGLE_SERVICE_ACCOUNT environment variable cannot be empty") + t.Fatal("Unable to run the test suite, GOOGLE_SERVICE_ACCOUNT environment variable cannot be empty") } // Act. GCE does not support CentOS. @@ -718,7 +718,7 @@ func TestHetznerProvisioningE2E(t *testing.T) { // test data hzToken := os.Getenv("HZ_E2E_TOKEN") if len(hzToken) == 0 { - t.Fatal("unable to run the test suite, HZ_E2E_TOKEN environment variable cannot be empty") + t.Fatal("Unable to run the test suite, HZ_E2E_TOKEN environment variable cannot be empty") } selector := OsSelector("ubuntu", "centos", "rockylinux") @@ -736,12 +736,12 @@ func TestEquinixMetalProvisioningE2E(t *testing.T) { // test data token := os.Getenv("METAL_AUTH_TOKEN") if len(token) == 0 { - t.Fatal("unable to run the test suite, METAL_AUTH_TOKEN environment variable cannot be empty") + t.Fatal("Unable to run the test suite, METAL_AUTH_TOKEN environment variable cannot be empty") } projectID := os.Getenv("METAL_PROJECT_ID") if len(projectID) == 0 { - t.Fatal("unable to run the test suite, METAL_PROJECT_ID environment variable cannot be empty") + t.Fatal("Unable to run the test suite, METAL_PROJECT_ID environment variable cannot be empty") } selector := And(OsSelector("ubuntu", "centos", "rockylinux", "flatcar"), Not(NameSelector("migrateUID"))) @@ -760,12 +760,12 @@ func TestAlibabaProvisioningE2E(t *testing.T) { // test data accessKeyID := os.Getenv("ALIBABA_ACCESS_KEY_ID") if len(accessKeyID) == 0 { - t.Fatal("unable to run the test suite, ALIBABA_ACCESS_KEY_ID environment variable cannot be empty") + t.Fatal("Unable to run the test suite, ALIBABA_ACCESS_KEY_ID environment variable cannot be empty") } accessKeySecret := os.Getenv("ALIBABA_ACCESS_KEY_SECRET") if len(accessKeySecret) == 0 { - t.Fatal("unable to run the test suite, ALIBABA_ACCESS_KEY_SECRET environment variable cannot be empty") + t.Fatal("Unable to run the test suite, ALIBABA_ACCESS_KEY_SECRET environment variable cannot be empty") } selector := OsSelector("ubuntu") @@ -788,7 +788,7 @@ func TestLinodeProvisioningE2E(t *testing.T) { // test data linodeToken := os.Getenv("LINODE_E2E_TESTS_TOKEN") if len(linodeToken) == 0 { - t.Fatal("unable to run the test suite, LINODE_E2E_TESTS_TOKEN environment variable cannot be empty") + t.Fatal("Unable to run the test suite, LINODE_E2E_TESTS_TOKEN environment variable cannot be empty") } // we're shimming userdata through Linode stackscripts and the stackscript hasn't been verified for use with centos @@ -808,7 +808,7 @@ func getVMwareCloudDirectorTestParams(t *testing.T) []string { vdc := os.Getenv("VCD_VDC") if password == "" || username == "" || organization == "" || url == "" || vdc == "" { - t.Fatal("unable to run the test suite, VCD_PASSWORD, VCD_USER, VCD_ORG, " + + t.Fatal("Unable to run the test suite, VCD_PASSWORD, VCD_USER, VCD_ORG, " + "VCD_URL, or VCD_VDC environment variables cannot be empty") } @@ -838,7 +838,7 @@ func getVSphereTestParams(t *testing.T) []string { vsAddress := os.Getenv("VSPHERE_E2E_ADDRESS") if vsPassword == "" || vsUsername == "" || vsAddress == "" { - t.Fatal("unable to run the test suite, VSPHERE_E2E_PASSWORD, VSPHERE_E2E_USERNAME" + + t.Fatal("Unable to run the test suite, VSPHERE_E2E_PASSWORD, VSPHERE_E2E_USERNAME" + "or VSPHERE_E2E_ADDRESS environment variables cannot be empty") } @@ -903,17 +903,17 @@ func TestScalewayProvisioningE2E(t *testing.T) { // test data scwAccessKey := os.Getenv("SCW_ACCESS_KEY") if len(scwAccessKey) == 0 { - t.Fatal("unable to run the test suite, SCW_E2E_TEST_ACCESS_KEY environment variable cannot be empty") + t.Fatal("Unable to run the test suite, SCW_E2E_TEST_ACCESS_KEY environment variable cannot be empty") } scwSecretKey := os.Getenv("SCW_SECRET_KEY") if len(scwSecretKey) == 0 { - t.Fatal("unable to run the test suite, SCW_E2E_TEST_SECRET_KEY environment variable cannot be empty") + t.Fatal("Unable to run the test suite, SCW_E2E_TEST_SECRET_KEY environment variable cannot be empty") } scwProjectID := os.Getenv("SCW_DEFAULT_PROJECT_ID") if len(scwProjectID) == 0 { - t.Fatal("unable to run the test suite, SCW_E2E_TEST_PROJECT_ID environment variable cannot be empty") + t.Fatal("Unable to run the test suite, SCW_E2E_TEST_PROJECT_ID environment variable cannot be empty") } selector := Not(OsSelector("rhel", "flatcar", "rockylinux")) @@ -937,7 +937,7 @@ func getNutanixTestParams(t *testing.T) []string { endpoint := os.Getenv("NUTANIX_E2E_ENDPOINT") if password == "" || username == "" || endpoint == "" || cluster == "" || project == "" || subnet == "" { - t.Fatal("unable to run the test suite, NUTANIX_E2E_PASSWORD, NUTANIX_E2E_USERNAME, NUTANIX_E2E_CLUSTER_NAME, " + + t.Fatal("Unable to run the test suite, NUTANIX_E2E_PASSWORD, NUTANIX_E2E_USERNAME, NUTANIX_E2E_CLUSTER_NAME, " + "NUTANIX_E2E_ENDPOINT, NUTANIX_E2E_PROJECT_NAME or NUTANIX_E2E_SUBNET_NAME environment variables cannot be empty") } @@ -978,7 +978,7 @@ func TestUbuntuProvisioningWithUpgradeE2E(t *testing.T) { osNetwork := os.Getenv("OS_NETWORK_NAME") if osAuthURL == "" || osUsername == "" || osPassword == "" || osDomain == "" || osRegion == "" || osTenant == "" { - t.Fatal("unable to run test suite, all of OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_REGION, and OS_TENANT OS_DOMAIN must be set!") + t.Fatal("Unable to run test suite, all of OS_AUTH_URL, OS_USERNAME, OS_PASSWORD, OS_REGION, and OS_TENANT OS_DOMAIN must be set!") } params := []string{ @@ -1010,7 +1010,7 @@ func TestDeploymentControllerUpgradesMachineE2E(t *testing.T) { // test data hzToken := os.Getenv("HZ_E2E_TOKEN") if len(hzToken) == 0 { - t.Fatal("unable to run the test suite, HZ_E2E_TOKEN environment variable cannot be empty") + t.Fatal("Unable to run the test suite, HZ_E2E_TOKEN environment variable cannot be empty") } // act @@ -1035,7 +1035,7 @@ func TestAnexiaProvisioningE2E(t *testing.T) { locationID := os.Getenv("ANEXIA_LOCATION_ID") if token == "" || vlanID == "" || templateID == "" || locationID == "" { - t.Fatal("unable to run test suite, all of ANEXIA_TOKEN, ANEXIA_VLAN_ID, ANEXIA_TEMPLATE_ID, and ANEXIA_LOCATION_ID must be set!") + t.Fatal("Unable to run test suite, all of ANEXIA_TOKEN, ANEXIA_VLAN_ID, ANEXIA_TEMPLATE_ID, and ANEXIA_LOCATION_ID must be set!") } selector := OsSelector("flatcar") @@ -1057,7 +1057,7 @@ func TestVultrProvisioningE2E(t *testing.T) { // test data apiKey := os.Getenv("VULTR_API_KEY") if len(apiKey) == 0 { - t.Fatal("unable to run the test suite, VULTR_API_KEY environment variable cannot be empty") + t.Fatal("Unable to run the test suite, VULTR_API_KEY environment variable cannot be empty") } selector := OsSelector("ubuntu", "centos", "rockylinux") diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index d3595c343..6e96db964 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -195,7 +195,7 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar rhsmOfflineToken := os.Getenv("REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN") if rhelSubscriptionManagerUser == "" || rhelSubscriptionManagerPassword == "" || rhsmOfflineToken == "" { - t.Fatalf("Unable to run e2e tests, RHEL_SUBSCRIPTION_MANAGER_USER, RHEL_SUBSCRIPTION_MANAGER_PASSWORD, and " + + t.Fatal("Unable to run e2e tests, RHEL_SUBSCRIPTION_MANAGER_USER, RHEL_SUBSCRIPTION_MANAGER_PASSWORD, and " + "REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN must be set when rhel is used as an os") } diff --git a/test/e2e/provisioning/migrateuidscenario.go b/test/e2e/provisioning/migrateuidscenario.go index 66e3537ad..89fcf3a0c 100644 --- a/test/e2e/provisioning/migrateuidscenario.go +++ b/test/e2e/provisioning/migrateuidscenario.go @@ -23,6 +23,8 @@ import ( "strings" "time" + "go.uber.org/zap" + "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" @@ -39,6 +41,8 @@ import ( ) func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, timeout time.Duration) error { + log := zap.NewNop().Sugar() + // prepare the manifest manifest, err := readAndModifyManifest(manifestPath, parameters) if err != nil { @@ -81,7 +85,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time if err != nil { return fmt.Errorf("failed to get cloud provider %q: %w", providerSpec.CloudProvider, err) } - defaultedSpec, err := prov.AddDefaults(machine.Spec) + defaultedSpec, err := prov.AddDefaults(log, machine.Spec) if err != nil { return fmt.Errorf("failed to add defaults: %w", err) } @@ -92,7 +96,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time // Step 0: Create instance with old UID maxTries := 15 for i := 0; i < maxTries; i++ { - _, err := prov.Get(ctx, machine, providerData) + _, err := prov.Get(ctx, log, machine, providerData) if err != nil { if !errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { if i < maxTries-1 { @@ -102,7 +106,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time } return fmt.Errorf("failed to get machine %s before creating it: %w", machine.Name, err) } - _, err := prov.Create(ctx, machine, providerData, "#cloud-config\n") + _, err := prov.Create(ctx, log, machine, providerData, "#cloud-config\n") if err != nil { if i < maxTries-1 { time.Sleep(10 * time.Second) @@ -117,7 +121,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time // Step 1: Verify we can successfully get the instance for i := 0; i < maxTries; i++ { - if _, err := prov.Get(ctx, machine, providerData); err != nil { + if _, err := prov.Get(ctx, log, machine, providerData); err != nil { if i < maxTries-1 { klog.V(4).Infof("failed to get instance for machine %s before migrating on try %v with err=%v, will retry", machine.Name, i, err) time.Sleep(10 * time.Second) @@ -130,7 +134,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time // Step 2: Migrate UID for i := 0; i < maxTries; i++ { - if err := prov.MigrateUID(ctx, machine, newUID); err != nil { + if err := prov.MigrateUID(ctx, log, machine, newUID); err != nil { if i < maxTries-1 { time.Sleep(10 * time.Second) klog.V(4).Infof("failed to migrate UID for machine %s on try %v with err=%v, will retry", machine.Name, i, err) @@ -144,7 +148,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time // Step 3: Verify we can successfully get the instance with the new UID for i := 0; i < maxTries; i++ { - if _, err := prov.Get(ctx, machine, providerData); err != nil { + if _, err := prov.Get(ctx, log, machine, providerData); err != nil { if i < maxTries-1 { time.Sleep(10 * time.Second) klog.V(4).Infof("failed to get instance for machine %s after migrating on try %v with err=%v, will retry", machine.Name, i, err) @@ -158,7 +162,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time // Step 4: Delete the instance and then verify instance is gone for i := 0; i < maxTries; i++ { // Deletion part 0: Delete and continue on err if there are tries left - done, err := prov.Cleanup(ctx, machine, providerData) + done, err := prov.Cleanup(ctx, log, machine, providerData) if err != nil { if i < maxTries-1 { klog.V(4).Infof("Failed to delete machine %s on try %v with err=%v, will retry", machine.Name, i, err) @@ -174,7 +178,7 @@ func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, time } // Deletion part 1: Get and continue if err != cloudprovidererrors.ErrInstanceNotFound if there are tries left - _, err = prov.Get(ctx, machine, providerData) + _, err = prov.Get(ctx, log, machine, providerData) if err != nil && errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { break } diff --git a/test/e2e/provisioning/verify.go b/test/e2e/provisioning/verify.go index 4e1f9bdc2..5980a453d 100644 --- a/test/e2e/provisioning/verify.go +++ b/test/e2e/provisioning/verify.go @@ -142,7 +142,7 @@ func createAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien // we expect that no node for machine exists in the cluster err := assureNodeForMachineDeployment(machineDeployment, client, false) if err != nil { - return nil, fmt.Errorf("unable to perform the verification, incorrect cluster state detected %w", err) + return nil, fmt.Errorf("failed to perform the verification, incorrect cluster state detected %w", err) } klog.Infof("Creating a new %q MachineDeployment", machineDeployment.Name) @@ -256,7 +256,7 @@ func deleteAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien klog.V(2).Infof("Deleting MachineDeployment %s", machineDeployment.Name) if err := client.Delete(context.Background(), machineDeployment); err != nil { - return fmt.Errorf("unable to remove MachineDeployment %s, due to %w", machineDeployment.Name, err) + return fmt.Errorf("failed to remove MachineDeployment %s, due to %w", machineDeployment.Name, err) } return wait.Poll(machineReadyCheckPeriod, timeout, func() (bool, error) { err := client.Get(context.Background(), types.NamespacedName{Namespace: machineDeployment.Namespace, Name: machineDeployment.Name}, &clusterv1alpha1.MachineDeployment{}) From 81db2c0553f4f9d00d97e464e5d70c5c7aca522e Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Tue, 25 Apr 2023 14:57:09 +0200 Subject: [PATCH 311/489] Use eu-central-1b for AWS arm tests (#1621) Signed-off-by: Marvin Beckers --- .../testdata/machinedeployment-aws-arm-machines.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml index 37567a878..9c489ed27 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml @@ -29,7 +29,7 @@ spec: accessKeyId: << AWS_ACCESS_KEY_ID >> secretAccessKey: << AWS_SECRET_ACCESS_KEY >> region: "eu-central-1" - availabilityZone: "eu-central-1a" + availabilityZone: "eu-central-1b" vpcId: "vpc-079f7648481a11e77" instanceType: "a1.medium" instanceProfile: "kubernetes-v1" From c862e2f64e0a7416e63256ae54ac6f268b99b929 Mon Sep 17 00:00:00 2001 From: Matthias Osthues Date: Wed, 26 Apr 2023 05:16:10 +0200 Subject: [PATCH 312/489] update container linux config transpiler (#1611) * replace unsupported github.com/coreos/container-linux-config-transpile with its supported equivalent * update fixtures --- go.mod | 5 ++- go.sum | 31 +++++++++++++++---- pkg/userdata/convert/ignition-converter.go | 23 +------------- .../flatcar/testdata/ignition_v1.24.0.json | 2 +- .../flatcar/testdata/ignition_v1.24.9.json | 2 +- .../flatcar/testdata/ignition_v1.25.0.json | 2 +- 6 files changed, 31 insertions(+), 34 deletions(-) diff --git a/go.mod b/go.mod index ee1bae4c6..29e93c454 100644 --- a/go.mod +++ b/go.mod @@ -18,9 +18,9 @@ require ( github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0 github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 github.com/aws/smithy-go v1.13.5 - github.com/coreos/container-linux-config-transpiler v0.9.0 github.com/davecgh/go-spew v1.1.1 github.com/digitalocean/godo v1.93.0 + github.com/flatcar/container-linux-config-transpiler v0.9.4 github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v1.2.3 github.com/go-logr/zapr v1.2.3 @@ -83,7 +83,6 @@ require ( github.com/PaesslerAG/gval v1.2.1 // indirect github.com/PaesslerAG/jsonpath v0.1.1 // indirect github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect - github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect @@ -98,12 +97,12 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect - github.com/coreos/ignition v0.35.0 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/docker/distribution v2.8.1+incompatible // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/flatcar/ignition v0.36.2 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect diff --git a/go.sum b/go.sum index be6269960..949e58247 100644 --- a/go.sum +++ b/go.sum @@ -97,15 +97,15 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/agnivade/levenshtein v1.1.0 h1:n6qGwyHG61v3ABce1rPVZklEYRT8NFpCMrpZdBUbYGM= github.com/agnivade/levenshtein v1.1.0/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= +github.com/ajeddeloh/go-json v0.0.0-20160803184958-73d058cf8437/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 h1:4SPQljF/GJ8Q+QlCWMWxRBepub4DresnOm4eI2ebFGc= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= -github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd h1:NlKlOv3aVJ5ODMC0JWPvddw05KENkL3cZttIuu8kJRo= -github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd/go.mod h1:idhzw68Q7v4j+rQ2AGyq3OlZW2Jij9mdmGA4/Sk6J0E= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.62.112 h1:49S6VGQeYyk2KIw85CHbAVaVF2lSgi8xrWDwSw0GCBM= @@ -118,6 +118,7 @@ github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoU github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/aws/aws-sdk-go v1.42.23/go.mod h1:gyRszuZ/icHmHAVE4gc/r+cfCmhA1AD+vqfWbgI+eHs= github.com/aws/aws-sdk-go-v2 v1.17.3 h1:shN7NlnVzvDUgPQ+1rLMSxY8OWRNDRYtiqe0p/PgrhY= github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= @@ -173,14 +174,12 @@ github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/coreos/container-linux-config-transpiler v0.9.0 h1:UBGpT8qWqzi48hNLrzMAgAUNJsR0LW8Gk5/dR/caI8U= -github.com/coreos/container-linux-config-transpiler v0.9.0/go.mod h1:SlcxXZQ2c42knj8pezMiQsM1f+ADxFMjGetuMKR/YSQ= +github.com/coreos/go-semver v0.1.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/ignition v0.35.0 h1:UFodoYq1mOPrbEjtxIsZbThcDyQwAI1owczRDqWmKkQ= -github.com/coreos/ignition v0.35.0/go.mod h1:WJQapxzEn9DE0ryxsGvm8QnBajm/XsS/PkrDqSpz+bA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -228,6 +227,10 @@ github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCv github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/flatcar/container-linux-config-transpiler v0.9.4 h1:yXQ0NB8PeNrKJPrZvbv5/DV63PNhTqt8vaf8YxmX/RA= +github.com/flatcar/container-linux-config-transpiler v0.9.4/go.mod h1:LxanhPvXkWgHG9PrkT4rX/p7YhUPdDGGsUdkNpV3L5U= +github.com/flatcar/ignition v0.36.2 h1:xGHgScUe0P4Fkprjqv7L2CE58emiQgP833OCCn9z2v4= +github.com/flatcar/ignition v0.36.2/go.mod h1:uk1tpzLFRXus4RrvzgMI+IqmmB8a/RGFSBlI+tMTbbA= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -247,6 +250,7 @@ github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -296,6 +300,7 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg78 github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/goccy/go-json v0.9.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -390,6 +395,7 @@ github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2c github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gophercloud/gophercloud v1.1.1 h1:MuGyqbSxiuVBqkPZ3+Nhbytk1xZxhmfCB2Rg1cJWFWM= github.com/gophercloud/gophercloud v1.1.1/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -425,6 +431,7 @@ github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -442,6 +449,7 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= @@ -570,11 +578,13 @@ github.com/packethost/pkg v0.0.0-20211110202003-387414657e83 h1:uhBvTY/Hnm7rLz7g github.com/packethost/pkg v0.0.0-20211110202003-387414657e83/go.mod h1:iF7Mj6XXQ6O+bCfrBCrsJrIGxG7ptrZwb0bW91+wzm8= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterhellberg/link v1.2.0 h1:UA5pg3Gp/E0F2WdX7GERiNrPQrM1K6CVJUUWfHa4t6c= github.com/peterhellberg/link v1.2.0/go.mod h1:gYfAh+oJgQu2SrZHg5hROVRQe1ICoK0/HHJTcE0edxc= +github.com/pin/tftp v2.1.0+incompatible/go.mod h1:xVpZOMCXTy+A5QMjEVN0Glwa1sUvaJhFXbr/aAxuxGY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -632,9 +642,13 @@ github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sigma/bdoor v0.0.0-20160202064022-babf2a4017b0/go.mod h1:WBu7REWbxC/s/J06jsk//d+9DOz9BbsmcIrimuGRFbs= +github.com/sigma/vmw-guestinfo v0.0.0-20160204083807-95dd4126d6e8/go.mod h1:JrRFFC0veyh0cibh0DAhriSY7/gV3kDdNaVUOmfx01U= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= +github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= @@ -684,6 +698,8 @@ github.com/vmware/go-vcloud-director/v2 v2.19.0 h1:A9p95VLn50dm7JbXqg5q+VmQxu3Rx github.com/vmware/go-vcloud-director/v2 v2.19.0/go.mod h1:QPxGFgrUcSyzy9IlpwDE4UNT3tsOy2047tJOPEJ4nlw= github.com/vmware/govmomi v0.30.0 h1:Fm8ugPnnlMSTSceDKY9goGvjmqc6eQLPUSUeNXdpeXA= github.com/vmware/govmomi v0.30.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= +github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= +github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs= github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -746,6 +762,7 @@ go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go4.org v0.0.0-20160314031811-03efcb870d84/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= @@ -812,6 +829,7 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -987,6 +1005,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= diff --git a/pkg/userdata/convert/ignition-converter.go b/pkg/userdata/convert/ignition-converter.go index 6eddf25ca..2d436b5e0 100644 --- a/pkg/userdata/convert/ignition-converter.go +++ b/pkg/userdata/convert/ignition-converter.go @@ -20,30 +20,9 @@ import ( "encoding/json" "fmt" - ctconfig "github.com/coreos/container-linux-config-transpiler/config" - "go.uber.org/zap" - - pluginapi "github.com/kubermatic/machine-controller/pkg/apis/plugin" - "github.com/kubermatic/machine-controller/pkg/userdata/plugin" + ctconfig "github.com/flatcar/container-linux-config-transpiler/config" ) -func NewIgnition(p plugin.Provider) *Ignition { - return &Ignition{p: p} -} - -type Ignition struct { - p plugin.Provider -} - -func (j *Ignition) UserData(log *zap.SugaredLogger, req pluginapi.UserDataRequest) (string, error) { - before, err := j.p.UserData(log, req) - if err != nil { - return "", err - } - - return ToIgnition(before) -} - func ToIgnition(s string) (string, error) { // Convert to ignition cfg, ast, report := ctconfig.Parse([]byte(s)) diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json index 72cd8f383..e12071b17 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json b/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json index 65be0a9d9..67bfc15b5 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.9%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.9%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json index 6baf1897e..cfeeea563 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.2.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.25.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.25.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file From 39e6eaf110547d7e3e37e4fb77fc727e7ace4a2f Mon Sep 17 00:00:00 2001 From: Kubermatic Bot <41968677+kubermatic-bot@users.noreply.github.com> Date: Sat, 29 Apr 2023 11:18:21 +0200 Subject: [PATCH 313/489] Synchronize OWNERS_ALIASES file with Github teams (#1624) --- OWNERS_ALIASES | 1 - 1 file changed, 1 deletion(-) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 879077cbf..d5d600835 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -15,5 +15,4 @@ aliases: - xmudrii - xrstf sig-virtualization: - - hdurand0710 - mfranczy From a8f4e0fd3df8201edf759e5c40862d5cce971644 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 3 May 2023 17:10:24 +0500 Subject: [PATCH 314/489] Support for Kubernetes v1.27 (#1623) * Support for Kubernetes v1.27 Signed-off-by: Waleed Malik * Update build image Signed-off-by: Waleed Malik * Update go dependencies and re-generate fixture data Signed-off-by: Waleed Malik * Refactored code Signed-off-by: Waleed Malik * Skip unsupported in-tree tests for AWS and OpenStack Signed-off-by: Waleed Malik * Refactored code Signed-off-by: Waleed Malik * Adjust spot pricing and tests Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- .prow/e2e-features.yaml | 8 +- .prow/postsubmits.yaml | 4 +- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 16 +- .prow/provider-azure.yaml | 6 +- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 8 +- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 4 +- .prow/provider-vsphere.yaml | 6 +- .prow/verify.yaml | 2 +- README.md | 1 + go.mod | 50 +- go.sum | 93 ++-- hack/run-machine-controller.sh | 1 - hack/verify-licenses.sh | 2 +- pkg/containerruntime/containerd.go | 1 - pkg/userdata/amzn2/provider_test.go | 9 + .../amzn2/testdata/kubelet-v1.24-aws.yaml | 3 +- .../kubelet-v1.24.9-aws-external.yaml | 3 +- .../amzn2/testdata/kubelet-v1.24.9-aws.yaml | 3 +- .../kubelet-v1.24.9-vsphere-mirrors.yaml | 3 +- .../kubelet-v1.24.9-vsphere-proxy.yaml | 3 +- .../testdata/kubelet-v1.24.9-vsphere.yaml | 3 +- .../amzn2/testdata/kubelet-v1.25-aws.yaml | 3 +- .../amzn2/testdata/kubelet-v1.26-aws.yaml | 3 +- .../amzn2/testdata/kubelet-v1.27-aws.yaml | 452 +++++++++++++++++ pkg/userdata/centos/provider_test.go | 9 + .../kubelet-v1.24.9-aws-external.yaml | 3 +- .../centos/testdata/kubelet-v1.24.9-aws.yaml | 3 +- .../testdata/kubelet-v1.24.9-nutanix.yaml | 3 +- .../kubelet-v1.24.9-vsphere-mirrors.yaml | 3 +- .../kubelet-v1.24.9-vsphere-proxy.yaml | 3 +- .../testdata/kubelet-v1.24.9-vsphere.yaml | 3 +- .../centos/testdata/kubelet-v1.25-aws.yaml | 3 +- .../centos/testdata/kubelet-v1.26-aws.yaml | 3 +- .../centos/testdata/kubelet-v1.27-aws.yaml | 458 ++++++++++++++++++ .../flatcar/testdata/cloud-init_v1.24.0.yaml | 3 +- .../flatcar/testdata/cloud-init_v1.24.9.yaml | 3 +- .../flatcar/testdata/cloud-init_v1.25.0.yaml | 3 +- pkg/userdata/flatcar/testdata/containerd.yaml | 3 +- .../flatcar/testdata/ignition_v1.24.0.json | 2 +- .../flatcar/testdata/ignition_v1.24.9.json | 2 +- .../flatcar/testdata/ignition_v1.25.0.json | 2 +- pkg/userdata/helper/common_test.go | 7 +- .../helper/download_binaries_script.go | 2 +- .../download_binaries_v1.24.13.golden | 17 + .../testdata/download_binaries_v1.25.9.golden | 17 + .../testdata/download_binaries_v1.26.4.golden | 17 + .../testdata/download_binaries_v1.27.1.golden | 17 + ...temd_unit_version-v1.24.13-external.golden | 36 ++ ...ublet_systemd_unit_version-v1.24.13.golden | 35 ++ ...stemd_unit_version-v1.25.9-external.golden | 36 ++ ...kublet_systemd_unit_version-v1.25.9.golden | 35 ++ ...stemd_unit_version-v1.26.4-external.golden | 36 ++ ...kublet_systemd_unit_version-v1.26.4.golden | 35 ++ ...stemd_unit_version-v1.27.1-external.golden | 36 ++ ...kublet_systemd_unit_version-v1.27.1.golden | 35 ++ .../safe_download_binaries_v1.24.9.golden | 2 +- .../testdata/kubelet-v1.24-aws-external.yaml | 3 +- .../rhel/testdata/kubelet-v1.24-aws.yaml | 3 +- .../kubelet-v1.24.9-aws-external.yaml | 3 +- .../rhel/testdata/kubelet-v1.24.9-aws.yaml | 3 +- .../kubelet-v1.24.9-vsphere-mirrors.yaml | 3 +- .../kubelet-v1.24.9-vsphere-proxy.yaml | 3 +- .../testdata/kubelet-v1.24.9-vsphere.yaml | 3 +- .../rhel/testdata/kubelet-v1.25-aws.yaml | 3 +- .../rhel/testdata/kubelet-v1.25-nutanix.yaml | 3 +- .../rhel/testdata/pod-cidr-azure-rhel.yaml | 3 +- .../testdata/kubelet-v1.24-aws.yaml | 3 +- .../kubelet-v1.24.9-aws-external.yaml | 3 +- .../testdata/kubelet-v1.24.9-aws.yaml | 3 +- .../testdata/kubelet-v1.24.9-nutanix.yaml | 3 +- .../kubelet-v1.24.9-vsphere-mirrors.yaml | 3 +- .../kubelet-v1.24.9-vsphere-proxy.yaml | 3 +- .../testdata/kubelet-v1.24.9-vsphere.yaml | 3 +- .../testdata/kubelet-v1.25-aws.yaml | 3 +- pkg/userdata/ubuntu/provider_test.go | 7 +- pkg/userdata/ubuntu/testdata/containerd.yaml | 3 +- .../digitalocean-dualstack-IPv6+IPv4.yaml | 3 +- .../testdata/digitalocean-dualstack.yaml | 3 +- .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 3 +- pkg/userdata/ubuntu/testdata/docker.yaml | 3 +- .../kubelet-version-without-v-prefix.yaml | 2 +- .../ubuntu/testdata/multiple-dns-servers.yaml | 3 +- .../ubuntu/testdata/multiple-ssh-keys.yaml | 2 +- pkg/userdata/ubuntu/testdata/nutanix.yaml | 3 +- .../openstack-dualstack-IPv6+IPv4.yaml | 3 +- .../ubuntu/testdata/openstack-dualstack.yaml | 3 +- .../openstack-overwrite-cloud-config.yaml | 2 +- pkg/userdata/ubuntu/testdata/openstack.yaml | 3 +- .../ubuntu/testdata/version-1.24.13.yaml | 457 +++++++++++++++++ .../ubuntu/testdata/version-1.24.9.yaml | 2 +- .../ubuntu/testdata/version-1.25.5.yaml | 2 +- .../ubuntu/testdata/version-1.25.9.yaml | 457 +++++++++++++++++ .../ubuntu/testdata/version-1.26.0.yaml | 2 +- .../ubuntu/testdata/version-1.26.4.yaml | 457 +++++++++++++++++ .../ubuntu/testdata/version-1.27.1.yaml | 457 +++++++++++++++++ .../ubuntu/testdata/vsphere-mirrors.yaml | 2 +- .../ubuntu/testdata/vsphere-proxy.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere.yaml | 2 +- test/e2e/provisioning/all_e2e_test.go | 42 +- test/e2e/provisioning/helper.go | 30 +- ...deployment-aws-ebs-encryption-enabled.yaml | 2 +- .../machinedeployment-aws-spot-instances.yaml | 2 +- .../testdata/machinedeployment-aws.yaml | 2 +- 113 files changed, 3334 insertions(+), 260 deletions(-) create mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml create mode 100644 pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.24.13.golden create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.25.9.golden create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.26.4.golden create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.27.1.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.13-external.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.13.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.9-external.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.9.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.4-external.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.4.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.1-external.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.1.golden create mode 100644 pkg/userdata/ubuntu/testdata/version-1.24.13.yaml create mode 100644 pkg/userdata/ubuntu/testdata/version-1.25.9.yaml create mode 100644 pkg/userdata/ubuntu/testdata/version-1.26.4.yaml create mode 100644 pkg/userdata/ubuntu/testdata/version-1.27.1.yaml diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index c0d2df969..769eb3614 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -91,7 +91,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -118,7 +118,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 65ba53edb..f4ecca9d1 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index c4a0375c1..a424ee67d 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index d00c6bdc5..99f79b4aa 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 6e1d29d08..dc4a5a3fe 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -156,7 +156,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -187,7 +187,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -218,7 +218,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -249,7 +249,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index e0cfdc5a7..957807079 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 6145fb7f4..44e09b822 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 7e1b222f0..6659d9802 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index f12c38ae7..21ee1756f 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 602f5706e..0200e89ee 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index b6c3e1d78..4ae7aed97 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index db653095f..802769760 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index d7b2b58ad..44a2e5f31 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 1a6913f88..0a73d28f0 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -15,10 +15,6 @@ presubmits: - name: pull-machine-controller-e2e-openstack run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" - # We've made the E2E tests for OpenStack optional since in-tree cloud provider for OpenStack was removed with k8s v1.26. Since MC depends on the in-tree cloud provider - # the tests on k8s v1.26+ will fail. - # TODO: These tests shouldn't be marked as optional. - optional: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -32,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -64,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 15c9a6a95..3897873b5 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index b51fe7283..73e7f4a6f 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -17,7 +17,7 @@ presubmits: always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - run_if_changed: "(pkg/cloudprovider/provider/vmwareclouddirector/|pkg/userdata)" + run_if_changed: "(pkg/cloudprovider/provider/vmwareclouddirector/)" labels: preset-vcloud-director: "true" preset-hetzner: "true" @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 86f0db674..2099a030a 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 9ab548f3e..ad2f48853 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 command: - ./hack/verify-licenses.sh resources: diff --git a/README.md b/README.md index cd9f9b6fb..aadaf94ca 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ machine-controller tries to follow the Kubernetes version Currently supported K8S versions are: +- 1.27 - 1.26 - 1.25 - 1.24 diff --git a/go.mod b/go.mod index 29e93c454..57dafd386 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/digitalocean/godo v1.93.0 github.com/flatcar/container-linux-config-transpiler v0.9.4 github.com/ghodss/yaml v1.0.0 - github.com/go-logr/logr v1.2.3 + github.com/go-logr/logr v1.2.4 github.com/go-logr/zapr v1.2.3 github.com/go-test/deep v1.0.8 github.com/google/uuid v1.3.0 @@ -36,7 +36,7 @@ require ( github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 - github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_golang v1.15.0 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10 github.com/sethvargo/go-password v0.2.0 github.com/spf13/pflag v1.0.5 @@ -47,22 +47,22 @@ require ( go.anx.io/go-anxcloud v0.5.0 go.uber.org/zap v1.24.0 golang.org/x/crypto v0.4.0 - golang.org/x/oauth2 v0.3.0 + golang.org/x/oauth2 v0.5.0 gomodules.xyz/jsonpatch/v2 v2.2.0 google.golang.org/api v0.105.0 google.golang.org/grpc v1.51.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.26.0 - k8s.io/apiextensions-apiserver v0.26.0 - k8s.io/apimachinery v0.26.0 + k8s.io/api v0.26.4 + k8s.io/apiextensions-apiserver v0.26.4 + k8s.io/apimachinery v0.26.4 k8s.io/client-go v12.0.0+incompatible k8s.io/klog v1.0.0 - k8s.io/kubelet v0.26.0 - k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 + k8s.io/kubelet v0.26.4 + k8s.io/utils v0.0.0-20230209194617-a36077c30491 kubevirt.io/api v0.58.0 kubevirt.io/containerized-data-importer-api v1.55.2 - sigs.k8s.io/controller-runtime v0.14.1 + sigs.k8s.io/controller-runtime v0.14.6 sigs.k8s.io/yaml v1.3.0 ) @@ -109,14 +109,16 @@ require ( github.com/go-openapi/jsonreference v0.20.1 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/go-resty/resty/v2 v2.7.0 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -138,8 +140,8 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/ginkgo/v2 v2.6.0 // indirect - github.com/onsi/gomega v1.24.1 // indirect + github.com/onsi/ginkgo/v2 v2.9.2 // indirect + github.com/onsi/gomega v1.27.6 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect @@ -147,9 +149,9 @@ require ( github.com/packethost/pkg v0.0.0-20211110202003-387414657e83 // indirect github.com/peterhellberg/link v1.2.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.39.0 // indirect + github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect - github.com/rogpeppe/go-internal v1.9.0 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/cobra v1.6.1 // indirect @@ -162,26 +164,28 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/net v0.8.0 // indirect golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/term v0.6.0 // indirect + golang.org/x/text v0.8.0 // indirect golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.7.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/component-base v0.26.0 // indirect - k8s.io/klog/v2 v2.80.1 // indirect - k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715 // indirect + k8s.io/component-base v0.26.4 // indirect + k8s.io/klog/v2 v2.90.1 // indirect + k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) -replace k8s.io/client-go => k8s.io/client-go v0.26.0 +// TODO: Upgrade to 0.27.x once sigs.k8s.io/controller-runtime 0.14.7 is released. For more details https://github.com/kubernetes/client-go/issues/1245#issuecomment-1523434471 +replace k8s.io/client-go => k8s.io/client-go v0.26.4 diff --git a/go.sum b/go.sum index 949e58247..6c4c601dc 100644 --- a/go.sum +++ b/go.sum @@ -262,8 +262,9 @@ github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= @@ -297,6 +298,8 @@ github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPr github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/goccy/go-json v0.9.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -338,8 +341,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -379,6 +383,8 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -548,8 +554,8 @@ github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47 github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= -github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc= -github.com/onsi/ginkgo/v2 v2.6.0/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= +github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= +github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= @@ -560,8 +566,8 @@ github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeR github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= -github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= -github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 h1:+ShYlGoPriGahTTFTjQ0RtNXW0srxDodk2STdc238Rk= @@ -596,8 +602,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= +github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -608,8 +614,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.31.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= -github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -623,8 +629,9 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rollbar/rollbar-go v1.4.2/go.mod h1:kLQ9gP3WCRGrvJmF0ueO3wK9xWocej8GRX98D8sa39w= github.com/rollbar/rollbar-go/errors v0.0.0-20210929193720-32947096267e/go.mod h1:Ie0xEc1Cyj+T4XMO8s0Vf7pMfvSAAy1sb4AYc8aJsao= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -874,9 +881,9 @@ golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -884,8 +891,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= -golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= +golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= +golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -968,17 +975,17 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -988,9 +995,9 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1055,6 +1062,8 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1159,8 +1168,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1208,22 +1218,22 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= -k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I= -k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= -k8s.io/apiextensions-apiserver v0.26.0 h1:Gy93Xo1eg2ZIkNX/8vy5xviVSxwQulsnUdQ00nEdpDo= -k8s.io/apiextensions-apiserver v0.26.0/go.mod h1:7ez0LTiyW5nq3vADtK6C3kMESxadD51Bh6uz3JOlqWQ= +k8s.io/api v0.26.4 h1:qSG2PmtcD23BkYiWfoYAcak870eF/hE7NNYBYavTT94= +k8s.io/api v0.26.4/go.mod h1:WwKEXU3R1rgCZ77AYa7DFksd9/BAIKyOmRlbVxgvjCk= +k8s.io/apiextensions-apiserver v0.26.4 h1:9D2RTxYGxrG5uYg6D7QZRcykXvavBvcA59j5kTaedQI= +k8s.io/apiextensions-apiserver v0.26.4/go.mod h1:cd4uGFGIgzEqUghWpRsr9KE8j2KNTjY8Ji8pnMMazyw= k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apimachinery v0.26.0 h1:1feANjElT7MvPqp0JT6F3Ss6TWDwmcjLypwoPpEf7zg= -k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= -k8s.io/client-go v0.26.0 h1:lT1D3OfO+wIi9UFolCrifbjUUgu7CpLca0AD8ghRLI8= -k8s.io/client-go v0.26.0/go.mod h1:I2Sh57A79EQsDmn7F7ASpmru1cceh3ocVT9KlX2jEZg= +k8s.io/apimachinery v0.26.4 h1:rZccKdBLg9vP6J09JD+z8Yr99Ce8gk3Lbi9TCx05Jzs= +k8s.io/apimachinery v0.26.4/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= +k8s.io/client-go v0.26.4 h1:/7P/IbGBuT73A+G97trf44NTPSNqvuBREpOfdLbHvD4= +k8s.io/client-go v0.26.4/go.mod h1:6qOItWm3EwxJdl/8p5t7FWtWUOwyMdA8N9ekbW4idpI= k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.26.0 h1:0IkChOCohtDHttmKuz+EP3j3+qKmV55rM9gIFTXA7Vs= -k8s.io/component-base v0.26.0/go.mod h1:lqHwlfV1/haa14F/Z5Zizk5QmzaVf23nQzCwVOQpfC8= +k8s.io/component-base v0.26.4 h1:Bg2xzyXNKL3eAuiTEu3XE198d6z22ENgFgGQv2GGOUk= +k8s.io/component-base v0.26.4/go.mod h1:lTuWL1Xz/a4e80gmIC3YZG2JCO4xNwtKWHJWeJmsq20= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= @@ -1234,23 +1244,24 @@ k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= +k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715 h1:tBEbstoM+K0FiBV5KGAKQ0kuvf54v/hwpldiJt69w1s= -k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= -k8s.io/kubelet v0.26.0 h1:08bDb5IoUH/1K1t2NUwnGIIWxjm9LSqn6k3FWw1tJGI= -k8s.io/kubelet v0.26.0/go.mod h1:DluF+d8jS2nE/Hs7CC3QM+OZlIEb22NTOihQ3EDwCQ4= +k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a h1:gmovKNur38vgoWfGtP5QOGNOA7ki4n6qNYoFAgMlNvg= +k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= +k8s.io/kubelet v0.26.4 h1:SEQPfjN4lu4uL9O8NdeN7Aum3liQ4kOnp/yC3jMRMUo= +k8s.io/kubelet v0.26.4/go.mod h1:ZMPGTCnrQ5UOlC7igXhbW9cgna1LtTRWLaHub4dA2FU= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= -k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= +k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v0.58.0 h1:qeNeRtD6AIJ5WVJuRXajmmXtnrO5dYchy+hpCm6QwhE= kubevirt.io/api v0.58.0/go.mod h1:U0CQlZR0JoJCaC+Va0wz4dMOtYDdVywJ98OT1KmOkzI= kubevirt.io/containerized-data-importer-api v1.55.2 h1:AzYnKIUFkKwO6c0uCQZYlAIxfzbiPkJXP29hFhauaQ8= @@ -1261,8 +1272,8 @@ mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.14.1 h1:vThDes9pzg0Y+UbCPY3Wj34CGIYPgdmspPm2GIpxpzM= -sigs.k8s.io/controller-runtime v0.14.1/go.mod h1:GaRkrY8a7UZF0kqFFbUKG7n9ICiTY5T55P1RiE3UZlU= +sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= +sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= diff --git a/hack/run-machine-controller.sh b/hack/run-machine-controller.sh index a21eddc05..2eaff7d45 100755 --- a/hack/run-machine-controller.sh +++ b/hack/run-machine-controller.sh @@ -27,7 +27,6 @@ make -C $(dirname $0)/.. build-machine-controller $(dirname $0)/../machine-controller \ -kubeconfig=$MC_KUBECONFIG \ -worker-count=50 \ - -logtostderr \ -log-debug \ -cluster-dns=169.254.20.10 \ -enable-profiling \ diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 4eaa345c0..74ea661bd 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.20-node-18-kind-0.17-4 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 containerize ./hack/verify-licenses.sh go mod vendor diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index 02bc3e85f..7b9d3469a 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -52,7 +52,6 @@ func (eng *Containerd) AuthConfigFileName() string { func (eng *Containerd) KubeletFlags() []string { return []string{ - "--container-runtime=remote", "--container-runtime-endpoint=unix:///run/containerd/containerd.sock", } } diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index f9c93ae16..53866c402 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -185,6 +185,15 @@ func TestUserDataGeneration(t *testing.T) { }, }, }, + { + name: "kubelet-v1.27-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.27.0", + }, + }, + }, } defaultCloudProvider := &fakeCloudConfigProvider{ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml index 6cbbe4ad6..8d7237688 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml @@ -131,7 +131,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -227,7 +227,6 @@ write_files: --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml index 0e2a2237b..25422f0f0 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml @@ -131,7 +131,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -227,7 +227,6 @@ write_files: --hostname-override=${KUBELET_HOSTNAME} \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml index 43a9853b2..c71e4fccc 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml @@ -131,7 +131,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -227,7 +227,6 @@ write_files: --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml index b90489c90..7b65738b3 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml @@ -144,7 +144,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -244,7 +244,6 @@ write_files: --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml index 5e0a08c15..3fa66891b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml @@ -144,7 +144,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -244,7 +244,6 @@ write_files: --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml index 880f0ec4b..36afcc388 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml @@ -136,7 +136,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -235,7 +235,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml index 318b1b646..119a07ae3 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml @@ -131,7 +131,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -227,7 +227,6 @@ write_files: --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml index 4115f30da..8fb671824 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml @@ -131,7 +131,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -227,7 +227,6 @@ write_files: --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml new file mode 100644 index 000000000..d4c24db96 --- /dev/null +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml @@ -0,0 +1,452 @@ +#cloud-config + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + + mkdir -p /etc/systemd/system/containerd.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index 9dec8b334..fccccab59 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -186,6 +186,15 @@ func TestUserDataGeneration(t *testing.T) { }, }, }, + { + name: "kubelet-v1.27-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.27.0", + }, + }, + }, } defaultCloudProvider := &fakeCloudConfigProvider{ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml index e9998c449..4704bf099 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml @@ -137,7 +137,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -233,7 +233,6 @@ write_files: --hostname-override=${KUBELET_HOSTNAME} \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml index a3213cfad..28c514eb6 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml @@ -137,7 +137,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -233,7 +233,6 @@ write_files: --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml index 98e8e1f9a..78cbe025e 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml @@ -144,7 +144,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -241,7 +241,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml index 78d8f31a7..5226b46ee 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml @@ -150,7 +150,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -250,7 +250,6 @@ write_files: --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml index e746b4110..04204ee53 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml @@ -150,7 +150,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -250,7 +250,6 @@ write_files: --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml index e2000bbb3..05284d026 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml @@ -142,7 +142,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -241,7 +241,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml index be894d2a5..f95d97cd0 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml @@ -137,7 +137,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -233,7 +233,6 @@ write_files: --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml index 847bd55cf..32442cf78 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml @@ -137,7 +137,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -233,7 +233,6 @@ write_files: --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml new file mode 100644 index 000000000..4fa5f0fbc --- /dev/null +++ b/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml @@ -0,0 +1,458 @@ +#cloud-config + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml index 55cd7654d..89b6727e1 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.0.yaml @@ -133,7 +133,6 @@ coreos: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} @@ -419,7 +418,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml index 4cea15a05..c2c598fd9 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml @@ -133,7 +133,6 @@ coreos: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} @@ -419,7 +418,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml index 9be63ca86..c746bdfad 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml @@ -133,7 +133,6 @@ coreos: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} @@ -419,7 +418,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index 785e85e8a..8bd1f94a6 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -112,7 +112,6 @@ coreos: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} @@ -398,7 +397,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json index e12071b17..ee168b4a6 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.24.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json b/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json index 67bfc15b5..3dea0b8e8 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.24.9.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.9%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.24.9%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json index cfeeea563..0104db49f 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.26.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.25.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.25.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/common_test.go b/pkg/userdata/helper/common_test.go index 3917f2599..894e2bb34 100644 --- a/pkg/userdata/helper/common_test.go +++ b/pkg/userdata/helper/common_test.go @@ -26,8 +26,9 @@ var update = flag.Bool("update", false, "update testdata files") var ( versions = []*semver.Version{ - semver.MustParse("v1.24.9"), - semver.MustParse("v1.25.5"), - semver.MustParse("v1.26.0"), + semver.MustParse("v1.24.13"), + semver.MustParse("v1.25.9"), + semver.MustParse("v1.26.4"), + semver.MustParse("v1.27.1"), } ) diff --git a/pkg/userdata/helper/download_binaries_script.go b/pkg/userdata/helper/download_binaries_script.go index 6a10bfaa2..8495c450b 100644 --- a/pkg/userdata/helper/download_binaries_script.go +++ b/pkg/userdata/helper/download_binaries_script.go @@ -168,7 +168,7 @@ func SafeDownloadBinariesScript(log *zap.SugaredLogger, kubeVersion string) (str const ( CNIVersion = "v1.2.0" - CRIToolsVersion = "v1.26.0" + CRIToolsVersion = "v1.27.0" ) // force v in case if it's not there diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.24.13.golden b/pkg/userdata/helper/testdata/download_binaries_v1.24.13.golden new file mode 100644 index 000000000..b50e48e2a --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.24.13.golden @@ -0,0 +1,17 @@ +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - +fi +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.24.13/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.25.9.golden b/pkg/userdata/helper/testdata/download_binaries_v1.25.9.golden new file mode 100644 index 000000000..864f2924f --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.25.9.golden @@ -0,0 +1,17 @@ +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - +fi +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.25.9/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.26.4.golden b/pkg/userdata/helper/testdata/download_binaries_v1.26.4.golden new file mode 100644 index 000000000..727746ddf --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.26.4.golden @@ -0,0 +1,17 @@ +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - +fi +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.26.4/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.27.1.golden b/pkg/userdata/helper/testdata/download_binaries_v1.27.1.golden new file mode 100644 index 000000000..589d5eaca --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.27.1.golden @@ -0,0 +1,17 @@ +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - +fi +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.27.1/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.13-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.13-external.golden new file mode 100644 index 000000000..50f4f5138 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.13-external.golden @@ -0,0 +1,36 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.13.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.13.golden new file mode 100644 index 000000000..e70567560 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.24.13.golden @@ -0,0 +1,35 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.9-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.9-external.golden new file mode 100644 index 000000000..50f4f5138 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.9-external.golden @@ -0,0 +1,36 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.9.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.9.golden new file mode 100644 index 000000000..e70567560 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.9.golden @@ -0,0 +1,35 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.4-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.4-external.golden new file mode 100644 index 000000000..50f4f5138 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.4-external.golden @@ -0,0 +1,36 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.4.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.4.golden new file mode 100644 index 000000000..e70567560 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.4.golden @@ -0,0 +1,35 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.1-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.1-external.golden new file mode 100644 index 000000000..50f4f5138 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.1-external.golden @@ -0,0 +1,36 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.1.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.1.golden new file mode 100644 index 000000000..e70567560 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.1.golden @@ -0,0 +1,35 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/safe_download_binaries_v1.24.9.golden b/pkg/userdata/helper/testdata/safe_download_binaries_v1.24.9.golden index 98f57b9f7..5bc84770c 100644 --- a/pkg/userdata/helper/testdata/safe_download_binaries_v1.24.9.golden +++ b/pkg/userdata/helper/testdata/safe_download_binaries_v1.24.9.golden @@ -28,7 +28,7 @@ sha256sum -c <<<"$cni_sum" tar xvf "$cni_filename" rm -f "$cni_filename" cd - -CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" +CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml index 9bdf86c67..df2211926 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml @@ -131,7 +131,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -240,7 +240,6 @@ write_files: --hostname-override=${KUBELET_HOSTNAME} \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml index d67ddaf1f..6b0e675fa 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml @@ -131,7 +131,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -240,7 +240,6 @@ write_files: --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws-external.yaml index d77002e64..2cfbb566e 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws-external.yaml @@ -131,7 +131,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -240,7 +240,6 @@ write_files: --hostname-override=${KUBELET_HOSTNAME} \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml index ae510dc1d..8b5e11e13 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml @@ -131,7 +131,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -240,7 +240,6 @@ write_files: --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml index a4341ba36..4e5864485 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml @@ -145,7 +145,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -258,7 +258,6 @@ write_files: --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml index 9a3a0586d..c476db0f3 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml @@ -145,7 +145,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -258,7 +258,6 @@ write_files: --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml index 854c91ec6..c570a43f7 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml @@ -137,7 +137,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -249,7 +249,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml index 599aa75eb..d9faed107 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml @@ -131,7 +131,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -240,7 +240,6 @@ write_files: --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml index 8e0a31352..8b961fb3b 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml @@ -139,7 +139,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -249,7 +249,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index f3bc71e64..41562a0b3 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -136,7 +136,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -246,7 +246,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml index 910ed957c..6220c331d 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml @@ -132,7 +132,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -240,7 +240,6 @@ write_files: --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml index ea98aa141..6bd394ef3 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml @@ -132,7 +132,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -240,7 +240,6 @@ write_files: --hostname-override=${KUBELET_HOSTNAME} \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml index eb87f3395..bd622831e 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml @@ -132,7 +132,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -240,7 +240,6 @@ write_files: --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml index 9cbbf5b50..1d25a9ce1 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml @@ -139,7 +139,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -248,7 +248,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml index b4452384f..03e7e7b37 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml @@ -145,7 +145,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -257,7 +257,6 @@ write_files: --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml index 1070b5fd3..42348f33d 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml @@ -145,7 +145,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -257,7 +257,6 @@ write_files: --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml index 64521cbc6..64f29c5c4 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml @@ -137,7 +137,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -248,7 +248,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml index ee32e7e03..4827b5600 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml @@ -132,7 +132,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -240,7 +240,6 @@ write_files: --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index f3f722a50..351d9c909 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -129,9 +129,10 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.24.9"), - semver.MustParse("v1.25.5"), - semver.MustParse("v1.26.0"), + semver.MustParse("v1.24.13"), + semver.MustParse("v1.25.9"), + semver.MustParse("v1.26.4"), + semver.MustParse("v1.27.1"), } var tests []userDataTestCase diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index dd7dd8d91..bedf7108a 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -140,7 +140,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -231,7 +231,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml index 71c2a3f56..dd81f757d 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml @@ -138,7 +138,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -229,7 +229,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml index e1bf0ec28..52f389862 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml @@ -138,7 +138,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -229,7 +229,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 2e56201ab..1c361c0ea 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -140,7 +140,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -231,7 +231,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml index 311ae4731..a8e936d7e 100644 --- a/pkg/userdata/ubuntu/testdata/docker.yaml +++ b/pkg/userdata/ubuntu/testdata/docker.yaml @@ -140,7 +140,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -231,7 +231,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index 99f32fe25..30056aeaa 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -138,7 +138,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 77949c18c..e434071a0 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -138,7 +138,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -229,7 +229,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 38e8e3432..3b2ec3f6d 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -140,7 +140,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 3b453a2b6..26c10bc00 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -141,7 +141,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -234,7 +234,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml index 331eea036..2e0b658e1 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml @@ -138,7 +138,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -230,7 +230,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ [Install] diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml index 8cc558edf..6b1ae8c98 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml @@ -138,7 +138,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -230,7 +230,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ [Install] diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index b8ada4586..ed9bd6495 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -138,7 +138,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index d35ee8ecd..3fbef6c8f 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -138,7 +138,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" @@ -231,7 +231,6 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.13.yaml b/pkg/userdata/ubuntu/testdata/version-1.24.13.yaml new file mode 100644 index 000000000..45876c521 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.24.13.yaml @@ -0,0 +1,457 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml b/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml index b38c94941..d1b970bef 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml @@ -138,7 +138,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml index 0c53f84ed..6fbbae81d 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml @@ -138,7 +138,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/version-1.25.9.yaml b/pkg/userdata/ubuntu/testdata/version-1.25.9.yaml new file mode 100644 index 000000000..f1d40f1f9 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.25.9.yaml @@ -0,0 +1,457 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml index 6545373b7..e284cdb1b 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml @@ -138,7 +138,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/version-1.26.4.yaml b/pkg/userdata/ubuntu/testdata/version-1.26.4.yaml new file mode 100644 index 000000000..e35832c2d --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.26.4.yaml @@ -0,0 +1,457 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.27.1.yaml b/pkg/userdata/ubuntu/testdata/version-1.27.1.yaml new file mode 100644 index 000000000..86dd288d9 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.27.1.yaml @@ -0,0 +1,457 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index c24a42c5d..686624445 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -148,7 +148,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index eee9a28b5..258296571 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -148,7 +148,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index c5341ba32..7435de199 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -139,7 +139,7 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index ba16c682d..85c796519 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -82,7 +82,7 @@ const ( ) const ( - defaultKubernetesVersion = "1.24.9" + defaultKubernetesVersion = "1.25.9" defaultContainerRuntime = "containerd" ) @@ -340,7 +340,8 @@ func TestOpenstackProvisioningE2E(t *testing.T) { fmt.Sprintf("<< NETWORK_NAME >>=%s", osNetwork), } - selector := Not(OsSelector("amzn2")) + // In-tree cloud provider is not supported from Kubernetes v1.26. + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.26.4", "1.27.1"))) runScenarios(t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -419,7 +420,8 @@ func TestAWSProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } - selector := Not(OsSelector("sles")) + // In-tree cloud provider is not supported from Kubernetes v1.27. + selector := Not(VersionSelector("1.27.1")) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -472,7 +474,9 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. - selector := OsSelector("ubuntu") + // In-tree cloud provider is not supported from Kubernetes v1.27. + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.1"))) + // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), fmt.Sprintf("<< AWS_SECRET_ACCESS_KEY >>=%s", awsSecret), @@ -492,7 +496,9 @@ func TestAWSARMProvisioningE2E(t *testing.T) { if len(awsKeyID) == 0 || len(awsSecret) == 0 { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } - selector := OsSelector("ubuntu") + // In-tree cloud provider is not supported from Kubernetes v1.27. + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.1"))) + // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), fmt.Sprintf("<< AWS_SECRET_ACCESS_KEY >>=%s", awsSecret), @@ -522,32 +528,6 @@ func TestAWSFlatcarCoreOSCloudInit8ProvisioningE2E(t *testing.T) { runScenarios(t, selector, params, AWSManifest, fmt.Sprintf("aws-%s", *testRunIdentifier)) } -func TestAWSFlatcarContainerdProvisioningE2E(t *testing.T) { - t.Parallel() - - // test data - awsKeyID := os.Getenv("AWS_E2E_TESTS_KEY_ID") - awsSecret := os.Getenv("AWS_E2E_TESTS_SECRET") - if len(awsKeyID) == 0 || len(awsSecret) == 0 { - t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") - } - - params := []string{ - fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), - fmt.Sprintf("<< AWS_SECRET_ACCESS_KEY >>=%s", awsSecret), - fmt.Sprintf("<< PROVISIONING_UTILITY >>=%s", flatcar.Ignition), - } - - scenario := scenario{ - name: "flatcar with containerd in AWS", - osName: "flatcar", - containerRuntime: defaultContainerRuntime, - kubernetesVersion: defaultKubernetesVersion, - executor: verifyCreateAndDelete, - } - testScenario(t, scenario, *testRunIdentifier, params, AWSManifest, false) -} - func TestAWSCentOS8ProvisioningE2E(t *testing.T) { t.Parallel() diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 6e96db964..11a0e7c98 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,9 +33,10 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.24.10"), - semver.MustParse("v1.25.6"), - semver.MustParse("v1.26.1"), + semver.MustParse("v1.24.13"), + semver.MustParse("v1.25.9"), + semver.MustParse("v1.26.4"), + semver.MustParse("v1.27.1"), } operatingSystems = []providerconfigtypes.OperatingSystem{ @@ -156,9 +157,30 @@ func (n *name) Match(tc scenario) bool { return tc.name == n.name } +// VersionSelector is used to match against the kubernetes version used for a test case. +func VersionSelector(v ...string) Selector { + return &version{v} +} + +type version struct { + versions []string +} + +var _ Selector = &version{} + +func (v *version) Match(testCase scenario) bool { + for _, version := range v.versions { + if testCase.kubernetesVersion == version { + return true + } + } + return false +} + func runScenarios(st *testing.T, selector Selector, testParams []string, manifestPath string, cloudProvider string) { for _, testCase := range scenarios { if selector != nil && !selector.Match(testCase) { + fmt.Printf("Skipping test %s\n", testCase.name) continue } @@ -212,7 +234,7 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar scenarioParams = append(scenarioParams, fmt.Sprintf("<< DATA_DISK_SIZE >>=%v", 30)) scenarioParams = append(scenarioParams, fmt.Sprintf("<< DISK_SIZE >>=%v", 25)) scenarioParams = append(scenarioParams, fmt.Sprintf("<< CUSTOM-IMAGE >>=%v", "")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.03")) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.02")) } if strings.Contains(cloudProvider, string(providerconfigtypes.CloudProviderEquinixMetal)) { diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml index a496f0693..cb22438fb 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml @@ -31,7 +31,7 @@ spec: region: "eu-central-1" availabilityZone: "eu-central-1a" vpcId: "vpc-079f7648481a11e77" - instanceType: "t2.medium" + instanceType: "t3a.small" instanceProfile: "kubernetes-v1" diskSize: 50 diskType: "gp2" diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml index aa35c5cfa..2b93bc7aa 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml @@ -31,7 +31,7 @@ spec: region: "eu-central-1" availabilityZone: "eu-central-1a" vpcId: "vpc-079f7648481a11e77" - instanceType: "t2.medium" + instanceType: "t3a.small" instanceProfile: "kubernetes-v1" diskSize: 50 diskType: "gp2" diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws.yaml index 77ac2bb3b..e9ef7ba47 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws.yaml @@ -33,7 +33,7 @@ spec: region: "eu-central-1" availabilityZone: "eu-central-1a" vpcId: "vpc-079f7648481a11e77" - instanceType: "t2.medium" + instanceType: "t3a.small" instanceProfile: "kubernetes-v1" diskSize: 50 diskType: "gp2" From 0ee7860419bf68b4353ac373c1284a46467b4278 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Thu, 4 May 2023 08:01:17 +0200 Subject: [PATCH 315/489] fix missing formatting in log line (#1628) --- pkg/node/eviction/eviction.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/node/eviction/eviction.go b/pkg/node/eviction/eviction.go index 10d81e139..e51bd1c7d 100644 --- a/pkg/node/eviction/eviction.go +++ b/pkg/node/eviction/eviction.go @@ -59,7 +59,7 @@ func (ne *NodeEviction) Run(ctx context.Context, log *zap.SugaredLogger) (bool, return false, fmt.Errorf("failed to get node from lister: %w", err) } if _, exists := node.Annotations[evictiontypes.SkipEvictionAnnotationKey]; exists { - nodeLog.Info("Skipping eviction for node as it has a %s annotation", evictiontypes.SkipEvictionAnnotationKey) + nodeLog.Infof("Skipping eviction for node as it has a %s annotation", evictiontypes.SkipEvictionAnnotationKey) return false, nil } From fdd3c561e2d1b73a2a792041126bce28f3b238dc Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Fri, 5 May 2023 11:51:22 +0200 Subject: [PATCH 316/489] relax CSR validation to support Kubernetes 1.27 (#1629) * relax CSR validation to support Kubernetes 1.27 * adjust unit tests --- pkg/controller/nodecsrapprover/controller.go | 5 +- .../nodecsrapprover/controller_test.go | 48 ------------------- 2 files changed, 1 insertion(+), 52 deletions(-) diff --git a/pkg/controller/nodecsrapprover/controller.go b/pkg/controller/nodecsrapprover/controller.go index de00fc679..92f3ad688 100644 --- a/pkg/controller/nodecsrapprover/controller.go +++ b/pkg/controller/nodecsrapprover/controller.go @@ -199,10 +199,7 @@ func (r *reconciler) validateCSRObject(csr *certificatesv1.CertificateSigningReq return "", fmt.Errorf("'%s' and/or '%s' are not in its groups", nodeGroup, authenticatedGroup) } - // Check are present usages matching allowed usages - if len(csr.Spec.Usages) != 3 { - return "", fmt.Errorf("there are no exactly three usages defined") - } + // Check that present usages matching allowed usages for _, usage := range csr.Spec.Usages { if !isUsageInUsageList(usage, allowedUsages) { return "", fmt.Errorf("usage %v is not in the list of allowed usages (%v)", usage, allowedUsages) diff --git a/pkg/controller/nodecsrapprover/controller_test.go b/pkg/controller/nodecsrapprover/controller_test.go index 18bbb5625..cb71c523f 100644 --- a/pkg/controller/nodecsrapprover/controller_test.go +++ b/pkg/controller/nodecsrapprover/controller_test.go @@ -289,54 +289,6 @@ func TestValidateCSRObject(t *testing.T) { nodeName: "", err: fmt.Errorf("'%s' and/or '%s' are not in its groups", nodeGroup, authenticatedGroup), }, - { - name: "validate csr with less than 3 usages", - csr: &certificatesv1.CertificateSigningRequest{ - ObjectMeta: metav1.ObjectMeta{ - Name: "csr", - Namespace: metav1.NamespaceSystem, - }, - Spec: certificatesv1.CertificateSigningRequestSpec{ - Request: []byte(testValidCSR), - Usages: []certificatesv1.KeyUsage{ - certificatesv1.UsageDigitalSignature, - certificatesv1.UsageKeyEncipherment, - }, - Username: "system:node:ip-172-31-114-48.eu-west-3.compute.internal", - Groups: []string{ - "system:nodes", - "system:authenticated", - }, - }, - }, - nodeName: "", - err: fmt.Errorf("there are no exactly three usages defined"), - }, - { - name: "validate csr with more than 3 usages", - csr: &certificatesv1.CertificateSigningRequest{ - ObjectMeta: metav1.ObjectMeta{ - Name: "csr", - Namespace: metav1.NamespaceSystem, - }, - Spec: certificatesv1.CertificateSigningRequestSpec{ - Request: []byte(testValidCSR), - Usages: []certificatesv1.KeyUsage{ - certificatesv1.UsageDigitalSignature, - certificatesv1.UsageKeyEncipherment, - certificatesv1.UsageServerAuth, - certificatesv1.UsageClientAuth, - }, - Username: "system:node:ip-172-31-114-48.eu-west-3.compute.internal", - Groups: []string{ - "system:nodes", - "system:authenticated", - }, - }, - }, - nodeName: "", - err: fmt.Errorf("there are no exactly three usages defined"), - }, { name: "validate csr with usages not matching expected usages", csr: &certificatesv1.CertificateSigningRequest{ From c92595c98f0d25d2dac6992c070d649e8e7496ed Mon Sep 17 00:00:00 2001 From: Georg Gadinger Date: Fri, 5 May 2023 19:28:18 +0200 Subject: [PATCH 317/489] Add support for OpenNebula as a cloud provider (#1450) * add cloud provider for opennebula Signed-off-by: Georg Gadinger * use a flatcar image in example opennebula machinedeployment Signed-off-by: Georg Gadinger * opennebula: update function signatures to include logger Signed-off-by: Georg Gadinger * opennebula: update after review Signed-off-by: Georg Gadinger * opennebula: add SET_HOSTNAME to context Signed-off-by: Georg Gadinger * opennebula: fix lints Signed-off-by: Georg Gadinger --------- Signed-off-by: Georg Gadinger --- .gitignore | 1 + docs/cloud-provider.md | 38 ++ examples/opennebula-machinedeployment.yaml | 70 +++ go.mod | 2 + go.sum | 5 + pkg/cloudprovider/provider.go | 4 + .../provider/opennebula/provider.go | 465 ++++++++++++++++++ .../provider/opennebula/types/types.go | 46 ++ pkg/providerconfig/types/types.go | 2 + test/e2e/provisioning/all_e2e_test.go | 38 ++ test/e2e/provisioning/helper.go | 8 + .../machinedeployment-opennebula.yaml | 55 +++ 12 files changed, 734 insertions(+) create mode 100644 examples/opennebula-machinedeployment.yaml create mode 100644 pkg/cloudprovider/provider/opennebula/provider.go create mode 100644 pkg/cloudprovider/provider/opennebula/types/types.go create mode 100644 test/e2e/provisioning/testdata/machinedeployment-opennebula.yaml diff --git a/.gitignore b/.gitignore index 8cc8d65e8..7c379f554 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ examples/*.srl .vscode .gitpod.yml cmd/machine-controller/__debug_bin +!pkg diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index 38295e68a..2d882241c 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -146,6 +146,44 @@ tags: tagKey: tagValue ``` +## OpenNebula + +### machine.spec.providerConfig.cloudProviderSpec + +```yaml +# XML-RPC endpoint of your OpenNebula installation +endpoint: "" +# your OpenNebula username +username: "" +# your OpenNebula password +password: "" + +# cpu (float64) +cpu: 1 +# vcpu +vcpu: 2 +# memory in MB +memory: 1024 + +# the name of the image to use, needs to be owned by the current user +image: "Amazon Linux 2" +# which datastore to use for the image +datastore: "" +# size of the disk in MB +diskSize: 51200 + +# network name, needs to be owned by the current user +network: "" + +# whether to enable the VNC console +enableVNC: true + +# optional key/value pairs to add to the VM template +vmTemplateExtra: + # useful for e.g. setting the placement attributes as defined in https://docs.opennebula.io/6.4/management_and_operations/references/template.html#template-placement-section + SCHED_REQUIREMENTS: 'RACK="G4"' +``` + ## Google Cloud Platform ### machine.spec.providerConfig.cloudProviderSpec diff --git a/examples/opennebula-machinedeployment.yaml b/examples/opennebula-machinedeployment.yaml new file mode 100644 index 000000000..0bc101881 --- /dev/null +++ b/examples/opennebula-machinedeployment.yaml @@ -0,0 +1,70 @@ +apiVersion: v1 +kind: Secret +metadata: + # If you change the namespace/name, you must also + # adjust the rbac rules + name: machine-controller-opennebula + namespace: kube-system +type: Opaque +stringData: + password: << ONE_PASSWORD >> +--- +apiVersion: "cluster.k8s.io/v1alpha1" +kind: MachineDeployment +metadata: + name: opennebula-machinedeployment + namespace: kube-system +spec: + paused: false + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + minReadySeconds: 0 + selector: + matchLabels: + foo: bar + template: + metadata: + labels: + foo: bar + spec: + providerSpec: + value: + sshPublicKeys: + - "<< YOUR_PUBLIC_KEY >>" + cloudProvider: "opennebula" + cloudProviderSpec: + endpoint: "<< ONE_ENDPOINT including '/RPC2' >>" + username: "<< ONE_USERNAME >>" + # If empty, can be set via ONE_PASSWORD env var + password: + secretKeyRef: + namespace: kube-system + name: machine-controller-opennebula + key: password + cpu: 1 + vcpu: 2 + memory: 1024 + + image: "flatcar-stable" + datastore: "<< YOUR_DATASTORE_NAME >>" + diskSize: 51200 # MB + + network: "<< YOUR_NETWORK_NAME >>" + + enableVNC: true + + # if you want to have more control over e.g. placement of the VM you can do this: + #vmTemplateExtra: + # SCHED_REQUIREMENTS: 'RACK="G4"' + operatingSystem: "flatcar" + operatingSystemSpec: + distUpgradeOnBoot: false + + # use cloud-init for flatcar as ignition doesn't know anything about OpenNebula yet + provisioningUtility: "cloud-init" + versions: + kubelet: 1.24.9 diff --git a/go.mod b/go.mod index 57dafd386..883941e89 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/BurntSushi/toml v1.2.1 github.com/Masterminds/semver/v3 v3.2.0 github.com/Masterminds/sprig/v3 v3.2.3 + github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20220908162715-b27302cc7db5 github.com/aliyun/alibaba-cloud-sdk-go v1.62.112 github.com/aws/aws-sdk-go-v2 v1.17.3 github.com/aws/aws-sdk-go-v2/config v1.18.7 @@ -130,6 +131,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kolo/xmlrpc v0.0.0-20190717152603-07c4ee3fd181 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/go.sum b/go.sum index 6c4c601dc..ef4c00621 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,8 @@ github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20220908162715-b27302cc7db5 h1:3SAiuS+PAdJHhDlTdnqseo9mZAcolUgDRS1PYEDUaFY= +github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20220908162715-b27302cc7db5/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= github.com/PaesslerAG/gval v1.2.1 h1:Ggwtej1xCyt1994VuDCSjycybIDo3duDCDghK/xc/A0= github.com/PaesslerAG/gval v1.2.1/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac= @@ -408,6 +410,7 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= @@ -468,6 +471,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/kolo/xmlrpc v0.0.0-20190717152603-07c4ee3fd181 h1:TrxPzApUukas24OMMVDUMlCs1XCExJtnGaDEiIAR4oQ= +github.com/kolo/xmlrpc v0.0.0-20190717152603-07c4ee3fd181/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= diff --git a/pkg/cloudprovider/provider.go b/pkg/cloudprovider/provider.go index 55546bf90..1c20a2a3e 100644 --- a/pkg/cloudprovider/provider.go +++ b/pkg/cloudprovider/provider.go @@ -33,6 +33,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/kubevirt" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/linode" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/nutanix" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/opennebula" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/openstack" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/scaleway" vcd "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vmwareclouddirector" @@ -112,6 +113,9 @@ var ( providerconfigtypes.CloudProviderVMwareCloudDirector: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return vcd.New(cvr) }, + providerconfigtypes.CloudProviderOpenNebula: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return opennebula.New(cvr) + }, } ) diff --git a/pkg/cloudprovider/provider/opennebula/provider.go b/pkg/cloudprovider/provider/opennebula/provider.go new file mode 100644 index 000000000..1108bb8b6 --- /dev/null +++ b/pkg/cloudprovider/provider/opennebula/provider.go @@ -0,0 +1,465 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package opennebula + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/OpenNebula/one/src/oca/go/src/goca" + "github.com/OpenNebula/one/src/oca/go/src/goca/schemas/shared" + "github.com/OpenNebula/one/src/oca/go/src/goca/schemas/vm" + "github.com/OpenNebula/one/src/oca/go/src/goca/schemas/vm/keys" + + "go.uber.org/zap" + + "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" + opennebulatypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/opennebula/types" + cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "github.com/kubermatic/machine-controller/pkg/providerconfig" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +type provider struct { + configVarResolver *providerconfig.ConfigVarResolver +} + +type CloudProviderSpec struct { + PassValidation bool `json:"passValidation"` +} + +const ( + machineUIDContextKey = "K8S_MACHINE_UID" +) + +// New returns a OpenNebula provider. +func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return &provider{configVarResolver: configVarResolver} +} + +type Config struct { + // Auth details + Username string + Password string + Endpoint string + + // Machine details + CPU *float64 + VCPU *int + Memory *int + Image string + Datastore string + DiskSize *int + Network string + EnableVNC bool + VMTemplateExtra map[string]string +} + +func getClient(config *Config) *goca.Client { + return goca.NewDefaultClient(goca.NewConfig(config.Username, config.Password, config.Endpoint)) +} + +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { + if provSpec.Value == nil { + return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") + } + + pconfig, err := providerconfigtypes.GetConfig(provSpec) + if err != nil { + return nil, nil, err + } + + rawConfig, err := opennebulatypes.GetConfig(*pconfig) + if err != nil { + return nil, nil, err + } + + c := Config{} + c.Username, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Username, "ONE_USERNAME") + if err != nil { + return nil, nil, fmt.Errorf("failed to get the value of \"username\" field, error = %w", err) + } + + c.Password, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Password, "ONE_PASSWORD") + if err != nil { + return nil, nil, fmt.Errorf("failed to get the value of \"password\" field, error = %w", err) + } + + c.Endpoint, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Endpoint, "ONE_ENDPOINT") + if err != nil { + return nil, nil, fmt.Errorf("failed to get the value of \"endpoint\" field, error = %w", err) + } + + c.CPU = rawConfig.CPU + + c.VCPU = rawConfig.VCPU + + c.Memory = rawConfig.Memory + + c.Image, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Image) + if err != nil { + return nil, nil, err + } + + c.Datastore, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Datastore) + if err != nil { + return nil, nil, err + } + + c.DiskSize = rawConfig.DiskSize + + c.Network, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Network) + if err != nil { + return nil, nil, err + } + + c.EnableVNC, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.EnableVNC) + if err != nil { + return nil, nil, err + } + + c.VMTemplateExtra = rawConfig.VMTemplateExtra + + return &c, pconfig, err +} + +func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { + _, pc, err := p.getConfig(spec.ProviderSpec) + if err != nil { + return fmt.Errorf("failed to parse config: %w", err) + } + + opennebulaCloudProviderSpec := CloudProviderSpec{} + if err = json.Unmarshal(pc.CloudProviderSpec.Raw, &opennebulaCloudProviderSpec); err != nil { + return fmt.Errorf("failed to parse cloud provider spec: %w", err) + } + + return nil +} + +func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string, error) { + return "", "", nil +} + +func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { + c, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), + } + } + + client := getClient(c) + + // build a template + tpl := vm.NewTemplate() + + // add extra template vars first + for key, value := range c.VMTemplateExtra { + tpl.Add(keys.Template(key), value) + } + + tpl.Add(keys.Name, machine.Spec.Name) + tpl.CPU(*c.CPU).Memory(*c.Memory).VCPU(*c.VCPU) + + disk := tpl.AddDisk() + disk.Add(shared.Image, c.Image) + disk.Add(shared.Datastore, c.Datastore) + disk.Add(shared.Size, *c.DiskSize) + + nic := tpl.AddNIC() + nic.Add(shared.Network, c.Network) + nic.Add(shared.Model, "virtio") + + if c.EnableVNC { + err = tpl.AddIOGraphic(keys.GraphicType, "VNC") + if err != nil { + return nil, fmt.Errorf("failed to add graphic type to iographic in template: %w", err) + } + err = tpl.AddIOGraphic(keys.Listen, "0.0.0.0") + if err != nil { + return nil, fmt.Errorf("failed to add listen address to iographic in template: %w", err) + } + } + + err = tpl.AddCtx(keys.NetworkCtx, "YES") + if err != nil { + return nil, fmt.Errorf("failed to add network to context in template: %w", err) + } + err = tpl.AddCtx(keys.SSHPubKey, "$USER[SSH_PUBLIC_KEY]") + if err != nil { + return nil, fmt.Errorf("failed to add SSH public key to context in template: %w", err) + } + + err = tpl.AddCtx(machineUIDContextKey, string(machine.UID)) + if err != nil { + return nil, fmt.Errorf("failed to add machine UID to context in template: %w", err) + } + err = tpl.AddCtx("USER_DATA", base64.StdEncoding.EncodeToString([]byte(userdata))) + if err != nil { + return nil, fmt.Errorf("failed to add user data to context in template: %w", err) + } + err = tpl.AddCtx("USER_DATA_ENCODING", "base64") + if err != nil { + return nil, fmt.Errorf("failed to add user data encoding to context in template: %w", err) + } + err = tpl.AddCtx("SET_HOSTNAME", machine.Spec.Name) + if err != nil { + return nil, fmt.Errorf("failed to add desired hostname to context in template: %w", err) + } + + controller := goca.NewController(client) + + // create VM from the generated template above + vmID, err := controller.VMs().Create(tpl.String(), false) + if err != nil { + return nil, fmt.Errorf("failed to create VM: %w", err) + } + + vm, err := controller.VM(vmID).Info(false) + if err != nil { + return nil, fmt.Errorf("failed to fetch VM information: %w", err) + } + + return &openNebulaInstance{vm}, nil +} + +func (p *provider) Cleanup(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { + instance, err := p.get(machine) + if err != nil { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { + return true, nil + } + return false, err + } + + c, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return false, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), + } + } + + client := getClient(c) + controller := goca.NewController(client) + + vmctrl := controller.VM(instance.vm.ID) + err = vmctrl.TerminateHard() + // ignore error of nonexistent machines by matching for "NO_EXISTS", the error string is something like "OpenNebula error [NO_EXISTS]: [one.vm.action] Error getting virtual machine [999914743]." + if err != nil && !strings.Contains(err.Error(), "NO_EXISTS") { + return false, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("failed to delete virtual machine, due to %v", err), + } + } + + return true, nil +} + +func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { + return p.get(machine) +} + +func (p *provider) get(machine *clusterv1alpha1.Machine) (*openNebulaInstance, error) { + c, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), + } + } + + client := getClient(c) + controller := goca.NewController(client) + + vmPool, err := controller.VMs().Info() + if err != nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("failed to list virtual machines, due to %v", err), + } + } + + // first collect all IDs, the vm infos in the vmPool don't contain the context which has the uid + var vmIDs []int + for _, vm := range vmPool.VMs { + if vm.Name != machine.Spec.Name { + continue + } + + vmIDs = append(vmIDs, vm.ID) + } + + // go over each vm that matches the name and check if the uid is the same + for _, vmID := range vmIDs { + vm, err := controller.VM(vmID).Info(false) + if err != nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("failed to get info for VM %v, due to %v", vmID, err), + } + } + + uid, err := vm.Template.GetCtx(machineUIDContextKey) + if err != nil { + // ignore errors like "key blabla not found" + continue + } + + if uid == string(machine.UID) { + return &openNebulaInstance{vm}, nil + } + } + + return nil, cloudprovidererrors.ErrInstanceNotFound +} + +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { + return spec, nil +} + +func (p *provider) MigrateUID(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { + c, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), + } + } + + instance, err := p.get(machine) + if err != nil { + return cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Failed to get instance, due to %v", err), + } + } + + client := getClient(c) + + // get current template + tpl := &instance.vm.Template + contextVector, err := tpl.GetVector(keys.ContextVec) + if err != nil { + return cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Failed to get VM template context vector, due to %v", err), + } + } + + // replace the old uid in context with the new one + contextVector.Del(machineUIDContextKey) + err = contextVector.AddPair(machineUIDContextKey, string(newUID)) + if err != nil { + return fmt.Errorf("failed to add the new machine UID to context in template: %w", err) + } + + // create a new template that only has the context vector in it so it gets properly replaced + tpl = vm.NewTemplate() + for _, pair := range contextVector.Pairs { + key := pair.XMLName.Local + value := pair.Value + err = tpl.AddCtx(keys.Context(key), value) + if err != nil { + return fmt.Errorf("failed to add %s to context in template: %w", key, err) + } + } + + // finally, update the VM template + controller := goca.NewController(client) + vmCtrl := controller.VM(instance.vm.ID) + err = vmCtrl.UpdateConf(tpl.String()) + if err != nil { + return cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Failed to update VM template, due to %v", err), + } + } + + return nil +} + +func (p *provider) MachineMetricsLabels(_ *clusterv1alpha1.Machine) (map[string]string, error) { + return map[string]string{}, nil +} + +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { + return nil +} + +type openNebulaInstance struct { + vm *vm.VM +} + +func (i *openNebulaInstance) Name() string { + return i.vm.Name +} + +func (i *openNebulaInstance) ID() string { + return strconv.Itoa(i.vm.ID) +} + +func (i *openNebulaInstance) ProviderID() string { + return "opennebula://" + strconv.Itoa(i.vm.ID) +} + +func (i *openNebulaInstance) Addresses() map[string]v1.NodeAddressType { + addresses := map[string]v1.NodeAddressType{} + + for _, nic := range i.vm.Template.GetNICs() { + ip, _ := nic.Get(shared.IP) + addresses[ip] = v1.NodeInternalIP + } + + return addresses +} + +func (i *openNebulaInstance) Status() instance.Status { + // state is the general state of the VM, lcmState is the state of the life-cycle manager of the VM + // lcmState is anything else other than LcmInit when the VM's state is Active + state, lcmState, _ := i.vm.State() + switch state { + case vm.Init, vm.Pending, vm.Hold: + return instance.StatusCreating + case vm.Active: + switch lcmState { + case vm.LcmInit, vm.Prolog, vm.Boot: + return instance.StatusCreating + case vm.Epilog: + return instance.StatusDeleting + default: + return instance.StatusRunning + } + case vm.Done: + return instance.StatusDeleted + default: + return instance.StatusUnknown + } +} diff --git a/pkg/cloudprovider/provider/opennebula/types/types.go b/pkg/cloudprovider/provider/opennebula/types/types.go new file mode 100644 index 000000000..6e69755fc --- /dev/null +++ b/pkg/cloudprovider/provider/opennebula/types/types.go @@ -0,0 +1,46 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" +) + +type RawConfig struct { + // Auth details + Username providerconfigtypes.ConfigVarString `json:"username,omitempty"` + Password providerconfigtypes.ConfigVarString `json:"password,omitempty"` + Endpoint providerconfigtypes.ConfigVarString `json:"endpoint,omitempty"` + + // Machine details + CPU *float64 `json:"cpu"` + VCPU *int `json:"vcpu"` + Memory *int `json:"memory"` + Image providerconfigtypes.ConfigVarString `json:"image"` + Datastore providerconfigtypes.ConfigVarString `json:"datastore"` + DiskSize *int `json:"diskSize"` + Network providerconfigtypes.ConfigVarString `json:"network"` + EnableVNC providerconfigtypes.ConfigVarBool `json:"enableVNC"` + VMTemplateExtra map[string]string `json:"vmTemplateExtra,omitempty"` +} + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 02a589fbb..18bd2010d 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -66,6 +66,7 @@ const ( CloudProviderScaleway CloudProvider = "scaleway" CloudProviderBaremetal CloudProvider = "baremetal" CloudProviderExternal CloudProvider = "external" + CloudProviderOpenNebula CloudProvider = "opennebula" ) var ( @@ -102,6 +103,7 @@ var ( CloudProviderScaleway, CloudProviderBaremetal, CloudProviderVultr, + CloudProviderOpenNebula, } ) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 85c796519..92d299bb8 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -79,6 +79,7 @@ const ( anexiaManifest = "./testdata/machinedeployment-anexia.yaml" nutanixManifest = "./testdata/machinedeployment-nutanix.yaml" vultrManifest = "./testdata/machinedeployment-vultr.yaml" + openNebulaManifest = "./testdata/machinedeployment-opennebula.yaml" ) const ( @@ -944,6 +945,43 @@ func TestNutanixProvisioningE2E(t *testing.T) { runScenarios(t, selector, params, nutanixManifest, fmt.Sprintf("nx-%s", *testRunIdentifier)) } +func TestOpenNebulaProvisioningE2E(t *testing.T) { + t.Parallel() + + oneEndpoint := os.Getenv("ONE_ENDPOINT") + oneUsername := os.Getenv("ONE_USERNAME") + onePassword := os.Getenv("ONE_PASSWORD") + + // required parameters + if oneEndpoint == "" || oneUsername == "" || onePassword == "" { + t.Fatal("unable to run test suite, all of ONE_ENDPOINT, ONE_USERNAME, and ONE_PASSWORD must be set!") + } + + // optional parameters + oneDatastore := os.Getenv("ONE_DATASTORE") + oneNetwork := os.Getenv("ONE_NETWORK") + + // set defaults for minione deployments + if oneDatastore == "" { + oneDatastore = "default" + } + + if oneNetwork == "" { + oneNetwork = "vnet" + } + + params := []string{ + fmt.Sprintf("<< ONE_ENDPOINT >>=%s", oneEndpoint), + fmt.Sprintf("<< ONE_USERNAME >>=%s", oneUsername), + fmt.Sprintf("<< ONE_PASSWORD >>=%s", onePassword), + fmt.Sprintf("<< ONE_DATASTORE_NAME >>=%s", oneDatastore), + fmt.Sprintf("<< ONE_NETWORK_NAME >>=%s", oneNetwork), + } + + selector := OsSelector("rockylinux", "flatcar") + runScenarios(t, selector, params, openNebulaManifest, fmt.Sprintf("one-%s", *testRunIdentifier)) +} + // TestUbuntuProvisioningWithUpgradeE2E will create an instance from an old Ubuntu 1604 // image and upgrade it prior to joining the cluster. func TestUbuntuProvisioningWithUpgradeE2E(t *testing.T) { diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 11a0e7c98..45720c76a 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -56,6 +56,11 @@ var ( string(providerconfigtypes.OperatingSystemRockyLinux): "machine-controller-e2e-rockylinux", } + openNebulaImages = map[string]string{ + string(providerconfigtypes.OperatingSystemFlatcar): "machine-controller-e2e-flatcar", + string(providerconfigtypes.OperatingSystemRockyLinux): "machine-controller-e2e-rockylinux", + } + vSphereOSImageTemplates = map[string]string{ string(providerconfigtypes.OperatingSystemCentOS): "kkp-centos-7", string(providerconfigtypes.OperatingSystemFlatcar): "kkp-flatcar-3139.2.0", @@ -261,6 +266,9 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar // only used by OpenStack scenarios scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_IMAGE >>=%s", openStackImages[testCase.osName])) + // only used by OpenNebula scenarios + scenarioParams = append(scenarioParams, fmt.Sprintf("<< ONE_IMAGE >>=%s", openNebulaImages[testCase.osName])) + // only use by vSphere scenarios scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_Image_Template >>=%s", vSphereOSImageTemplates[testCase.osName])) diff --git a/test/e2e/provisioning/testdata/machinedeployment-opennebula.yaml b/test/e2e/provisioning/testdata/machinedeployment-opennebula.yaml new file mode 100644 index 000000000..ba1a73d84 --- /dev/null +++ b/test/e2e/provisioning/testdata/machinedeployment-opennebula.yaml @@ -0,0 +1,55 @@ +apiVersion: "cluster.k8s.io/v1alpha1" +kind: MachineDeployment +metadata: + name: << MACHINE_NAME >> + namespace: kube-system +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + name: << MACHINE_NAME >> + template: + metadata: + labels: + name: << MACHINE_NAME >> + spec: + providerSpec: + value: + sshPublicKeys: + - "<< YOUR_PUBLIC_KEY >>" + cloudProvider: "opennebula" + cloudProviderSpec: + endpoint: "<< ONE_ENDPOINT >>" + username: "<< ONE_USERNAME >>" + password: "<< ONE_PASSWORD >>" + + cpu: 1 + vcpu: 2 + memory: 1024 + + image: "<< ONE_IMAGE >>" + datastore: "<< ONE_DATASTORE_NAME >>" + diskSize: 51200 # MB + + network: "<< ONE_NETWORK_NAME >>" + + enableVNC: true + operatingSystem: "<< OS_NAME >>" + operatingSystemSpec: + distUpgradeOnBoot: false + disableAutoUpdate: true + # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` + rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" + # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` + rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" + rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" + + # use cloud-init for flatcar as ignition doesn't know anything about OpenNebula yet + provisioningUtility: "cloud-init" + versions: + kubelet: "<< KUBERNETES_VERSION >>" From 28ee3c0c8fce71f93fe10e208911704f491836cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Wed, 10 May 2023 18:53:23 +0200 Subject: [PATCH 318/489] Update Go to 1.20.4 (#1633) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 16 ++++++++-------- .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 6 +++--- .prow/verify.yaml | 8 ++++---- Dockerfile | 2 +- Makefile | 2 +- hack/update-fixtures.sh | 2 +- 21 files changed, 40 insertions(+), 40 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 769eb3614..665516748 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -91,7 +91,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -118,7 +118,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index f4ecca9d1..da280c4d5 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index a424ee67d..a07194561 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 99f79b4aa..02afe5787 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index dc4a5a3fe..9203abd02 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -156,7 +156,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -187,7 +187,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -218,7 +218,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -249,7 +249,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 957807079..f3e02990c 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 44e09b822..fd84cc73b 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 6659d9802..f1c6f4ba6 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 21ee1756f..5631bf420 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 0200e89ee..7e7a9e959 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 4ae7aed97..642349925 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 802769760..f6bea60ff 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 44a2e5f31..149c649d9 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 0a73d28f0..e59fde74e 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 3897873b5..c1c359473 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 73e7f4a6f..25b63ceea 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 2099a030a..4c4df8177 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index ad2f48853..d0745f3e3 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.20.3 + - image: golang:1.20.4 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.20.3 + - image: golang:1.20.4 command: - make args: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.20.3 + - image: golang:1.20.4 command: - make args: diff --git a/Dockerfile b/Dockerfile index 2d0c45d20..7329a4f1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.20.3 +ARG GO_VERSION=1.20.4 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index e6f40e700..0f468ea32 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.20.3 +GO_VERSION ?= 1.20.4 GOOS ?= $(shell go env GOOS) diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 790c2e140..b34ec94b0 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.20.3 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.20.4 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... From b9a1fcc7317c8b267ac904798c329b249e0717da Mon Sep 17 00:00:00 2001 From: eiabea Date: Thu, 11 May 2023 16:57:43 +0200 Subject: [PATCH 319/489] Anexia Provider: Configure dns servers (#1637) * Add DNS servers to vm definition Signed-off-by: Manuel Zangl * Add ProviderConfig to reconcileContext Signed-off-by: Manuel Zangl * Test dns config Signed-off-by: Manuel Zangl --------- Signed-off-by: Manuel Zangl --- pkg/cloudprovider/provider/anexia/provider.go | 26 ++++++++++++++----- .../provider/anexia/provider_test.go | 19 ++++++++++++++ .../provider/anexia/reconcile_context.go | 12 +++++---- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 6b51e714a..8b2ca2065 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -96,17 +96,18 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * // ensure conditions are present on machine ensureConditions(&status) - config, _, err := p.getConfig(ctx, log, machine.Spec.ProviderSpec) + config, providerCfg, err := p.getConfig(ctx, log, machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("failed to get provider config: %w", err) } ctx = createReconcileContext(ctx, reconcileContext{ - Status: &status, - UserData: userdata, - Config: *config, - ProviderData: data, - Machine: machine, + Status: &status, + UserData: userdata, + Config: *config, + ProviderData: data, + ProviderConfig: providerCfg, + Machine: machine, }) _, client, err := getClient(config.Token) @@ -165,6 +166,19 @@ func provisionVM(ctx context.Context, log *zap.SugaredLogger, client anxclient.C vm.Script = base64.StdEncoding.EncodeToString([]byte(reconcileContext.UserData)) + for index, dnsServer := range reconcileContext.ProviderConfig.Network.DNS.Servers { + switch index { + case 0: + vm.DNS1 = dnsServer + case 1: + vm.DNS2 = dnsServer + case 2: + vm.DNS3 = dnsServer + case 3: + vm.DNS4 = dnsServer + } + } + // We generate a fresh SSH key but will never actually use it - we just want a valid public key to disable password authentication for our fresh VM. sshKey, err := ssh.NewKey() if err != nil { diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index 085aa70d2..b0faddd75 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -45,6 +45,7 @@ import ( cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -107,6 +108,11 @@ func TestAnexiaProvider(t *testing.T) { testhelper.AssertEquals(t, expectedJSON["cpu_performance_type"], jsonBody["cpu_performance_type"]) testhelper.AssertEquals(t, expectedJSON["hostname"], jsonBody["hostname"]) testhelper.AssertEquals(t, expectedJSON["memory_mb"], jsonBody["memory_mb"]) + + testhelper.AssertEquals(t, jsonBody["dns1"], "1.1.1.1") + testhelper.AssertEquals(t, jsonBody["dns2"], nil) + testhelper.AssertEquals(t, jsonBody["dns3"], "192.168.0.1") + testhelper.AssertEquals(t, jsonBody["dns4"], "192.168.0.2") testhelper.AssertEquals(t, expectedJSON["count"], jsonBody["count"]) expectedNetwork := expectedJSON["network"].([]jsonObject)[0] @@ -165,6 +171,19 @@ func TestAnexiaProvider(t *testing.T) { return nil }, }, + ProviderConfig: &providerconfigtypes.Config{ + Network: &providerconfigtypes.NetworkConfig{ + DNS: providerconfigtypes.DNSConfig{ + Servers: []string{ + "1.1.1.1", + "", + "192.168.0.1", + "192.168.0.2", + "192.168.0.3", + }, + }, + }, + }, }) err := provisionVM(ctx, log, client) diff --git a/pkg/cloudprovider/provider/anexia/reconcile_context.go b/pkg/cloudprovider/provider/anexia/reconcile_context.go index dea3577c8..2a14d8724 100644 --- a/pkg/cloudprovider/provider/anexia/reconcile_context.go +++ b/pkg/cloudprovider/provider/anexia/reconcile_context.go @@ -22,6 +22,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) type contextKey byte @@ -29,11 +30,12 @@ type contextKey byte const machineReconcileContextKey contextKey = 0 type reconcileContext struct { - Machine *v1alpha1.Machine - Status *anxtypes.ProviderStatus - UserData string - Config resolvedConfig - ProviderData *cloudprovidertypes.ProviderData + Machine *v1alpha1.Machine + Status *anxtypes.ProviderStatus + UserData string + Config resolvedConfig + ProviderData *cloudprovidertypes.ProviderData + ProviderConfig *providerconfigtypes.Config } func createReconcileContext(ctx context.Context, cc reconcileContext) context.Context { From bc2e86d29674094a24f9cc4d2e502c8aac658bd9 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Thu, 11 May 2023 19:19:24 +0200 Subject: [PATCH 320/489] remove DisableStrictZoneCheck from AWS 1.27 cloud-config (#1638) --- pkg/cloudprovider/provider/aws/types/cloudconfig.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/cloudprovider/provider/aws/types/cloudconfig.go b/pkg/cloudprovider/provider/aws/types/cloudconfig.go index 2fca4788e..40de37151 100644 --- a/pkg/cloudprovider/provider/aws/types/cloudconfig.go +++ b/pkg/cloudprovider/provider/aws/types/cloudconfig.go @@ -36,7 +36,9 @@ RoleARN={{ .Global.RoleARN | iniEscape }} KubernetesClusterID={{ .Global.KubernetesClusterID | iniEscape }} DisableSecurityGroupIngress={{ .Global.DisableSecurityGroupIngress }} ElbSecurityGroup={{ .Global.ElbSecurityGroup | iniEscape }} -DisableStrictZoneCheck={{ .Global.DisableStrictZoneCheck }} +{{- if .Global.DisableStrictZoneCheck }} +DisableStrictZoneCheck=true +{{- end }} {{- range .Global.NodeIPFamilies }} NodeIPFamilies={{ . | iniEscape}} {{- end }} @@ -57,8 +59,10 @@ type GlobalOpts struct { KubernetesClusterID string ElbSecurityGroup string DisableSecurityGroupIngress bool - DisableStrictZoneCheck bool - NodeIPFamilies []string + // DisableStrictZoneCheck has been removed in Kubernetes 1.27+. + // See https://github.com/kubernetes/cloud-provider-aws/pull/573 for more information. + DisableStrictZoneCheck bool + NodeIPFamilies []string } func CloudConfigToString(c *CloudConfig) (string, error) { From 571a7b7676f5a4a60b5ce1b8a636e12339ac3ba3 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 12 May 2023 20:31:32 +0500 Subject: [PATCH 321/489] E2E: Fix configuration for spot instances on AWS (#1643) Signed-off-by: Waleed Malik --- .../machinedeployment-aws-ebs-encryption-enabled.yaml | 4 ++-- .../testdata/machinedeployment-aws-spot-instances.yaml | 4 ++-- test/e2e/provisioning/testdata/machinedeployment-aws.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml index cb22438fb..9b5653b92 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml @@ -29,9 +29,9 @@ spec: accessKeyId: << AWS_ACCESS_KEY_ID >> secretAccessKey: << AWS_SECRET_ACCESS_KEY >> region: "eu-central-1" - availabilityZone: "eu-central-1a" + availabilityZone: "eu-central-1b" vpcId: "vpc-079f7648481a11e77" - instanceType: "t3a.small" + instanceType: "t2.medium" instanceProfile: "kubernetes-v1" diskSize: 50 diskType: "gp2" diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml index 2b93bc7aa..bce0b19c5 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml @@ -29,9 +29,9 @@ spec: accessKeyId: << AWS_ACCESS_KEY_ID >> secretAccessKey: << AWS_SECRET_ACCESS_KEY >> region: "eu-central-1" - availabilityZone: "eu-central-1a" + availabilityZone: "eu-central-1b" vpcId: "vpc-079f7648481a11e77" - instanceType: "t3a.small" + instanceType: "t2.medium" instanceProfile: "kubernetes-v1" diskSize: 50 diskType: "gp2" diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws.yaml index e9ef7ba47..6f7a7c2df 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws.yaml @@ -31,9 +31,9 @@ spec: assumeRoleARN: "<< AWS_ASSUME_ROLE_ARN >>" assumeRoleExternalID: "<< AWS_ASSUME_ROLE_EXTERNAL_ID >>" region: "eu-central-1" - availabilityZone: "eu-central-1a" + availabilityZone: "eu-central-1b" vpcId: "vpc-079f7648481a11e77" - instanceType: "t3a.small" + instanceType: "t2.medium" instanceProfile: "kubernetes-v1" diskSize: 50 diskType: "gp2" From c998d9047fc20e3f22a821298d723ca7144d5c74 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 12 May 2023 18:15:30 +0200 Subject: [PATCH 322/489] Bump github.com/docker/distribution (#1639) Bumps [github.com/docker/distribution](https://github.com/docker/distribution) from 2.8.1+incompatible to 2.8.2+incompatible. - [Release notes](https://github.com/docker/distribution/releases) - [Commits](https://github.com/docker/distribution/compare/v2.8.1...v2.8.2) --- updated-dependencies: - dependency-name: github.com/docker/distribution dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 883941e89..8418da9ad 100644 --- a/go.mod +++ b/go.mod @@ -99,7 +99,7 @@ require ( github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect diff --git a/go.sum b/go.sum index ef4c00621..ab7ec1186 100644 --- a/go.sum +++ b/go.sum @@ -205,8 +205,8 @@ github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= -github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= From d0f467240072c9ca29b14a49d79d252d92996219 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 12 May 2023 21:15:37 +0500 Subject: [PATCH 323/489] Deprecate user data plugins (#1642) * Deprecate user data plugins Signed-off-by: Waleed Malik * Refactored code Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index aadaf94ca..0d03a09be 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Kubermatic machine-controller +**Important Note: User data plugins for machine-controller are deprecated and will soon be removed. [Operating System Manager](https://github.com/kubermatic/operating-system-manager) is the successor of user data plugins. It's responsible for creating and managing the required configurations for worker nodes in a Kubernetes cluster with better modularity and extensibility. Please refer to [Operating System Manager][8] for more details.** + ## Table of Contents - [Kubermatic machine-controller](#kubermatic-machine-controller) @@ -53,7 +55,14 @@ Currently supported K8S versions are: ### Deploy the machine-controller -`make deploy` +- Install [cert-manager](https://cert-manager.io/) for generating certificates used by webhooks since they serve using HTTPS + +```terminal +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.2/cert-manager.yaml +``` + +- Run `kubectl apply -f examples/operating-system-manager.yaml` to deploy the operating-system-manager which is responsible for managing user data for worker machines. +- Run `make deploy` to deploy the machine-controller. ### Creating a machineDeployment @@ -147,3 +156,4 @@ See [the list of releases][7] to find out about feature changes. [5]: CONTRIBUTING.md [6]: Zenhub.md [7]: https://github.com/kubermatic/machine-controller/releases +[8]: https://docs.kubermatic.com/operatingsystemmanager \ No newline at end of file From 8e5884837711fb0fc6b568d734f09a7b809fc28e Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 16 May 2023 15:21:33 +0500 Subject: [PATCH 324/489] Switch to cert-manager for certificates (#1644) Signed-off-by: Waleed Malik --- Makefile | 41 +-------------------- README.md | 2 +- cmd/webhook/main.go | 4 +- examples/machine-controller.yaml | 41 ++++++++++++++------- hack/ci/setup-machine-controller-in-kind.sh | 22 +++++------ 5 files changed, 43 insertions(+), 67 deletions(-) diff --git a/Makefile b/Makefile index 0f468ea32..5c39b60be 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ machine-controller-userdata-%: cmd/userdata/% $(shell find cmd/userdata/$* pkg - github.com/kubermatic/machine-controller/cmd/$* .PHONY: clean -clean: clean-certs +clean: rm -f machine-controller \ webhook \ $(USERDATA_BIN) @@ -102,45 +102,6 @@ build-tests: go test -run nope ./... go test -tags e2e -run nope ./... -examples/ca-key.pem: - openssl genrsa -out examples/ca-key.pem 4096 - -examples/ca-cert.pem: examples/ca-key.pem - openssl req -x509 -new -nodes -key examples/ca-key.pem \ - -subj "/C=US/ST=CA/O=Acme/CN=k8s-machine-controller-ca" \ - -sha256 -days 10000 -out examples/ca-cert.pem - -examples/admission-key.pem: examples/ca-cert.pem - openssl genrsa -out examples/admission-key.pem 2048 - chmod 0600 examples/admission-key.pem - -examples/admission-cert.pem: examples/admission-key.pem - openssl req -new -sha256 \ - -key examples/admission-key.pem \ - -config examples/webhook-certificate.cnf -extensions v3_req \ - -out examples/admission.csr - openssl x509 -req \ - -sha256 \ - -days 10000 \ - -extensions v3_req \ - -extfile examples/webhook-certificate.cnf \ - -in examples/admission.csr \ - -CA examples/ca-cert.pem \ - -CAkey examples/ca-key.pem \ - -CAcreateserial \ - -out examples/admission-cert.pem - -clean-certs: - cd examples/ && rm -f admission.csr admission-cert.pem admission-key.pem ca-cert.pem ca-key.pem - -.PHONY: deploy -deploy: examples/admission-cert.pem - @cat examples/machine-controller.yaml \ - |sed "s/__admission_ca_cert__/$(shell cat examples/ca-cert.pem|$(BASE64_ENC))/g" \ - |sed "s/__admission_cert__/$(shell cat examples/admission-cert.pem|$(BASE64_ENC))/g" \ - |sed "s/__admission_key__/$(shell cat examples/admission-key.pem|$(BASE64_ENC))/g" \ - |kubectl apply -f - - .PHONY: check-dependencies check-dependencies: go mod verify diff --git a/README.md b/README.md index 0d03a09be..913b2a16b 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/ ``` - Run `kubectl apply -f examples/operating-system-manager.yaml` to deploy the operating-system-manager which is responsible for managing user data for worker machines. -- Run `make deploy` to deploy the machine-controller. +- Run `kubectl apply -f examples/machine-controller.yaml` to deploy the machine-controller. ### Creating a machineDeployment diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index f9c6d8239..d501bafaf 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -63,8 +63,8 @@ func main() { flag.StringVar(&opt.masterURL, "master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.") } flag.StringVar(&opt.admissionListenAddress, "listen-address", ":9876", "The address on which the MutatingWebhook will listen on") - flag.StringVar(&opt.admissionTLSCertPath, "tls-cert-path", "/tmp/cert/cert.pem", "The path of the TLS cert for the MutatingWebhook") - flag.StringVar(&opt.admissionTLSKeyPath, "tls-key-path", "/tmp/cert/key.pem", "The path of the TLS key for the MutatingWebhook") + flag.StringVar(&opt.admissionTLSCertPath, "tls-cert-path", "/tmp/cert/tls.crt", "The path of the TLS cert for the MutatingWebhook") + flag.StringVar(&opt.admissionTLSKeyPath, "tls-key-path", "/tmp/cert/tls.key", "The path of the TLS key for the MutatingWebhook") flag.StringVar(&opt.caBundleFile, "ca-bundle", "", "path to a file containing all PEM-encoded CA certificates (will be used instead of the host's certificates if set)") flag.StringVar(&opt.namespace, "namespace", "kubermatic", "The namespace where the webhooks will run") flag.StringVar(&opt.workerClusterKubeconfig, "worker-cluster-kubeconfig", "", "Path to kubeconfig of worker/user cluster where machines and machinedeployments exist. If not specified, value from --kubeconfig or in-cluster config will be used") diff --git a/examples/machine-controller.yaml b/examples/machine-controller.yaml index 957deca48..e3848f2fc 100644 --- a/examples/machine-controller.yaml +++ b/examples/machine-controller.yaml @@ -155,6 +155,28 @@ spec: jsonPath: .metadata.deletionTimestamp priority: 1 --- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: machine-controller-selfsigned-issuer + namespace: kube-system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: machine-controller-serving-cert + namespace: kube-system +spec: + dnsNames: + - "machine-controller-webhook.kube-system.svc" + - "machine-controller-webhook.kube-system.svc.cluster.local" + issuerRef: + kind: Issuer + name: machine-controller-selfsigned-issuer + secretName: machine-controller-webhook-server-cert +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: @@ -267,7 +289,8 @@ spec: - -listen-address=0.0.0.0:9876 volumeMounts: - name: machine-controller-admission-cert - mountPath: /tmp/cert + mountPath: /tmp/cert/ + readOnly: true livenessProbe: httpGet: path: /healthz @@ -284,16 +307,8 @@ spec: volumes: - name: machine-controller-admission-cert secret: - secretName: machine-controller-admission-cert ---- -apiVersion: v1 -kind: Secret -metadata: - name: machine-controller-admission-cert - namespace: kube-system -data: - "cert.pem": __admission_cert__ - "key.pem": __admission_key__ + defaultMode: 420 + secretName: machine-controller-webhook-server-cert --- apiVersion: v1 kind: Service @@ -602,6 +617,8 @@ apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: name: machinedeployments.machine-controller.kubermatic.io + annotations: + cert-manager.io/inject-ca-from: kube-system/machine-controller-serving-cert webhooks: - name: machinedeployments.machine-controller.kubermatic.io failurePolicy: Fail @@ -622,7 +639,6 @@ webhooks: namespace: kube-system name: machine-controller-webhook path: /machinedeployments - caBundle: __admission_ca_cert__ - name: machines.machine-controller.kubermatic.io failurePolicy: Fail sideEffects: None @@ -642,4 +658,3 @@ webhooks: namespace: kube-system name: machine-controller-webhook path: /machines - caBundle: __admission_ca_cert__ diff --git a/hack/ci/setup-machine-controller-in-kind.sh b/hack/ci/setup-machine-controller-in-kind.sh index 011644a4d..29cb6cf90 100755 --- a/hack/ci/setup-machine-controller-in-kind.sh +++ b/hack/ci/setup-machine-controller-in-kind.sh @@ -26,6 +26,16 @@ export OPERATING_SYSTEM_MANAGER="${OPERATING_SYSTEM_MANAGER:-true}" OSM_REPO_URL="${OSM_REPO_URL:-https://github.com/kubermatic/operating-system-manager.git}" OSM_REPO_TAG="${OSM_REPO_TAG:-main}" +# cert-manager is required by OSM for generating TLS Certificates +echodate "Installing cert-manager" +( + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.11.2/cert-manager.yaml + # Wait for cert-manager to be ready + kubectl -n cert-manager rollout status deploy/cert-manager + kubectl -n cert-manager rollout status deploy/cert-manager-cainjector + kubectl -n cert-manager rollout status deploy/cert-manager-webhook +) + # Build the Docker image for machine-controller beforeDockerBuild=$(nowms) @@ -57,23 +67,13 @@ if [ ! -f machine-controller-deployed ]; then # e2e tests logs are primarily read by humans, if ever sed -i 's/log-format=json/log-format=console/g' examples/machine-controller.yaml - make deploy + kubectl apply -f examples/machine-controller.yaml touch machine-controller-deployed protokol --kubeconfig "$KUBECONFIG" --flat --output "$ARTIFACTS/logs" --namespace kube-system 'machine-controller-*' > /dev/null 2>&1 & fi if [[ "$OPERATING_SYSTEM_MANAGER" == "true" ]]; then - # cert-manager is required by OSM for generating TLS Certificates - echodate "Installing cert-manager" - ( - kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.7.1/cert-manager.yaml - # Wait for cert-manager to be ready - kubectl -n cert-manager rollout status deploy/cert-manager - kubectl -n cert-manager rollout status deploy/cert-manager-cainjector - kubectl -n cert-manager rollout status deploy/cert-manager-webhook - ) - OSM_TMP_DIR=/tmp/osm echodate "Clone OSM respository" ( From 79bc81b9e1c0bec5da5c2b098fbfbf6d7ace72da Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Fri, 26 May 2023 14:12:38 +0200 Subject: [PATCH 325/489] Add documentation for community providers (#1631) * add documentation for community providers Signed-off-by: Marvin Beckers * correct note about CentOS 7 and Rocky Linux 8 Signed-off-by: Marvin Beckers --------- Signed-off-by: Marvin Beckers --- README.md | 71 +++++++++++++++++++++-------------- docs/cloud-provider.md | 6 +++ pkg/cloudprovider/provider.go | 20 +++++++--- 3 files changed, 62 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 913b2a16b..b1da95a64 100644 --- a/README.md +++ b/README.md @@ -7,35 +7,36 @@ - [Kubermatic machine-controller](#kubermatic-machine-controller) - [Table of Contents](#table-of-contents) - [Features](#features) - - [What works](#what-works) - - [Supported Kubernetes versions](#supported-kubernetes-versions) - - [What does not work](#what-does-not-work) + - [What Works](#what-works) + - [Supported Kubernetes Versions](#supported-kubernetes-versions) + - [Community Providers](#community-providers) + - [What doesn't Work](#what-doesnt-work) - [Quickstart](#quickstart) - - [Deploy the machine-controller](#deploy-the-machine-controller) - - [Creating a machineDeployment](#creating-a-machinedeployment) - - [Advanced usage](#advanced-usage) - - [Specifying the apiserver endpoint](#specifying-the-apiserver-endpoint) - - [CA-data](#ca-data) - - [Apiserver endpoint](#apiserver-endpoint) + - [Deploy machine-controller](#deploy-the-machine-controller) + - [Creating a MachineDeployment](#creating-a-machinedeployment) + - [Advanced Usage](#advanced-usage) + - [Specifying the Apiserver Endpoint](#specifying-the-apiserver-endpoint) + - [CA Data](#ca-data) + - [Apiserver Endpoint](#apiserver-endpoint) - [Example cluster-info ConfigMap](#example-cluster-info-configmap) - [Development](#development) - [Testing](#testing) - - [Unittests](#unittests) - - [End-to-End locally](#end-to-end-locally) + - [Unit Tests](#unit-tests) + - [End-to-End Locally](#end-to-end-locally) - [Troubleshooting](#troubleshooting) - [Contributing](#contributing) - - [Before you start](#before-you-start) - - [Pull requests](#pull-requests) + - [Before You Start](#before-you-start) + - [Pull Requests](#pull-requests) - [Changelog](#changelog) ## Features -### What works +### What Works -- Creation of worker nodes on AWS, Digitalocean, Openstack, Azure, Google Cloud Platform, Nutanix, VMWare Cloud Director, VMWare Vsphere, Linode, Hetzner cloud and Kubevirt (experimental) -- Using Ubuntu, Flatcar or CentOS 7 distributions ([not all distributions work on all providers](/docs/operating-system.md)) +- Creation of worker nodes on AWS, Digitalocean, Openstack, Azure, Google Cloud Platform, Nutanix, VMWare Cloud Director, VMWare vSphere, Hetzner Cloud and Kubevirt +- Using Ubuntu, Flatcar, CentOS 7 or Rocky Linux 8 distributions ([not all distributions work on all providers](/docs/operating-system.md)) -### Supported Kubernetes versions +### Supported Kubernetes Versions machine-controller tries to follow the Kubernetes version [support policy](https://kubernetes.io/docs/setup/release/version-skew-policy/) as close as possible. @@ -47,13 +48,25 @@ Currently supported K8S versions are: - 1.25 - 1.24 -## What does not work +### Community Providers + +Some cloud providers implemented in machine-controller have been graciously contributed by community members. Those cloud providers are not part of the automated end-to-end +tests run by the machine-controller developers and thus, their status cannot be guaranteed. The machine-controller developers assume that they are functional, but can only +offer limited support for new features or bugfixes in those providers. + +The current list of community providers is: + +- Linode +- Vultr +- OpenNebula + +## What Doesn't Work - Master creation (Not planned at the moment) ## Quickstart -### Deploy the machine-controller +### Deploy machine-controller - Install [cert-manager](https://cert-manager.io/) for generating certificates used by webhooks since they serve using HTTPS @@ -64,16 +77,16 @@ kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/ - Run `kubectl apply -f examples/operating-system-manager.yaml` to deploy the operating-system-manager which is responsible for managing user data for worker machines. - Run `kubectl apply -f examples/machine-controller.yaml` to deploy the machine-controller. -### Creating a machineDeployment +### Creating a `MachineDeployment` ```bash # edit examples/$cloudprovider-machinedeployment.yaml & create the machineDeployment kubectl create -f examples/$cloudprovider-machinedeployment.yaml ``` -## Advanced usage +## Advanced Usage -### Specifying the apiserver endpoint +### Specifying the Apiserver Endpoint By default the controller looks for a `cluster-info` ConfigMap within the `kube-public` Namespace. If one is found which contains a minimal kubeconfig (kubeadm cluster have them by default), this kubeconfig will be used for the node bootstrapping. @@ -84,11 +97,11 @@ The kubeconfig only needs to contain two things: If no ConfigMap can be found: -### CA-data +### CA Data -The CA will be loaded from the passed kubeconfig when running outside the cluster or from `/var/run/secrets/kubernetes.io/serviceaccount/ca.crt` when running inside the cluster. +The Certificate Authority (CA) will be loaded from the passed kubeconfig when running outside the cluster or from `/var/run/secrets/kubernetes.io/serviceaccount/ca.crt` when running inside the cluster. -### Apiserver endpoint +### Apiserver Endpoint The first endpoint from the kubernetes endpoints will be taken. `kubectl get endpoints kubernetes -o yaml` @@ -119,11 +132,11 @@ data: ### Testing -#### Unittests +#### Unit Tests Simply run `make test-unit` -#### End-to-End locally +#### End-to-End Locally **_[WIP]_** @@ -135,13 +148,13 @@ If you encounter issues [file an issue][1] or talk to us on the [#kubermatic cha Thanks for taking the time to join our community and start contributing! -### Before you start +### Before You Start - Please familiarize yourself with the [Code of Conduct][4] before contributing. - See [CONTRIBUTING.md][5] for instructions on the developer certificate of origin that we require. - Read how [we're using ZenHub][6] for project and roadmap planning -### Pull requests +### Pull Requests - We welcome pull requests. Feel free to dig through the [issues][1] and jump in. diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index 2d882241c..74edb7a4a 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -148,6 +148,8 @@ tags: ## OpenNebula +**Note:** This is a [community provider](../README.md#community-providers). + ### machine.spec.providerConfig.cloudProviderSpec ```yaml @@ -233,6 +235,8 @@ labels: ## Linode +**Note:** This is a [community provider](../README.md#community-providers). + ### machine.spec.providerConfig.cloudProviderSpec ```yaml # your linode token @@ -361,6 +365,8 @@ Refer to the [VSphere](./vsphere.md#provider-configuration) specific documentati ## Vultr +**Note:** This is a [community provider](../README.md#community-providers). + ### machine.spec.providerConfig.cloudProviderSpec ```yaml apiKey: "<< VULTR_API_KEY >>" diff --git a/pkg/cloudprovider/provider.go b/pkg/cloudprovider/provider.go index 1c20a2a3e..28161700c 100644 --- a/pkg/cloudprovider/provider.go +++ b/pkg/cloudprovider/provider.go @@ -66,9 +66,6 @@ var ( providerconfigtypes.CloudProviderHetzner: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return hetzner.New(cvr) }, - providerconfigtypes.CloudProviderLinode: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return linode.New(cvr) - }, providerconfigtypes.CloudProviderVsphere: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return vsphere.New(cvr) }, @@ -78,9 +75,6 @@ var ( providerconfigtypes.CloudProviderEquinixMetal: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return equinixmetal.New(cvr) }, - providerconfigtypes.CloudProviderVultr: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { - return vultr.New(cvr) - }, // NB: This is explicitly left to allow old Packet machines to be deleted. // We can handle those machines in the same way as Equinix Metal machines // because there are no API changes. @@ -113,6 +107,17 @@ var ( providerconfigtypes.CloudProviderVMwareCloudDirector: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return vcd.New(cvr) }, + } + + // communityProviders holds a map of cloud providers that have been implemented by community members and + // contributed to machine-controller. They are not end-to-end tested by the machine-controller development team. + communityProviders = map[providerconfigtypes.CloudProvider]func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider{ + providerconfigtypes.CloudProviderLinode: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return linode.New(cvr) + }, + providerconfigtypes.CloudProviderVultr: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return vultr.New(cvr) + }, providerconfigtypes.CloudProviderOpenNebula: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return opennebula.New(cvr) }, @@ -124,5 +129,8 @@ func ForProvider(p providerconfigtypes.CloudProvider, cvr *providerconfig.Config if p, found := providers[p]; found { return NewValidationCacheWrappingCloudProvider(p(cvr)), nil } + if p, found := communityProviders[p]; found { + return NewValidationCacheWrappingCloudProvider(p(cvr)), nil + } return nil, ErrProviderNotFound } From 28336b0918c795d6d934c59b5f50b853d4946c0a Mon Sep 17 00:00:00 2001 From: eiabea Date: Tue, 30 May 2023 09:01:06 +0200 Subject: [PATCH 326/489] Fix empty network config handling (#1646) * Check if providerConfig is set Add test for empty network provider config Signed-off-by: Manuel Zangl * Refactor provision vm test cases Create test cases to reduce code duplication Move common types and functions to helper_test Signed-off-by: Manuel Zangl --------- Signed-off-by: Manuel Zangl --- .../provider/anexia/helper_test.go | 87 +++++++ pkg/cloudprovider/provider/anexia/provider.go | 23 +- .../provider/anexia/provider_test.go | 212 ++++++++---------- 3 files changed, 191 insertions(+), 131 deletions(-) diff --git a/pkg/cloudprovider/provider/anexia/helper_test.go b/pkg/cloudprovider/provider/anexia/helper_test.go index 38c3a37ef..c6256fa20 100644 --- a/pkg/cloudprovider/provider/anexia/helper_test.go +++ b/pkg/cloudprovider/provider/anexia/helper_test.go @@ -20,15 +20,27 @@ import ( "encoding/json" "testing" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/gophercloud/gophercloud/testhelper" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" + cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" "k8s.io/apimachinery/pkg/runtime" ) +type jsonObject = map[string]interface{} + +type ProvisionVMTestCase struct { + ReconcileContext reconcileContext + AssertJSONBody func(jsonBody jsonObject) +} + type ConfigTestCase struct { Config anxtypes.RawConfig Error error @@ -67,3 +79,78 @@ func newConfigVarString(str string) types.ConfigVarString { Value: str, } } + +// this generates a full config and allows hooking into it to e.g. remove a value. +func hookableConfig(hook func(*anxtypes.RawConfig)) anxtypes.RawConfig { + config := anxtypes.RawConfig{ + CPUs: 1, + + Memory: 2, + + Disks: []anxtypes.RawDisk{ + {Size: 5, PerformanceType: newConfigVarString("ENT6")}, + }, + + Token: newConfigVarString("test-token"), + VlanID: newConfigVarString("test-vlan"), + LocationID: newConfigVarString("test-location"), + TemplateID: newConfigVarString("test-template-id"), + } + + if hook != nil { + hook(&config) + } + + return config +} + +// this generates a full reconcileContext with some default values and allows hooking into it to e.g. remove/overwrite a value. +func hookableReconcileContext(locationID string, templateID string, hook func(*reconcileContext)) reconcileContext { + context := reconcileContext{ + Machine: &v1alpha1.Machine{ + ObjectMeta: metav1.ObjectMeta{Name: "TestMachine"}, + }, + Status: &anxtypes.ProviderStatus{}, + UserData: "", + Config: resolvedConfig{ + VlanID: "VLAN-ID", + LocationID: locationID, + TemplateID: templateID, + Disks: []resolvedDisk{ + { + RawDisk: anxtypes.RawDisk{ + Size: 5, + }, + }, + }, + RawConfig: anxtypes.RawConfig{ + CPUs: 5, + Memory: 5, + }, + }, + ProviderData: &cloudprovidertypes.ProviderData{ + Update: func(m *clusterv1alpha1.Machine, mods ...cloudprovidertypes.MachineModifier) error { + return nil + }, + }, + ProviderConfig: &providerconfigtypes.Config{ + Network: &providerconfigtypes.NetworkConfig{ + DNS: providerconfigtypes.DNSConfig{ + Servers: []string{ + "1.1.1.1", + "", + "192.168.0.1", + "192.168.0.2", + "192.168.0.3", + }, + }, + }, + }, + } + + if hook != nil { + hook(&context) + } + + return context +} diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 8b2ca2065..842954be6 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -166,16 +166,19 @@ func provisionVM(ctx context.Context, log *zap.SugaredLogger, client anxclient.C vm.Script = base64.StdEncoding.EncodeToString([]byte(reconcileContext.UserData)) - for index, dnsServer := range reconcileContext.ProviderConfig.Network.DNS.Servers { - switch index { - case 0: - vm.DNS1 = dnsServer - case 1: - vm.DNS2 = dnsServer - case 2: - vm.DNS3 = dnsServer - case 3: - vm.DNS4 = dnsServer + providerCfg := reconcileContext.ProviderConfig + if providerCfg.Network != nil { + for index, dnsServer := range providerCfg.Network.DNS.Servers { + switch index { + case 0: + vm.DNS1 = dnsServer + case 1: + vm.DNS2 = dnsServer + case 2: + vm.DNS3 = dnsServer + case 3: + vm.DNS4 = dnsServer + } } } diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index b0faddd75..ab2b40ba1 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" "net/http/httptest" "net/url" @@ -41,7 +42,6 @@ import ( "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" @@ -66,6 +66,7 @@ func TestAnexiaProvider(t *testing.T) { a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID-OLD-BUILD", Name: testTemplateName, Build: "b01"}) a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID", Name: testTemplateName, Build: "b02"}) a.FakeExisting(&vspherev1.Template{Identifier: "WRONG-TEMPLATE-NAME", Name: "Wrong Template Name", Build: "b02"}) + a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID-NO-NETWORK-CONFIG", Name: "no-network-config", Build: "b03"}) t.Cleanup(func() { testhelper.TeardownHTTP() @@ -74,6 +75,61 @@ func TestAnexiaProvider(t *testing.T) { t.Run("Test provision VM", func(t *testing.T) { t.Parallel() + + testCases := []ProvisionVMTestCase{ + { + // Provision a generic VM with some custom dns entries + ReconcileContext: hookableReconcileContext("LOCATION-ID", "TEMPLATE-ID", func(rc *reconcileContext) { + rc.ProviderConfig = &providerconfigtypes.Config{ + Network: &providerconfigtypes.NetworkConfig{ + DNS: providerconfigtypes.DNSConfig{ + Servers: []string{ + "1.1.1.1", + "", + "192.168.0.1", + "192.168.0.2", + "192.168.0.3", + }, + }, + }, + } + }), + AssertJSONBody: func(jsonBody jsonObject) { + testhelper.AssertEquals(t, jsonBody["cpu_performance_type"], "performance") + testhelper.AssertEquals(t, jsonBody["hostname"], "TestMachine") + testhelper.AssertEquals(t, jsonBody["memory_mb"], json.Number("5")) + + testhelper.AssertEquals(t, jsonBody["dns1"], "1.1.1.1") + _, exists := jsonBody["dns2"] + testhelper.AssertEquals(t, exists, false) + testhelper.AssertEquals(t, jsonBody["dns3"], "192.168.0.1") + testhelper.AssertEquals(t, jsonBody["dns4"], "192.168.0.2") + + networkArray := jsonBody["network"].([]interface{}) + networkObject := networkArray[0].(jsonObject) + testhelper.AssertEquals(t, networkObject["vlan"], "VLAN-ID") + testhelper.AssertEquals(t, networkObject["nic_type"], "vmxnet3") + testhelper.AssertEquals(t, networkObject["ips"].([]interface{})[0], "8.8.8.8") + }, + }, + { + // Provision a VM without any ProviderConfig + ReconcileContext: hookableReconcileContext("LOCATION-ID", "TEMPLATE-ID-NO-NETWORK-CONFIG", func(rc *reconcileContext) { + rc.ProviderConfig = &providerconfigtypes.Config{} + }), + AssertJSONBody: func(jsonBody jsonObject) { + _, exists := jsonBody["dns1"] + testhelper.AssertEquals(t, exists, false) + _, exists = jsonBody["dns2"] + testhelper.AssertEquals(t, exists, false) + _, exists = jsonBody["dns3"] + testhelper.AssertEquals(t, exists, false) + _, exists = jsonBody["dns4"] + testhelper.AssertEquals(t, exists, false) + }, + }, + } + testhelper.Mux.HandleFunc("/api/ipam/v1/address/reserve/ip/count.json", func(writer http.ResponseWriter, request *http.Request) { err := json.NewEncoder(writer).Encode(address.ReserveRandomSummary{ Data: []address.ReservedIP{ @@ -86,108 +142,46 @@ func TestAnexiaProvider(t *testing.T) { testhelper.AssertNoErr(t, err) }) - testhelper.Mux.HandleFunc("/api/vsphere/v1/provisioning/vm.json/LOCATION-ID/templates/TEMPLATE-ID", func(writer http.ResponseWriter, request *http.Request) { - testhelper.TestMethod(t, request, http.MethodPost) - type jsonObject = map[string]interface{} - expectedJSON := map[string]interface{}{ - "cpu_performance_type": "performance", - "hostname": "TestMachine", - "memory_mb": json.Number("5"), - "network": []jsonObject{ - { - "vlan": "VLAN-ID", - "nic_type": "vmxnet3", - "ips": []interface{}{"8.8.8.8"}, - }, - }, - } - var jsonBody jsonObject - decoder := json.NewDecoder(request.Body) - decoder.UseNumber() - testhelper.AssertNoErr(t, decoder.Decode(&jsonBody)) - testhelper.AssertEquals(t, expectedJSON["cpu_performance_type"], jsonBody["cpu_performance_type"]) - testhelper.AssertEquals(t, expectedJSON["hostname"], jsonBody["hostname"]) - testhelper.AssertEquals(t, expectedJSON["memory_mb"], jsonBody["memory_mb"]) - - testhelper.AssertEquals(t, jsonBody["dns1"], "1.1.1.1") - testhelper.AssertEquals(t, jsonBody["dns2"], nil) - testhelper.AssertEquals(t, jsonBody["dns3"], "192.168.0.1") - testhelper.AssertEquals(t, jsonBody["dns4"], "192.168.0.2") - testhelper.AssertEquals(t, expectedJSON["count"], jsonBody["count"]) - - expectedNetwork := expectedJSON["network"].([]jsonObject)[0] - bodyNetwork := jsonBody["network"].([]interface{})[0].(jsonObject) - testhelper.AssertEquals(t, expectedNetwork["vlan"], bodyNetwork["vlan"]) - testhelper.AssertEquals(t, expectedNetwork["nic_type"], bodyNetwork["nic_type"]) - testhelper.AssertEquals(t, expectedNetwork["ips"].([]interface{})[0], bodyNetwork["ips"].([]interface{})[0]) - - err := json.NewEncoder(writer).Encode(vm.ProvisioningResponse{ - Progress: 100, - Errors: nil, - Identifier: "TEST-IDENTIFIER", - Queued: false, + for _, testCase := range testCases { + templateID := testCase.ReconcileContext.Config.TemplateID + locationID := testCase.ReconcileContext.Config.LocationID + + testhelper.Mux.HandleFunc(fmt.Sprintf("/api/vsphere/v1/provisioning/vm.json/%s/templates/%s", locationID, templateID), func(writer http.ResponseWriter, request *http.Request) { + testhelper.TestMethod(t, request, http.MethodPost) + var jsonBody jsonObject + decoder := json.NewDecoder(request.Body) + decoder.UseNumber() + testhelper.AssertNoErr(t, decoder.Decode(&jsonBody)) + + testCase.AssertJSONBody(jsonBody) + + err := json.NewEncoder(writer).Encode(vm.ProvisioningResponse{ + Progress: 100, + Errors: nil, + Identifier: templateID, + Queued: false, + }) + testhelper.AssertNoErr(t, err) }) - testhelper.AssertNoErr(t, err) - }) - testhelper.Mux.HandleFunc("/api/vsphere/v1/provisioning/progress.json/TEST-IDENTIFIER", func(writer http.ResponseWriter, request *http.Request) { - testhelper.TestMethod(t, request, http.MethodGet) + testhelper.Mux.HandleFunc(fmt.Sprintf("/api/vsphere/v1/provisioning/progress.json/%s", templateID), func(writer http.ResponseWriter, request *http.Request) { + testhelper.TestMethod(t, request, http.MethodGet) - err := json.NewEncoder(writer).Encode(progress.Progress{ - TaskIdentifier: "TEST-IDENTIFIER", - Queued: false, - Progress: 100, - VMIdentifier: "VM-IDENTIFIER", - Errors: nil, + err := json.NewEncoder(writer).Encode(progress.Progress{ + TaskIdentifier: templateID, + Queued: false, + Progress: 100, + VMIdentifier: "VM-IDENTIFIER", + Errors: nil, + }) + testhelper.AssertNoErr(t, err) }) - testhelper.AssertNoErr(t, err) - }) - providerStatus := anxtypes.ProviderStatus{} - ctx := createReconcileContext(context.Background(), reconcileContext{ - Machine: &v1alpha1.Machine{ - ObjectMeta: metav1.ObjectMeta{Name: "TestMachine"}, - }, - Status: &providerStatus, - UserData: "", - Config: resolvedConfig{ - VlanID: "VLAN-ID", - LocationID: "LOCATION-ID", - TemplateID: "TEMPLATE-ID", - Disks: []resolvedDisk{ - { - RawDisk: anxtypes.RawDisk{ - Size: 5, - }, - }, - }, - RawConfig: anxtypes.RawConfig{ - CPUs: 5, - Memory: 5, - }, - }, - ProviderData: &cloudprovidertypes.ProviderData{ - Update: func(m *clusterv1alpha1.Machine, mods ...cloudprovidertypes.MachineModifier) error { - return nil - }, - }, - ProviderConfig: &providerconfigtypes.Config{ - Network: &providerconfigtypes.NetworkConfig{ - DNS: providerconfigtypes.DNSConfig{ - Servers: []string{ - "1.1.1.1", - "", - "192.168.0.1", - "192.168.0.2", - "192.168.0.3", - }, - }, - }, - }, - }) + ctx := createReconcileContext(context.Background(), testCase.ReconcileContext) - err := provisionVM(ctx, log, client) - testhelper.AssertNoErr(t, err) + err := provisionVM(ctx, log, client) + testhelper.AssertNoErr(t, err) + } }) t.Run("Test resolve template", func(t *testing.T) { @@ -298,30 +292,6 @@ func TestAnexiaProvider(t *testing.T) { }) } -// this generates a full config and allows hooking into it to e.g. remove a value. -func hookableConfig(hook func(*anxtypes.RawConfig)) anxtypes.RawConfig { - config := anxtypes.RawConfig{ - CPUs: 1, - - Memory: 2, - - Disks: []anxtypes.RawDisk{ - {Size: 5, PerformanceType: newConfigVarString("ENT6")}, - }, - - Token: newConfigVarString("test-token"), - VlanID: newConfigVarString("test-vlan"), - LocationID: newConfigVarString("test-location"), - TemplateID: newConfigVarString("test-template-id"), - } - - if hook != nil { - hook(&config) - } - - return config -} - func TestValidate(t *testing.T) { t.Parallel() From bfd2fd7cf54150606466e64831e366b923336e0d Mon Sep 17 00:00:00 2001 From: Marcin Franczyk Date: Thu, 1 Jun 2023 14:29:44 +0200 Subject: [PATCH 327/489] Allow external CCMs to handle node objects before MC (#1645) * Run VMs with run strategy 'Once' Signed-off-by: Marcin Franczyk * Fix KubeVirt unit tests Signed-off-by: Marcin Franczyk * Allow external CCMs to handle node objects before MC. Otherwise we have a race condition between MC and CCM. Both try to check status of instances at cloud provider. If MC reconciles instances first then kubelet will reuse the old node object which is problematic in case of IP change. Signed-off-by: Marcin Franczyk * Update doc of handleNodeFailuresWithExternalCCM func Signed-off-by: Marcin Franczyk * Use ceph instead of portworx for KubeVirt tests Signed-off-by: Marcin Franczyk --------- Signed-off-by: Marcin Franczyk --- go.mod | 1 + go.sum | 2 + .../provider/kubevirt/provider.go | 17 +--- .../kubevirt/testdata/affinity-no-values.yaml | 2 +- .../provider/kubevirt/testdata/affinity.yaml | 2 +- .../kubevirt/testdata/custom-local-disk.yaml | 2 +- .../kubevirt/testdata/http-image-source.yaml | 2 +- .../instancetype-preference-custom.yaml | 2 +- .../instancetype-preference-standard.yaml | 2 +- .../kubevirt/testdata/nominal-case.yaml | 2 +- .../kubevirt/testdata/pvc-image-source.yaml | 2 +- .../kubevirt/testdata/secondary-disks.yaml | 2 +- .../testdata/topologyspreadconstraints.yaml | 2 +- pkg/controller/machine/controller.go | 98 +++++++++++++++---- .../testdata/machinedeployment-kubevirt.yaml | 2 +- 15 files changed, 96 insertions(+), 44 deletions(-) diff --git a/go.mod b/go.mod index 8418da9ad..551e32cfb 100644 --- a/go.mod +++ b/go.mod @@ -58,6 +58,7 @@ require ( k8s.io/apiextensions-apiserver v0.26.4 k8s.io/apimachinery v0.26.4 k8s.io/client-go v12.0.0+incompatible + k8s.io/cloud-provider v0.26.4 k8s.io/klog v1.0.0 k8s.io/kubelet v0.26.4 k8s.io/utils v0.0.0-20230209194617-a36077c30491 diff --git a/go.sum b/go.sum index ab7ec1186..eeb0a4ab2 100644 --- a/go.sum +++ b/go.sum @@ -1235,6 +1235,8 @@ k8s.io/apimachinery v0.26.4 h1:rZccKdBLg9vP6J09JD+z8Yr99Ce8gk3Lbi9TCx05Jzs= k8s.io/apimachinery v0.26.4/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= k8s.io/client-go v0.26.4 h1:/7P/IbGBuT73A+G97trf44NTPSNqvuBREpOfdLbHvD4= k8s.io/client-go v0.26.4/go.mod h1:6qOItWm3EwxJdl/8p5t7FWtWUOwyMdA8N9ekbW4idpI= +k8s.io/cloud-provider v0.26.4 h1:mqN4vhC4mRoMi+ujI92ImkIOuYS7ZS55FvXB10d6Wp4= +k8s.io/cloud-provider v0.26.4/go.mod h1:F9xY0PvBuZDuGIHOM28dNiPLHxQnWfsiUuCSUikHevo= k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/component-base v0.26.4 h1:Bg2xzyXNKL3eAuiTEu3XE198d6z22ENgFgGQv2GGOUk= diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index e5aba1041..a2ba1f58c 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -445,18 +445,6 @@ func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clust return nil, cloudprovidererrors.ErrInstanceNotFound } - if virtualMachineInstance.Status.Phase == kubevirtv1.Failed || - // The VMI enters phase succeeded if someone issues a kubectl - // delete pod on the virt-launcher pod it runs in - virtualMachineInstance.Status.Phase == kubevirtv1.Succeeded { - // The pod got deleted, delete the VMI and return ErrNotFound so the VMI - // will get recreated - if err := sigClient.Delete(ctx, virtualMachineInstance); err != nil { - return nil, fmt.Errorf("failed to delete failed VMI %s: %w", machine.Name, err) - } - return nil, cloudprovidererrors.ErrInstanceNotFound - } - return &kubeVirtServer{vmi: *virtualMachineInstance}, nil } @@ -633,6 +621,8 @@ func (p *provider) newVirtualMachine(ctx context.Context, c *Config, pc *provide return nil, fmt.Errorf("could not compute a random MAC address") } + runStrategyOnce := kubevirtv1.RunStrategyOnce + virtualMachine := &kubevirtv1.VirtualMachine{ ObjectMeta: metav1.ObjectMeta{ Name: machine.Name, @@ -640,7 +630,7 @@ func (p *provider) newVirtualMachine(ctx context.Context, c *Config, pc *provide Labels: labels, }, Spec: kubevirtv1.VirtualMachineSpec{ - Running: utilpointer.Bool(true), + RunStrategy: &runStrategyOnce, Instancetype: c.Instancetype, Preference: c.Preference, Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ @@ -692,7 +682,6 @@ func (p *provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *c if !kerrors.IsNotFound(err) { return false, fmt.Errorf("failed to get VirtualMachineInstance %s: %w", machine.Name, err) } - // VMI is gone return true, nil } diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml index f304b2a6e..7d01a42c7 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml @@ -25,7 +25,7 @@ spec: source: http: url: http://x.y.z.t/ubuntu.img - running: true + runStrategy: Once template: metadata: labels: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml index 309e30d7b..a28aded56 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml @@ -25,7 +25,7 @@ spec: source: http: url: http://x.y.z.t/ubuntu.img - running: true + runStrategy: Once template: metadata: labels: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml index e10d07c62..b77494b63 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml @@ -25,7 +25,7 @@ spec: pvc: namespace: ns name: dvname - running: true + runStrategy: Once template: metadata: creationTimestamp: null diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml index 3bf7731e7..caef1ed52 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml @@ -24,7 +24,7 @@ spec: source: http: url: http://x.y.z.t/ubuntu.img - running: true + runStrategy: Once template: metadata: creationTimestamp: null diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml index 060171c4b..ba699302e 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml @@ -25,7 +25,7 @@ spec: source: http: url: http://x.y.z.t/ubuntu.img - running: true + runStrategy: Once instancetype: kind: VirtualMachineClusterInstancetype name: custom-it diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml index b56229f69..1f54c87a7 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml @@ -24,7 +24,7 @@ spec: source: http: url: http://x.y.z.t/ubuntu.img - running: true + runStrategy: Once instancetype: kind: VirtualMachineInstancetype name: standard-it diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml index 5dddb4b52..24afa309c 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml @@ -24,7 +24,7 @@ spec: source: http: url: http://x.y.z.t/ubuntu.img - running: true + runStrategy: Once template: metadata: creationTimestamp: null diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml index 2cafabbd7..978213ec3 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml @@ -25,7 +25,7 @@ spec: pvc: namespace: ns name: dvname - running: true + runStrategy: Once template: metadata: creationTimestamp: null diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml index 576172c93..0ce57a4b7 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml @@ -50,7 +50,7 @@ spec: source: http: url: http://x.y.z.t/ubuntu.img - running: true + runStrategy: Once template: metadata: creationTimestamp: null diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml index a36f6b84d..363460724 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml @@ -24,7 +24,7 @@ spec: source: http: url: http://x.y.z.t/ubuntu.img - running: true + runStrategy: Once template: metadata: creationTimestamp: null diff --git a/pkg/controller/machine/controller.go b/pkg/controller/machine/controller.go index 67a537497..d952ec113 100644 --- a/pkg/controller/machine/controller.go +++ b/pkg/controller/machine/controller.go @@ -67,6 +67,8 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/tools/reference" "k8s.io/client-go/util/retry" + + ccmapi "k8s.io/cloud-provider/api" "sigs.k8s.io/controller-runtime/pkg/client" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -434,9 +436,8 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, mach // step 2: check if a user requested to delete the machine if machine.DeletionTimestamp != nil { - return r.deleteMachine(ctx, log, prov, providerConfig.CloudProvider, machine) - // deleteResult, deleteErr := r.deleteMachine(ctx, log, prov, providerConfig.CloudProvider, machine) - // if client.IgnoreNotFound(deleteErr) + skipEviction := false + return r.deleteMachine(ctx, log, prov, providerConfig.CloudProvider, machine, skipEviction) } // Step 3: Essentially creates an instance for the given machine. @@ -471,8 +472,10 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, mach return nil, fmt.Errorf("failed to set nodeReady condition on machine: %w", err) } } else { - // Node is not ready anymore? Maybe it got deleted - return r.ensureInstanceExistsForMachine(ctx, nodeLog, prov, machine, userdataPlugin, providerConfig) + if r.nodeSettings.ExternalCloudProvider { + return r.handleNodeFailuresWithExternalCCM(ctx, log, prov, providerConfig, node, machine) + } + return r.ensureInstanceExistsForMachine(ctx, log, prov, machine, userdataPlugin, providerConfig) } // case 3.3: if the node exists make sure if it has labels and taints attached to it. @@ -534,7 +537,7 @@ func (r *Reconciler) shouldCleanupVolumes(ctx context.Context, log *zap.SugaredL func (r *Reconciler) shouldEvict(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine) (bool, error) { // If the deletion got triggered a few hours ago, skip eviction. // We assume here that the eviction is blocked by misconfiguration or a misbehaving kubelet and/or controller-runtime - if time.Since(machine.DeletionTimestamp.Time) > r.skipEvictionAfter { + if machine.DeletionTimestamp != nil && time.Since(machine.DeletionTimestamp.Time) > r.skipEvictionAfter { log.Infow("Skipping eviction since the deletion got triggered too long ago", "threshold", r.skipEvictionAfter) return false, nil } @@ -583,10 +586,24 @@ func (r *Reconciler) shouldEvict(ctx context.Context, log *zap.SugaredLogger, ma } // deleteMachine makes sure that an instance has gone in a series of steps. -func (r *Reconciler) deleteMachine(ctx context.Context, log *zap.SugaredLogger, prov cloudprovidertypes.Provider, providerName providerconfigtypes.CloudProvider, machine *clusterv1alpha1.Machine) (*reconcile.Result, error) { - shouldEvict, err := r.shouldEvict(ctx, log, machine) - if err != nil { - return nil, err +func (r *Reconciler) deleteMachine( + ctx context.Context, + log *zap.SugaredLogger, + prov cloudprovidertypes.Provider, + providerName providerconfigtypes.CloudProvider, + machine *clusterv1alpha1.Machine, + skipEviction bool, +) (*reconcile.Result, error) { + var ( + shouldEvict bool + err error + ) + + if !skipEviction { + shouldEvict, err = r.shouldEvict(ctx, log, machine) + if err != nil { + return nil, err + } } shouldCleanUpVolumes, err := r.shouldCleanupVolumes(ctx, log, machine, providerName) if err != nil { @@ -691,7 +708,6 @@ func (r *Reconciler) deleteCloudProviderInstance(ctx context.Context, log *zap.S message := fmt.Sprintf("%v. Please manually delete %s finalizer from the machine object.", err, FinalizerDeleteInstance) return nil, r.updateMachineErrorIfTerminalError(machine, common.DeleteMachineError, message, err, "failed to delete machine at cloud provider") } - if !completelyGone { // As the instance is not completely gone yet, we need to recheck in a few seconds. return &reconcile.Result{RequeueAfter: deletionRetryWaitPeriod}, nil @@ -1051,14 +1067,6 @@ func (r *Reconciler) ensureNodeLabelsAnnotationsAndTaints(ctx context.Context, n modifiers = append(modifiers, f(AnnotationAutoscalerIdentifier, autoscalerAnnotationValue)) } - taintExists := func(node *corev1.Node, taint corev1.Taint) bool { - for _, t := range node.Spec.Taints { - if t.MatchTaint(&taint) { - return true - } - } - return false - } for _, t := range machine.Spec.Taints { if !taintExists(node, t) { f := func(t corev1.Taint) func(*corev1.Node) { @@ -1181,6 +1189,15 @@ func findNodeByProviderID(instance instance.Instance, provider providerconfigtyp return nil } +func taintExists(node *corev1.Node, taint corev1.Taint) bool { + for _, t := range node.Spec.Taints { + if t.MatchTaint(&taint) { + return true + } + } + return false +} + func (r *Reconciler) ReadinessChecks(ctx context.Context) map[string]healthcheck.Check { return map[string]healthcheck.Check{ "valid-info-kubeconfig": func() error { @@ -1242,3 +1259,46 @@ func (r *Reconciler) updateNode(ctx context.Context, node *corev1.Node, modifier return r.client.Update(ctx, node) }) } + +// handleNodeFailuresWithExternalCCM reacts to node status discovery of CCM's node lifecycle controller. +// If an instance at cloud provider is not found then it waits till CCM deletes node objects, that allows: +// - create a new instance at cloud provider +// - initialize a new node object - the object should not be reused between instance creation +// for example, instance foo that got deleted and recreated should initialize a completely new node object +// instead of reusing the old one as it can cause problems to update node's metadata, like IP address. +// +// If node is shut-down it allows MC to react accordingly to specific cloud provider requirements, those are: +// - wait for node to become online again or +// - delete a machine which cannot be recovered +func (r *Reconciler) handleNodeFailuresWithExternalCCM( + ctx context.Context, + log *zap.SugaredLogger, + prov cloudprovidertypes.Provider, + provConfig *providerconfigtypes.Config, + node *corev1.Node, + machine *clusterv1alpha1.Machine, +) (*reconcile.Result, error) { + taintShutdown := corev1.Taint{ + Key: ccmapi.TaintNodeShutdown, + Effect: corev1.TaintEffectNoSchedule, + } + + _, err := prov.Get(ctx, log, machine, r.providerData) + if err != nil { + if cloudprovidererrors.IsNotFound(err) { + log.Info("The node does not have corresponding instance, waiting for CCM to delete it") + return &reconcile.Result{RequeueAfter: deletionRetryWaitPeriod}, nil + } + return nil, err + } else if taintExists(node, taintShutdown) { + switch provConfig.CloudProvider { + case providerconfigtypes.CloudProviderKubeVirt: + log.Infof("Deleting a shut-down machine %q that cannot recover", machine.Name) + skipEviction := true + return r.deleteMachine(ctx, log, prov, providerconfigtypes.CloudProviderKubeVirt, machine, skipEviction) + } + } + + log.Debug("Waiting for a node to become %q", corev1.NodeReady) + return &reconcile.Result{RequeueAfter: deletionRetryWaitPeriod}, err +} diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index 90a46bfc1..d97169adf 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -38,7 +38,7 @@ spec: primaryDisk: osImage: http://image-repo.kube-system.svc/images/<< KUBEVIRT_OS_IMAGE >>.img size: "25Gi" - storageClassName: px-csi-db + storageClassName: rook-ceph-block dnsPolicy: "None" dnsConfig: nameservers: From 2fefe6e3996f1279dccf5007e7b798585030485e Mon Sep 17 00:00:00 2001 From: Matthias Osthues Date: Mon, 5 Jun 2023 10:40:29 +0200 Subject: [PATCH 328/489] Add vmware anti-affinity for machine sets (#1647) * Add vmware anti-affinity for machins * fix linting * add to test * fix comments and add validation * query cluster if required for validation * fix test cluster name * increase timeout and move update call after vm is deleted --- ...e-datastore-cluster-machinedeployment.yaml | 4 + examples/vsphere-machinedeployment.yaml | 4 + .../provider/vsphere/provider.go | 39 ++++ .../provider/vsphere/provider_test.go | 2 + pkg/cloudprovider/provider/vsphere/rule.go | 196 ++++++++++++++++++ .../provider/vsphere/types/types.go | 14 +- ...edeployment-vsphere-datastore-cluster.yaml | 2 + ...chinedeployment-vsphere-resource-pool.yaml | 2 + .../machinedeployment-vsphere-static-ip.yaml | 4 +- .../testdata/machinedeployment-vsphere.yaml | 2 + 10 files changed, 262 insertions(+), 7 deletions(-) create mode 100644 pkg/cloudprovider/provider/vsphere/rule.go diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml index 9018db197..f693fdbf6 100644 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ b/examples/vsphere-datastore-cluster-machinedeployment.yaml @@ -57,6 +57,10 @@ spec: datastoreCluster: datastorecluster1 # Can also be set via the env var 'VSPHERE_ALLOW_INSECURE' on the machine-controller allowInsecure: true + # Cluster to configure vm anti affinity rules + cluster: cl-1 + # Automatically create anti affinity rules for machines + vmAntiAffinity: true cpus: 2 memoryMB: 2048 # Optional: Resize the root disk to this size. Must be bigger than the existing size diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index 59ad3072a..aadf49d33 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -57,6 +57,10 @@ spec: datastore: datastore1 # Can also be set via the env var 'VSPHERE_ALLOW_INSECURE' on the machine-controller allowInsecure: true + # Cluster to configure vm anti affinity rules + cluster: cl-1 + # Automatically create anti affinity rules for machines + vmAntiAffinity: true cpus: 2 memoryMB: 2048 # Optional: Resize the root disk to this size. Must be bigger than the existing size diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 62efd7871..ceefc9c99 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -23,6 +23,7 @@ import ( "net/url" "os" "strings" + "sync" "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/object" @@ -48,6 +49,7 @@ import ( type provider struct { configVarResolver *providerconfig.ConfigVarResolver + mutex sync.Mutex } // New returns a VSphere provider. @@ -64,11 +66,13 @@ type Config struct { Password string VSphereURL string Datacenter string + Cluster string Folder string ResourcePool string Datastore string DatastoreCluster string AllowInsecure bool + VMAntiAffinity bool CPUs int32 MemoryMB int64 DiskSizeGB *int64 @@ -164,6 +168,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, nil, err } + c.Cluster, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Cluster) + if err != nil { + return nil, nil, nil, err + } + c.Folder, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Folder) if err != nil { return nil, nil, nil, err @@ -189,6 +198,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, nil, err } + c.VMAntiAffinity, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.VMAntiAffinity) + if err != nil { + return nil, nil, nil, err + } + c.CPUs = rawConfig.CPUs c.MemoryMB = rawConfig.MemoryMB c.DiskSizeGB = rawConfig.DiskSizeGB @@ -285,6 +299,17 @@ func (p *provider) Validate(ctx context.Context, log *zap.SugaredLogger, spec cl return err } } + + if config.VMAntiAffinity { + if config.Cluster == "" { + return fmt.Errorf("cluster is required for vm anti affinity") + } + _, err = session.Finder.ClusterComputeResource(ctx, config.Cluster) + if err != nil { + return fmt.Errorf("failed to get cluster %q, %w", config.Cluster, err) + } + } + return nil } @@ -341,6 +366,13 @@ func (p *provider) create(ctx context.Context, log *zap.SugaredLogger, machine * return nil, fmt.Errorf("failed to attach tags: %w", err) } + if config.VMAntiAffinity { + machineSetName := machine.Name[:strings.LastIndex(machine.Name, "-")] + if err := p.createOrUpdateVMAntiAffinityRule(ctx, session, machineSetName, config); err != nil { + return nil, fmt.Errorf("failed to add VM to anti affinity rule: %w", err) + } + } + if pc.OperatingSystem != providerconfigtypes.OperatingSystemFlatcar { localUserdataIsoFilePath, err := generateLocalUserdataISO(userdata, machine.Spec.Name) if err != nil { @@ -458,6 +490,13 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine return false, fmt.Errorf("failed to destroy vm %s: %w", virtualMachine.Name(), err) } + if config.VMAntiAffinity { + machineSetName := machine.Name[:strings.LastIndex(machine.Name, "-")] + if err := p.createOrUpdateVMAntiAffinityRule(ctx, session, machineSetName, config); err != nil { + return false, fmt.Errorf("failed to add VM to anti affinity rule: %w", err) + } + } + if pc.OperatingSystem != providerconfigtypes.OperatingSystemFlatcar { filemanager := datastore.NewFileManager(session.Datacenter, false) diff --git a/pkg/cloudprovider/provider/vsphere/provider_test.go b/pkg/cloudprovider/provider/vsphere/provider_test.go index e457a5506..463c9173e 100644 --- a/pkg/cloudprovider/provider/vsphere/provider_test.go +++ b/pkg/cloudprovider/provider/vsphere/provider_test.go @@ -48,6 +48,8 @@ func (v vsphereProviderSpecConf) rawProviderSpec(t *testing.T) []byte { "cloudProvider": "vsphere", "cloudProviderSpec": { "allowInsecure": false, + "vmAntiAffinity": true, + "cluster": "DC0_C0", "cpus": 1, "datacenter": "DC0", {{- if .Datastore }} diff --git a/pkg/cloudprovider/provider/vsphere/rule.go b/pkg/cloudprovider/provider/vsphere/rule.go new file mode 100644 index 000000000..e4dcf2e8b --- /dev/null +++ b/pkg/cloudprovider/provider/vsphere/rule.go @@ -0,0 +1,196 @@ +/* +Copyright 2023 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vsphere + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "time" + + "github.com/aws/smithy-go/ptr" + "github.com/vmware/govmomi/find" + "github.com/vmware/govmomi/object" + "github.com/vmware/govmomi/vim25/mo" + "github.com/vmware/govmomi/vim25/types" +) + +// createOrUpdateVMAntiAffinityRule creates or updates an anti affinity rule with the name in the given cluster. +// VMs are attached to the rule based on their folder path and name prefix in vsphere. +// A minimum of two VMs is required. +func (p *provider) createOrUpdateVMAntiAffinityRule(ctx context.Context, session *Session, name string, config *Config) error { + p.mutex.Lock() + defer p.mutex.Unlock() + + cluster, err := session.Finder.ClusterComputeResource(ctx, config.Cluster) + if err != nil { + return err + } + + vmsInFolder, err := session.Finder.VirtualMachineList(ctx, strings.Join([]string{config.Folder, "*"}, "/")) + if err != nil { + if errors.Is(err, &find.NotFoundError{}) { + return removeVMAntiAffinityRule(ctx, session, config.Cluster, name) + } + return err + } + + var ruleVMRef []types.ManagedObjectReference + for _, vm := range vmsInFolder { + if strings.HasPrefix(vm.Name(), name) { + ruleVMRef = append(ruleVMRef, vm.Reference()) + } + } + + // minimum of two vms required + if len(ruleVMRef) < 2 { + return removeVMAntiAffinityRule(ctx, session, config.Cluster, name) + } + + info, err := findClusterAntiAffinityRuleByName(ctx, cluster, name) + if err != nil { + return err + } + + operation := types.ArrayUpdateOperationEdit + + //create new rule + if info == nil { + info = &types.ClusterAntiAffinityRuleSpec{ + ClusterRuleInfo: types.ClusterRuleInfo{ + Enabled: ptr.Bool(true), + Mandatory: ptr.Bool(false), + Name: name, + UserCreated: ptr.Bool(true), + }, + } + operation = types.ArrayUpdateOperationAdd + } + + info.Vm = ruleVMRef + spec := &types.ClusterConfigSpecEx{ + RulesSpec: []types.ClusterRuleSpec{ + { + ArrayUpdateSpec: types.ArrayUpdateSpec{ + Operation: operation, + }, + Info: info, + }, + }, + } + + task, err := cluster.Reconfigure(ctx, spec, true) + if err != nil { + return err + } + + err = task.Wait(ctx) + if err != nil { + return err + } + + return waitForRule(ctx, cluster, info) +} + +// waitForRule checks periodically the vsphere api for the ClusterAntiAffinityRule and returns error if the rule was not found after a timeout. +func waitForRule(ctx context.Context, cluster *object.ClusterComputeResource, rule *types.ClusterAntiAffinityRuleSpec) error { + timeout := time.NewTimer(10 * time.Second) + ticker := time.NewTicker(500 * time.Millisecond) + defer timeout.Stop() + defer ticker.Stop() + + for { + select { + case <-timeout.C: + + info, err := findClusterAntiAffinityRuleByName(ctx, cluster, rule.Name) + if err != nil { + return err + } + + if !reflect.DeepEqual(rule, info) { + return fmt.Errorf("expected anti affinity changes not found in vsphere") + } + case <-ticker.C: + info, err := findClusterAntiAffinityRuleByName(ctx, cluster, rule.Name) + if err != nil { + return err + } + + if reflect.DeepEqual(rule, info) { + return nil + } + } + } +} + +// removeVMAntiAffinityRule removes an anti affinity rule with the name in the given cluster. +func removeVMAntiAffinityRule(ctx context.Context, session *Session, clusterPath string, name string) error { + cluster, err := session.Finder.ClusterComputeResource(ctx, clusterPath) + if err != nil { + return err + } + + info, err := findClusterAntiAffinityRuleByName(ctx, cluster, name) + if err != nil { + return err + } + + // no rule found + if info == nil { + return nil + } + + spec := &types.ClusterConfigSpecEx{ + RulesSpec: []types.ClusterRuleSpec{ + { + ArrayUpdateSpec: types.ArrayUpdateSpec{ + Operation: types.ArrayUpdateOperationRemove, + RemoveKey: info.Key, + }, + }, + }, + } + + task, err := cluster.Reconfigure(ctx, spec, true) + if err != nil { + return err + } + return task.Wait(ctx) +} + +func findClusterAntiAffinityRuleByName(ctx context.Context, cluster *object.ClusterComputeResource, name string) (*types.ClusterAntiAffinityRuleSpec, error) { + var props mo.ClusterComputeResource + if err := cluster.Properties(ctx, cluster.Reference(), nil, &props); err != nil { + return nil, err + } + + var info *types.ClusterAntiAffinityRuleSpec + for _, clusterRuleInfo := range props.ConfigurationEx.(*types.ClusterConfigInfoEx).Rule { + if clusterRuleInfo.GetClusterRuleInfo().Name == name { + if vmAffinityRuleInfo, ok := clusterRuleInfo.(*types.ClusterAntiAffinityRuleSpec); ok { + info = vmAffinityRuleInfo + break + } + return nil, fmt.Errorf("rule name %s in cluster %q is not a VM anti-affinity rule", name, cluster.Name()) + } + } + + return info, nil +} diff --git a/pkg/cloudprovider/provider/vsphere/types/types.go b/pkg/cloudprovider/provider/vsphere/types/types.go index 62331ee76..dded33405 100644 --- a/pkg/cloudprovider/provider/vsphere/types/types.go +++ b/pkg/cloudprovider/provider/vsphere/types/types.go @@ -30,7 +30,8 @@ type RawConfig struct { VSphereURL providerconfigtypes.ConfigVarString `json:"vsphereURL"` Datacenter providerconfigtypes.ConfigVarString `json:"datacenter"` - // Cluster is a noop field, it's not used anywhere but left here intentionally for backward compatibility purposes + // Cluster defines the cluster to use in vcenter. + // Only needed for vm anti affinity. Cluster providerconfigtypes.ConfigVarString `json:"cluster"` Folder providerconfigtypes.ConfigVarString `json:"folder"` @@ -40,11 +41,12 @@ type RawConfig struct { DatastoreCluster providerconfigtypes.ConfigVarString `json:"datastoreCluster"` Datastore providerconfigtypes.ConfigVarString `json:"datastore"` - CPUs int32 `json:"cpus"` - MemoryMB int64 `json:"memoryMB"` - DiskSizeGB *int64 `json:"diskSizeGB,omitempty"` - Tags []Tag `json:"tags,omitempty"` - AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` + CPUs int32 `json:"cpus"` + MemoryMB int64 `json:"memoryMB"` + DiskSizeGB *int64 `json:"diskSizeGB,omitempty"` + Tags []Tag `json:"tags,omitempty"` + AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` + VMAntiAffinity providerconfigtypes.ConfigVarBool `json:"vmAntiAffinity"` } // Tag represents vsphere tag. diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml index b87ea6000..b85102700 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml @@ -33,6 +33,8 @@ spec: folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically + cluster: Kubermatic + vmAntiAffinity: true datastoreCluster: 'dsc-1' cpus: 2 MemoryMB: 2048 diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml index ca48b60fb..54a7a345c 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml @@ -34,6 +34,8 @@ spec: password: << VSPHERE_PASSWORD >> datastore: 'vsan' resourcePool: 'e2e-resource-pool' + cluster: Kubermatic + vmAntiAffinity: true cpus: 2 MemoryMB: 2048 diskSizeGB: << DISK_SIZE >> diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml index 2114d717a..548ab6786 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml @@ -32,7 +32,9 @@ spec: datacenter: 'Hamburg' folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> - # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically + # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically3 + cluster: Kubermatic + vmAntiAffinity: true datastore: vsan cpus: 2 MemoryMB: 2048 diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml index f81548e04..c53ba3f43 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml @@ -33,6 +33,8 @@ spec: folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically + cluster: Kubermatic + vmAntiAffinity: true datastore: vsan cpus: 2 MemoryMB: 4096 From f0b1db6704fb2e880ef388187e619f5c8182acc0 Mon Sep 17 00:00:00 2001 From: pprzekwas Date: Fri, 9 Jun 2023 16:01:19 +0200 Subject: [PATCH 329/489] Update Golang to 1.20.5 (#1659) * Update golang outside .prow dir * Update golang in verify.yaml * Update golang inside .prow dir --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 16 ++++++++-------- .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 6 +++--- .prow/verify.yaml | 8 ++++---- Dockerfile | 2 +- Makefile | 2 +- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- 22 files changed, 41 insertions(+), 41 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 665516748..ad796bc18 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -91,7 +91,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -118,7 +118,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index da280c4d5..920e15064 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index a07194561..a458e8333 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 02afe5787..8695fe2cc 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 9203abd02..b784d50e2 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -156,7 +156,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -187,7 +187,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -218,7 +218,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -249,7 +249,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index f3e02990c..0e99aaf01 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index fd84cc73b..f1b16e888 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index f1c6f4ba6..e27aa4a98 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 5631bf420..d71c90168 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 7e7a9e959..2fbe5ea24 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 642349925..3da41e259 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index f6bea60ff..9163a84c6 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 149c649d9..876ad33ec 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index e59fde74e..b504a0ecf 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index c1c359473..ea094e84e 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 25b63ceea..11d74385a 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 4c4df8177..35becb03d 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index d0745f3e3..033e183e0 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.20.4 + - image: golang:1.20.5 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.20.4 + - image: golang:1.20.5 command: - make args: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-3 + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.20.4 + - image: golang:1.20.5 command: - make args: diff --git a/Dockerfile b/Dockerfile index 7329a4f1c..07f311f23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.20.4 +ARG GO_VERSION=1.20.5 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index 5c39b60be..a409c2cf5 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.20.4 +GO_VERSION ?= 1.20.5 GOOS ?= $(shell go env GOOS) diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index b34ec94b0..437cc1e4e 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.20.4 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.20.5 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 74ea661bd..bffa632f1 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-1 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 containerize ./hack/verify-licenses.sh go mod vendor From 02c047ff0d219f95c4def3cfc14ca9456b7fdce5 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 13 Jun 2023 11:25:19 +0500 Subject: [PATCH 330/489] Update CRDs for OSM (#1662) Signed-off-by: Waleed Malik --- examples/operating-system-manager.yaml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index 86e945774..7fc7a471e 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -195,6 +195,7 @@ spec: - baremetal - external - vmware-cloud-director + - opennebula type: string spec: description: Spec represents the os/image reference in the supported @@ -211,7 +212,6 @@ spec: - rhel - centos - ubuntu - - sles - amzn2 - rockylinux type: string @@ -348,6 +348,14 @@ spec: type: string type: array type: object + provisioningUtility: + default: cloud-init + description: ProvisioningUtility used for configuring the worker node. + Defaults to cloud-init. + enum: + - cloud-init + - ignition + type: string required: - bootstrapConfig - cloudProvider @@ -605,7 +613,6 @@ spec: - rhel - centos - ubuntu - - sles - amzn2 - rockylinux type: string @@ -809,6 +816,14 @@ spec: type: object type: array type: object + provisioningUtility: + default: cloud-init + description: ProvisioningUtility used for configuring the worker node. + Defaults to cloud-init. + enum: + - cloud-init + - ignition + type: string supportedCloudProviders: description: SupportedCloudProviders represent the cloud providers that support the given operating system version @@ -838,6 +853,7 @@ spec: - baremetal - external - vmware-cloud-director + - opennebula type: string spec: description: Spec represents the os/image reference in the supported From 94e2005cbd847a05e72d04ce97dc2d3bab59fa1c Mon Sep 17 00:00:00 2001 From: Mario Reggiori Date: Tue, 20 Jun 2023 07:46:27 +0200 Subject: [PATCH 331/489] Anexia Provider: allow configuration of multiple disks (#1665) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mario Schäfer --- examples/anexia-machinedeployment.yaml | 1 - go.mod | 20 ++++----- go.sum | 41 ++++++++++--------- pkg/cloudprovider/provider/anexia/provider.go | 14 +++---- .../provider/anexia/provider_test.go | 20 +++++++-- 5 files changed, 54 insertions(+), 42 deletions(-) diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index efb1564b7..ad08d8004 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -39,7 +39,6 @@ spec: cpus: 2 memory: 2048 - # only a single disk is currently supported, but support for multiple disks is planned already disks: - size: 60 performanceType: ENT6 diff --git a/go.mod b/go.mod index 551e32cfb..31c8eaed5 100644 --- a/go.mod +++ b/go.mod @@ -45,9 +45,9 @@ require ( github.com/vmware/go-vcloud-director/v2 v2.19.0 github.com/vmware/govmomi v0.30.0 github.com/vultr/govultr/v2 v2.17.2 - go.anx.io/go-anxcloud v0.5.0 + go.anx.io/go-anxcloud v0.5.3 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.4.0 + golang.org/x/crypto v0.10.0 golang.org/x/oauth2 v0.5.0 gomodules.xyz/jsonpatch/v2 v2.2.0 google.golang.org/api v0.105.0 @@ -143,8 +143,8 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/ginkgo/v2 v2.9.2 // indirect - github.com/onsi/gomega v1.27.6 // indirect + github.com/onsi/ginkgo/v2 v2.10.0 // indirect + github.com/onsi/gomega v1.27.8 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect @@ -167,13 +167,13 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/term v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sync v0.2.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/term v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.7.0 // indirect + golang.org/x/tools v0.9.3 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect google.golang.org/protobuf v1.30.0 // indirect diff --git a/go.sum b/go.sum index eeb0a4ab2..df2100f12 100644 --- a/go.sum +++ b/go.sum @@ -559,8 +559,8 @@ github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47 github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= -github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= -github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= +github.com/onsi/ginkgo/v2 v2.10.0 h1:sfUl4qgLdvkChZrWCYndY2EAu9BRIw1YphNAzy1VNWs= +github.com/onsi/ginkgo/v2 v2.10.0/go.mod h1:UDQOh5wbQUlMnkLfVaIUMtQ1Vus92oM+P2JX1aulgcE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= @@ -571,8 +571,8 @@ github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeR github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= -github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= -github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= +github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 h1:+ShYlGoPriGahTTFTjQ0RtNXW0srxDodk2STdc238Rk= @@ -729,8 +729,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.anx.io/go-anxcloud v0.5.0 h1:kKzAY+CRAXmQYCr+/lbEoO6JvPEVi5qjR2XgT0CMwx4= -go.anx.io/go-anxcloud v0.5.0/go.mod h1:IjUqXU0829myWH9015ES2KG2fBUnWNF5FChLwi5tUig= +go.anx.io/go-anxcloud v0.5.3 h1:ZKdHC2QaVvN6VdzT1O/aBxkabE/OoNJNLNpOyRzLqcw= +go.anx.io/go-anxcloud v0.5.3/go.mod h1:XTIg137l88FfE42X0/aOgvSm/fcXxWVp8k4+x9//zaQ= go.keploy.io/server v0.1.8 h1:b50vAt1+WKMscYVP5Bm8gx/iSaR7mpHox8VpaxjrQ88= go.keploy.io/server v0.1.8/go.mod h1:ZqhwTZOBb+dzx5t30Wt6eUGI6kO5QizvPg6coNPtbow= go.mongodb.org/mongo-driver v1.8.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= @@ -796,8 +796,8 @@ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= -golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= +golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= +golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -834,6 +834,7 @@ golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -887,8 +888,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -910,8 +911,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -981,16 +982,16 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= +golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1001,8 +1002,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1067,8 +1068,8 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 842954be6..e381a388b 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -61,9 +61,6 @@ const ( var ( // ErrConfigDiskSizeAndDisks is returned when the config has both DiskSize and Disks set, which is unsupported. ErrConfigDiskSizeAndDisks = errors.New("both the deprecated DiskSize and new Disks attribute are set") - - // ErrMultipleDisksNotYetImplemented is returned when multiple disks are configured. - ErrMultipleDisksNotYetImplemented = errors.New("multiple disks configured, but this feature is not yet implemented") ) type provider struct { @@ -164,6 +161,13 @@ func provisionVM(ctx context.Context, log *zap.SugaredLogger, client anxclient.C vm.DiskType = config.Disks[0].PerformanceType + for _, disk := range config.Disks[1:] { + vm.AdditionalDisks = append(vm.AdditionalDisks, anxvm.AdditionalDisk{ + SizeGBs: disk.Size, + Type: disk.PerformanceType, + }) + } + vm.Script = base64.StdEncoding.EncodeToString([]byte(reconcileContext.UserData)) providerCfg := reconcileContext.ProviderConfig @@ -431,10 +435,6 @@ func (p *provider) Validate(ctx context.Context, log *zap.SugaredLogger, machine return errors.New("no disks configured") } - if len(config.Disks) > 1 { - return ErrMultipleDisksNotYetImplemented - } - for _, disk := range config.Disks { if disk.Size == 0 { return errors.New("disk size is missing") diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index ab2b40ba1..7754ebb6e 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -67,6 +67,7 @@ func TestAnexiaProvider(t *testing.T) { a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID", Name: testTemplateName, Build: "b02"}) a.FakeExisting(&vspherev1.Template{Identifier: "WRONG-TEMPLATE-NAME", Name: "Wrong Template Name", Build: "b02"}) a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID-NO-NETWORK-CONFIG", Name: "no-network-config", Build: "b03"}) + a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID-ADDITIONAL-DISKS", Name: "additional-disks", Build: "b03"}) t.Cleanup(func() { testhelper.TeardownHTTP() @@ -128,6 +129,21 @@ func TestAnexiaProvider(t *testing.T) { testhelper.AssertEquals(t, exists, false) }, }, + { + ReconcileContext: hookableReconcileContext("LOCATION-ID", "ADDITIONAL-DISKS", func(rc *reconcileContext) { + rc.Config.Disks = append(rc.Config.Disks, resolvedDisk{ + RawDisk: anxtypes.RawDisk{ + Size: 10, + }, + PerformanceType: "STD1", + }) + }), + + AssertJSONBody: func(jsonBody jsonObject) { + testhelper.AssertEquals(t, json.Number("5"), jsonBody["disk_gb"]) + testhelper.AssertJSONEquals(t, `[{"gb":10,"type":"STD1"}]`, jsonBody["additional_disks"]) + }, + }, } testhelper.Mux.HandleFunc("/api/ipam/v1/address/reserve/ip/count.json", func(writer http.ResponseWriter, request *http.Request) { @@ -313,10 +329,6 @@ func TestValidate(t *testing.T) { Config: hookableConfig(func(c *anxtypes.RawConfig) { c.DiskSize = 10 }), Error: ErrConfigDiskSizeAndDisks, }, - ConfigTestCase{ - Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Disks = append(c.Disks, anxtypes.RawDisk{Size: 10}) }), - Error: ErrMultipleDisksNotYetImplemented, - }, ConfigTestCase{ Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Disks[0].Size = 0 }), Error: errors.New("disk size is missing"), From 6bfa9a0b5e70469a4794e3a61eee846d4630539d Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 3 Jul 2023 19:10:13 +0500 Subject: [PATCH 332/489] Support for configuring multiple NICs for vSphere (#1673) Signed-off-by: Waleed Malik --- docs/vsphere.md | 3 +- ...e-datastore-cluster-machinedeployment.yaml | 3 +- examples/vsphere-machinedeployment.yaml | 3 +- pkg/cloudprovider/provider/vsphere/helper.go | 4 +- pkg/cloudprovider/provider/vsphere/network.go | 55 +++++++++++-------- .../provider/vsphere/provider.go | 15 +++++ .../provider/vsphere/provider_test.go | 4 +- .../provider/vsphere/types/types.go | 12 ++-- 8 files changed, 64 insertions(+), 35 deletions(-) diff --git a/docs/vsphere.md b/docs/vsphere.md index 6a743ccd1..ab6962c94 100644 --- a/docs/vsphere.md +++ b/docs/vsphere.md @@ -226,7 +226,8 @@ datacenter: datacenter1 # VM template name templateVMName: ubuntu-template # Optional. Sets the networks on the VM. If no network is specified, the template default will be used. -vmNetName: network1 +networks: +- network1 # Optional folder: folder1 # Optional: Force VMs to be provisoned to the specified resourcePool diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml index f693fdbf6..6acd9acd9 100644 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ b/examples/vsphere-datastore-cluster-machinedeployment.yaml @@ -51,7 +51,8 @@ spec: datacenter: datacenter1 templateVMName: ubuntu-template # Optional. Sets the networks on the VM. If no network is specified, the template default will be used. - vmNetName: network1 + networks: + - network1 # Optional folder: folder1 datastoreCluster: datastorecluster1 diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index aadf49d33..4150dc76e 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -51,7 +51,8 @@ spec: datacenter: datacenter1 templateVMName: ubuntu-template # Optional. Sets the networks on the VM. If no network is specified, the template default will be used. - vmNetName: network1 + networks: + - network1 # Optional folder: folder1 datastore: datastore1 diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go index a3d5925c5..9e79dfebc 100644 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ b/pkg/cloudprovider/provider/vsphere/helper.go @@ -192,8 +192,8 @@ func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, deviceSpecs = append(deviceSpecs, diskspec) } - if config.VMNetName != "" { - networkSpecs, err := GetNetworkSpecs(ctx, session, vmDevices, config.VMNetName) + if config.VMNetName != "" || len(config.Networks) > 0 { + networkSpecs, err := GetNetworkSpecs(ctx, session, vmDevices, config.VMNetName, config.Networks) if err != nil { return nil, fmt.Errorf("failed to get network specifications: %w", err) } diff --git a/pkg/cloudprovider/provider/vsphere/network.go b/pkg/cloudprovider/provider/vsphere/network.go index e38d11135..e1d50b60a 100644 --- a/pkg/cloudprovider/provider/vsphere/network.go +++ b/pkg/cloudprovider/provider/vsphere/network.go @@ -29,7 +29,7 @@ const ( ) // Based on https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/pkg/cloud/vsphere/services/govmomi/vcenter/clone.go#L158 -func GetNetworkSpecs(ctx context.Context, session *Session, devices object.VirtualDeviceList, network string) ([]types.BaseVirtualDeviceConfigSpec, error) { +func GetNetworkSpecs(ctx context.Context, session *Session, devices object.VirtualDeviceList, network string, networks []string) ([]types.BaseVirtualDeviceConfigSpec, error) { var deviceSpecs []types.BaseVirtualDeviceConfigSpec // Remove any existing NICs. @@ -40,33 +40,40 @@ func GetNetworkSpecs(ctx context.Context, session *Session, devices object.Virtu }) } - // Add new NICs based on the machine config. - ref, err := session.Finder.Network(ctx, network) - if err != nil { - return nil, fmt.Errorf("failed to find network %q: %w", network, err) - } - backing, err := ref.EthernetCardBackingInfo(ctx) - if err != nil { - return nil, fmt.Errorf("failed to create new ethernet card backing info for network %q: %w", network, err) - } - dev, err := object.EthernetCardTypes().CreateEthernetCard(ethCardType, backing) - if err != nil { - return nil, fmt.Errorf("failed to create new ethernet card %q for network %q: %v", ethCardType, network, ctx) + // Add the default network if no networks are specified. + if network != "" { + networks = append(networks, network) } - // Get the actual NIC object. This is safe to assert without a check - // because "object.EthernetCardTypes().CreateEthernetCard" returns a - // "types.BaseVirtualEthernetCard" as a "types.BaseVirtualDevice". - nic := dev.(types.BaseVirtualEthernetCard).GetVirtualEthernetCard() + // Add NICs for each network. + for _, net := range networks { + // Add new NICs based on the machine config. + ref, err := session.Finder.Network(ctx, net) + if err != nil { + return nil, fmt.Errorf("failed to find network %q: %w", net, err) + } + backing, err := ref.EthernetCardBackingInfo(ctx) + if err != nil { + return nil, fmt.Errorf("failed to create new ethernet card backing info for network %q: %w", net, err) + } + dev, err := object.EthernetCardTypes().CreateEthernetCard(ethCardType, backing) + if err != nil { + return nil, fmt.Errorf("failed to create new ethernet card %q for network %q: %v", ethCardType, net, ctx) + } - // Assign a temporary device key to ensure that a unique one will be - // generated when the device is created. - nic.Key = devices.NewKey() + // Get the actual NIC object. This is safe to assert without a check + // because "object.EthernetCardTypes().CreateEthernetCard" returns a + // "types.BaseVirtualEthernetCard" as a "types.BaseVirtualDevice". + nic := dev.(types.BaseVirtualEthernetCard).GetVirtualEthernetCard() - deviceSpecs = append(deviceSpecs, &types.VirtualDeviceConfigSpec{ - Device: dev, - Operation: types.VirtualDeviceConfigSpecOperationAdd, - }) + // Assign a temporary device key to ensure that a unique one will be + // generated when the device is created. + nic.Key = devices.NewKey() + deviceSpecs = append(deviceSpecs, &types.VirtualDeviceConfigSpec{ + Device: dev, + Operation: types.VirtualDeviceConfigSpecOperationAdd, + }) + } return deviceSpecs, nil } diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index ceefc9c99..bf8b0f53b 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -62,6 +62,7 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes type Config struct { TemplateVMName string VMNetName string + Networks []string Username string Password string VSphereURL string @@ -143,11 +144,21 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, nil, err } + //nolint:staticcheck + //lint:ignore SA1019: rawConfig.VMNetName is deprecated: use networks instead. c.VMNetName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VMNetName) if err != nil { return nil, nil, nil, err } + for _, network := range rawConfig.Networks { + networkValue, err := p.configVarResolver.GetConfigVarStringValue(network) + if err != nil { + return nil, nil, rawConfig, err + } + c.Networks = append(c.Networks, networkValue) + } + c.Username, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Username, "VSPHERE_USERNAME") if err != nil { return nil, nil, nil, err @@ -231,6 +242,10 @@ func (p *provider) Validate(ctx context.Context, log *zap.SugaredLogger, spec cl } defer session.Logout(ctx) + if len(config.Networks) > 0 && config.VMNetName != "" { + return fmt.Errorf("both networks and vmNetName are specified, only one of them can be used") + } + if config.Tags != nil { restAPISession, err := NewRESTSession(ctx, config) if err != nil { diff --git a/pkg/cloudprovider/provider/vsphere/provider_test.go b/pkg/cloudprovider/provider/vsphere/provider_test.go index 463c9173e..f32c75275 100644 --- a/pkg/cloudprovider/provider/vsphere/provider_test.go +++ b/pkg/cloudprovider/provider/vsphere/provider_test.go @@ -64,7 +64,9 @@ func (v vsphereProviderSpecConf) rawProviderSpec(t *testing.T) []byte { "password": "{{ .Password }}", "templateVMName": "DC0_H0_VM0", "username": "{{ .User }}", - "vmNetName": "", + "networks": [ + "" + ], "vsphereURL": "{{ .URL }}" }, "operatingSystem": "flatcar", diff --git a/pkg/cloudprovider/provider/vsphere/types/types.go b/pkg/cloudprovider/provider/vsphere/types/types.go index dded33405..b0112d03c 100644 --- a/pkg/cloudprovider/provider/vsphere/types/types.go +++ b/pkg/cloudprovider/provider/vsphere/types/types.go @@ -24,11 +24,13 @@ import ( // RawConfig represents vsphere specific configuration. type RawConfig struct { TemplateVMName providerconfigtypes.ConfigVarString `json:"templateVMName"` - VMNetName providerconfigtypes.ConfigVarString `json:"vmNetName"` - Username providerconfigtypes.ConfigVarString `json:"username"` - Password providerconfigtypes.ConfigVarString `json:"password"` - VSphereURL providerconfigtypes.ConfigVarString `json:"vsphereURL"` - Datacenter providerconfigtypes.ConfigVarString `json:"datacenter"` + // Deprecated: use networks instead. + VMNetName providerconfigtypes.ConfigVarString `json:"vmNetName"` + Networks []providerconfigtypes.ConfigVarString `json:"networks"` + Username providerconfigtypes.ConfigVarString `json:"username"` + Password providerconfigtypes.ConfigVarString `json:"password"` + VSphereURL providerconfigtypes.ConfigVarString `json:"vsphereURL"` + Datacenter providerconfigtypes.ConfigVarString `json:"datacenter"` // Cluster defines the cluster to use in vcenter. // Only needed for vm anti affinity. From 102ad6f3b5cff5cb63dd71fa2c10d4ba37d6a1a7 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Mon, 3 Jul 2023 16:30:12 +0200 Subject: [PATCH 333/489] use `kubermatic-e2e-flatcar` for e2e tests (#1664) Signed-off-by: Marvin Beckers --- test/e2e/provisioning/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 45720c76a..4df03c90e 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -52,7 +52,7 @@ var ( string(providerconfigtypes.OperatingSystemUbuntu): "kubermatic-ubuntu", string(providerconfigtypes.OperatingSystemCentOS): "machine-controller-e2e-centos", string(providerconfigtypes.OperatingSystemRHEL): "machine-controller-e2e-rhel-8-5", - string(providerconfigtypes.OperatingSystemFlatcar): "machine-controller-e2e-flatcar-stable-2983", + string(providerconfigtypes.OperatingSystemFlatcar): "kubermatic-e2e-flatcar", string(providerconfigtypes.OperatingSystemRockyLinux): "machine-controller-e2e-rockylinux", } From f39e3198e3eb0af1576ae66d04aa8e9ae293cf5b Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Tue, 4 Jul 2023 16:36:16 +0200 Subject: [PATCH 334/489] KubeVirt: support for OCI VM image source (#1672) * kubevirt: support for OCI VM image source Signed-off-by: Jan Wozniak * kubevirt: support for OCI VM image source tests Signed-off-by: Jan Wozniak * kubevirt: allow configuring registry pull policy Signed-off-by: Jan Wozniak --------- Signed-off-by: Jan Wozniak --- .../provider/kubevirt/provider.go | 37 ++++++++- .../provider/kubevirt/provider_test.go | 16 +++- .../testdata/registry-image-source-pod.yaml | 78 +++++++++++++++++++ .../testdata/registry-image-source.yaml | 78 +++++++++++++++++++ .../provider/kubevirt/types/types.go | 2 + 5 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index a2ba1f58c..4d23f2c98 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -73,6 +73,8 @@ const ( machineDeploymentLabelKey = "md" // httpSource defines the http source type for VM Disk Image. httpSource imageSource = "http" + // registrySource defines the OCI registry source type for VM Disk Image. + registrySource imageSource = "registry" // pvcSource defines the pvc source type for VM Disk Image. pvcSource imageSource = "pvc" ) @@ -374,9 +376,15 @@ func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, nam if err != nil { return nil, fmt.Errorf(`failed to get value of "primaryDisk.source" field: %w`, err) } + pullMethod, err := p.getPullMethod(primaryDisk.PullMethod) + if err != nil { + return nil, fmt.Errorf(`failed to get value of "primaryDisk.pullMethod" field: %w`, err) + } switch imageSource(osImageSource) { case httpSource: return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage}}, nil + case registrySource: + return registryDataVolume(osImage, pullMethod), nil case pvcSource: if namespaceAndName := strings.Split(osImage, "/"); len(namespaceAndName) >= 2 { return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: namespaceAndName[1], Namespace: namespaceAndName[0]}}, nil @@ -384,7 +392,10 @@ func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, nam return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: namespace}}, nil default: // handle old API for backward compatibility. - if _, err = url.ParseRequestURI(osImage); err == nil { + if srcURL, err := url.ParseRequestURI(osImage); err == nil { + if srcURL.Scheme == cdiv1beta1.RegistrySchemeDocker || srcURL.Scheme == cdiv1beta1.RegistrySchemeOci { + return registryDataVolume(osImage, pullMethod), nil + } return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage}}, nil } if namespaceAndName := strings.Split(osImage, "/"); len(namespaceAndName) >= 2 { @@ -407,6 +418,30 @@ func getNamespace() string { return ns } +func (p *provider) getPullMethod(pullMethod providerconfigtypes.ConfigVarString) (cdiv1beta1.RegistryPullMethod, error) { + resolvedPM, err := p.configVarResolver.GetConfigVarStringValue(pullMethod) + if err != nil { + return "", err + } + switch pm := cdiv1beta1.RegistryPullMethod(resolvedPM); pm { + case cdiv1beta1.RegistryPullNode, cdiv1beta1.RegistryPullPod: + return pm, nil + case "": + return cdiv1beta1.RegistryPullNode, nil + default: + return "", fmt.Errorf("unsupported value: %v", resolvedPM) + } +} + +func registryDataVolume(imageURL string, pullMethod cdiv1beta1.RegistryPullMethod) *cdiv1beta1.DataVolumeSource { + return &cdiv1beta1.DataVolumeSource{ + Registry: &cdiv1beta1.DataVolumeSourceRegistry{ + URL: &imageURL, + PullMethod: &pullMethod, + }, + } +} + func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index 16cb82573..41de8082f 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -26,6 +26,7 @@ import ( "testing" kubevirtv1 "kubevirt.io/api/core/v1" + cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" cloudprovidertesting "github.com/kubermatic/machine-controller/pkg/cloudprovider/testing" "github.com/kubermatic/machine-controller/pkg/providerconfig" @@ -64,6 +65,8 @@ type kubevirtProviderSpecConf struct { AffinityValues bool SecondaryDisks bool OsImageSource imageSource + OsImageSourceURL string + PullMethod cdiv1beta1.RegistryPullMethod } func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { @@ -123,8 +126,11 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { {{- if .OsImageDV }} "osImage": "{{ .OsImageDV }}", {{- else }} - "osImage": "http://x.y.z.t/ubuntu.img", + "osImage": "{{ if .OsImageSourceURL }}{{ .OsImageSourceURL }}{{ else }}http://x.y.z.t/ubuntu.img{{ end }}", {{- end }} + {{- if .PullMethod }} + "pullMethod": "{{ .PullMethod }}", + {{- end}} "size": "10Gi", {{- if .OsImageSource }} "source": "{{ .OsImageSource }}", @@ -216,6 +222,14 @@ func TestNewVirtualMachine(t *testing.T) { name: "http-image-source", specConf: kubevirtProviderSpecConf{OsImageSource: httpSource}, }, + { + name: "registry-image-source", + specConf: kubevirtProviderSpecConf{OsImageSource: registrySource, OsImageSourceURL: "docker://x.y.z.t/ubuntu.img:latest"}, + }, + { + name: "registry-image-source-pod", + specConf: kubevirtProviderSpecConf{OsImageSource: registrySource, OsImageSourceURL: "docker://x.y.z.t/ubuntu.img:latest", PullMethod: cdiv1beta1.RegistryPullPod}, + }, { name: "pvc-image-source", specConf: kubevirtProviderSpecConf{OsImageSource: pvcSource, OsImageDV: "ns/dvname"}, diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml new file mode 100644 index 000000000..9a8115c1d --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml @@ -0,0 +1,78 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: registry-image-source-pod + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + md: md-name + name: registry-image-source-pod + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: registry-image-source-pod + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + registry: + url: docker://x.y.z.t/ubuntu.img:latest + pullMethod: pod + runStrategy: Once + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: registry-image-source-pod + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: registry-image-source-pod + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml new file mode 100644 index 000000000..ee0548b09 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml @@ -0,0 +1,78 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: registry-image-source + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + md: md-name + name: registry-image-source + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: registry-image-source + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + registry: + url: docker://x.y.z.t/ubuntu.img:latest + pullMethod: node + runStrategy: Once + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: registry-image-source + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: registry-image-source + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index 53f8e9dd8..e32a232fe 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -79,6 +79,8 @@ type PrimaryDisk struct { OsImage providerconfigtypes.ConfigVarString `json:"osImage,omitempty"` // Source describes the VM Disk Image source. Source providerconfigtypes.ConfigVarString `json:"source,omitempty"` + // PullMethod describes the VM Disk Image source optional pull method for registry source. Defaults to 'node'. + PullMethod providerconfigtypes.ConfigVarString `json:"pullMethod,omitempty"` } // SecondaryDisks. From 5f4e903ddafef24f0eb09560c74efb133819386d Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 4 Jul 2023 21:33:14 +0500 Subject: [PATCH 335/489] Remove support for Kubernetes 1.24 (#1675) * Remove support for Kubernetes 1.24 Signed-off-by: Waleed Malik * Refactored code Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- README.md | 9 +- examples/alibaba-machinedeployment.yaml | 2 +- examples/anexia-machinedeployment.yaml | 2 +- examples/aws-machinedeployment.yaml | 2 +- examples/azure-machinedeployment.yaml | 2 +- examples/digitalocean-machinedeployment.yaml | 2 +- examples/equinixmetal-machinedeployment.yaml | 2 +- examples/gce-machinedeployment.yaml | 2 +- examples/hetzner-machinedeployment.yaml | 2 +- examples/kubevirt-machinedeployment.yaml | 2 +- examples/linode-machinedeployment.yaml | 2 +- examples/nutanix-machinedeployment.yaml | 2 +- examples/opennebula-machinedeployment.yaml | 2 +- examples/openstack-machinedeployment.yaml | 2 +- examples/scaleway-machinedeployment.yaml | 2 +- ...ware-cloud-director-machinedeployment.yaml | 2 +- ...e-datastore-cluster-machinedeployment.yaml | 2 +- examples/vsphere-machinedeployment.yaml | 2 +- examples/vultr-machinedeployment.yaml | 4 +- pkg/userdata/amzn2/provider_test.go | 33 +- .../amzn2/testdata/kubelet-v1.24.9-aws.yaml | 452 --------------- ...yaml => kubelet-v1.26.6-aws-external.yaml} | 2 +- ...1.24-aws.yaml => kubelet-v1.26.6-aws.yaml} | 2 +- ...l => kubelet-v1.26.6-vsphere-mirrors.yaml} | 2 +- ...aml => kubelet-v1.26.6-vsphere-proxy.yaml} | 2 +- ...here.yaml => kubelet-v1.26.6-vsphere.yaml} | 2 +- pkg/userdata/centos/provider_test.go | 33 +- ...yaml => kubelet-v1.26.6-aws-external.yaml} | 2 +- ...24.9-aws.yaml => kubelet-v1.26.6-aws.yaml} | 2 +- ...anix.yaml => kubelet-v1.26.6-nutanix.yaml} | 2 +- ...l => kubelet-v1.26.6-vsphere-mirrors.yaml} | 2 +- ...aml => kubelet-v1.26.6-vsphere-proxy.yaml} | 2 +- ...here.yaml => kubelet-v1.26.6-vsphere.yaml} | 2 +- pkg/userdata/flatcar/provider_test.go | 22 +- .../flatcar/testdata/cloud-init_v1.24.9.yaml | 519 ------------------ ...t_v1.24.0.yaml => cloud-init_v1.26.6.yaml} | 2 +- pkg/userdata/flatcar/testdata/containerd.yaml | 2 +- .../flatcar/testdata/ignition_v1.24.0.json | 1 - ...ion_v1.24.9.json => ignition_v1.26.6.json} | 2 +- pkg/userdata/helper/common_test.go | 7 +- .../helper/download_binaries_script_test.go | 4 +- pkg/userdata/helper/kubelet_test.go | 8 +- ...lden => download_binaries_v1.25.11.golden} | 2 +- ...olden => download_binaries_v1.26.6.golden} | 2 +- .../testdata/download_binaries_v1.27.1.golden | 17 - ...olden => download_binaries_v1.27.3.golden} | 2 +- ...emd_unit_version-v1.25.11-external.golden} | 0 ...blet_systemd_unit_version-v1.25.11.golden} | 0 ...temd_unit_version-v1.26.6-external.golden} | 0 ...ublet_systemd_unit_version-v1.26.6.golden} | 0 ...stemd_unit_version-v1.27.3-external.golden | 36 ++ ...kublet_systemd_unit_version-v1.27.3.golden | 35 ++ ... => safe_download_binaries_v1.26.6.golden} | 2 +- pkg/userdata/rhel/provider_test.go | 28 +- ...l.yaml => kubelet-v1.26-aws-external.yaml} | 2 +- ...-v1.24-aws.yaml => kubelet-v1.26-aws.yaml} | 2 +- ...yaml => kubelet-v1.26.6-aws-external.yaml} | 2 +- ...24.9-aws.yaml => kubelet-v1.26.6-aws.yaml} | 2 +- ...l => kubelet-v1.26.6-vsphere-mirrors.yaml} | 2 +- ...aml => kubelet-v1.26.6-vsphere-proxy.yaml} | 2 +- ...here.yaml => kubelet-v1.26.6-vsphere.yaml} | 2 +- pkg/userdata/rockylinux/provider_test.go | 33 +- .../testdata/kubelet-v1.24.9-aws.yaml | 465 ---------------- ...yaml => kubelet-v1.26.6-aws-external.yaml} | 2 +- ...1.24-aws.yaml => kubelet-v1.26.6-aws.yaml} | 2 +- ...anix.yaml => kubelet-v1.26.6-nutanix.yaml} | 2 +- ...l => kubelet-v1.26.6-vsphere-mirrors.yaml} | 2 +- ...aml => kubelet-v1.26.6-vsphere-proxy.yaml} | 2 +- ...here.yaml => kubelet-v1.26.6-vsphere.yaml} | 2 +- pkg/userdata/ubuntu/provider_test.go | 9 +- pkg/userdata/ubuntu/testdata/nutanix.yaml | 2 +- .../ubuntu/testdata/version-1.24.9.yaml | 458 ---------------- ...sion-1.24.13.yaml => version-1.25.11.yaml} | 2 +- ...ersion-1.26.4.yaml => version-1.26.6.yaml} | 2 +- ...ersion-1.27.1.yaml => version-1.27.3.yaml} | 2 +- test/e2e/provisioning/all_e2e_test.go | 8 +- test/e2e/provisioning/helper.go | 7 +- ...hinedeployment-azure-redhat-satellite.yaml | 2 +- 78 files changed, 211 insertions(+), 2083 deletions(-) delete mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml rename pkg/userdata/amzn2/testdata/{kubelet-v1.24.9-aws-external.yaml => kubelet-v1.26.6-aws-external.yaml} (99%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.24-aws.yaml => kubelet-v1.26.6-aws.yaml} (99%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.24.9-vsphere-mirrors.yaml => kubelet-v1.26.6-vsphere-mirrors.yaml} (99%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.24.9-vsphere-proxy.yaml => kubelet-v1.26.6-vsphere-proxy.yaml} (99%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.24.9-vsphere.yaml => kubelet-v1.26.6-vsphere.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.24.9-aws-external.yaml => kubelet-v1.26.6-aws-external.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.24.9-aws.yaml => kubelet-v1.26.6-aws.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.24.9-nutanix.yaml => kubelet-v1.26.6-nutanix.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.24.9-vsphere-mirrors.yaml => kubelet-v1.26.6-vsphere-mirrors.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.24.9-vsphere-proxy.yaml => kubelet-v1.26.6-vsphere-proxy.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.24.9-vsphere.yaml => kubelet-v1.26.6-vsphere.yaml} (99%) delete mode 100644 pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml rename pkg/userdata/flatcar/testdata/{cloud-init_v1.24.0.yaml => cloud-init_v1.26.6.yaml} (99%) delete mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.24.0.json rename pkg/userdata/flatcar/testdata/{ignition_v1.24.9.json => ignition_v1.26.6.json} (99%) rename pkg/userdata/helper/testdata/{download_binaries_v1.24.13.golden => download_binaries_v1.25.11.golden} (92%) rename pkg/userdata/helper/testdata/{download_binaries_v1.24.9.golden => download_binaries_v1.26.6.golden} (92%) delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.27.1.golden rename pkg/userdata/helper/testdata/{download_binaries_v1.26.4.golden => download_binaries_v1.27.3.golden} (92%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.24.13-external.golden => kublet_systemd_unit_version-v1.25.11-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.24.13.golden => kublet_systemd_unit_version-v1.25.11.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.24.9-external.golden => kublet_systemd_unit_version-v1.26.6-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.24.9.golden => kublet_systemd_unit_version-v1.26.6.golden} (100%) create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.3-external.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.3.golden rename pkg/userdata/helper/testdata/{safe_download_binaries_v1.24.9.golden => safe_download_binaries_v1.26.6.golden} (98%) rename pkg/userdata/rhel/testdata/{kubelet-v1.24.9-aws-external.yaml => kubelet-v1.26-aws-external.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.24-aws.yaml => kubelet-v1.26-aws.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.24-aws-external.yaml => kubelet-v1.26.6-aws-external.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.24.9-aws.yaml => kubelet-v1.26.6-aws.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.24.9-vsphere-mirrors.yaml => kubelet-v1.26.6-vsphere-mirrors.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.24.9-vsphere-proxy.yaml => kubelet-v1.26.6-vsphere-proxy.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.24.9-vsphere.yaml => kubelet-v1.26.6-vsphere.yaml} (99%) delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml rename pkg/userdata/rockylinux/testdata/{kubelet-v1.24.9-aws-external.yaml => kubelet-v1.26.6-aws-external.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.24-aws.yaml => kubelet-v1.26.6-aws.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.24.9-nutanix.yaml => kubelet-v1.26.6-nutanix.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.24.9-vsphere-mirrors.yaml => kubelet-v1.26.6-vsphere-mirrors.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.24.9-vsphere-proxy.yaml => kubelet-v1.26.6-vsphere-proxy.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.24.9-vsphere.yaml => kubelet-v1.26.6-vsphere.yaml} (99%) delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.24.9.yaml rename pkg/userdata/ubuntu/testdata/{version-1.24.13.yaml => version-1.25.11.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.26.4.yaml => version-1.26.6.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.27.1.yaml => version-1.27.3.yaml} (99%) diff --git a/README.md b/README.md index b1da95a64..0dc5483dd 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,10 @@ - [What Works](#what-works) - [Supported Kubernetes Versions](#supported-kubernetes-versions) - [Community Providers](#community-providers) - - [What doesn't Work](#what-doesnt-work) + - [What Doesn't Work](#what-doesnt-work) - [Quickstart](#quickstart) - - [Deploy machine-controller](#deploy-the-machine-controller) - - [Creating a MachineDeployment](#creating-a-machinedeployment) + - [Deploy machine-controller](#deploy-machine-controller) + - [Creating a `MachineDeployment`](#creating-a-machinedeployment) - [Advanced Usage](#advanced-usage) - [Specifying the Apiserver Endpoint](#specifying-the-apiserver-endpoint) - [CA Data](#ca-data) @@ -46,7 +46,6 @@ Currently supported K8S versions are: - 1.27 - 1.26 - 1.25 -- 1.24 ### Community Providers @@ -169,4 +168,4 @@ See [the list of releases][7] to find out about feature changes. [5]: CONTRIBUTING.md [6]: Zenhub.md [7]: https://github.com/kubermatic/machine-controller/releases -[8]: https://docs.kubermatic.com/operatingsystemmanager \ No newline at end of file +[8]: https://docs.kubermatic.com/operatingsystemmanager diff --git a/examples/alibaba-machinedeployment.yaml b/examples/alibaba-machinedeployment.yaml index b36c7571b..e25a10b2b 100644 --- a/examples/alibaba-machinedeployment.yaml +++ b/examples/alibaba-machinedeployment.yaml @@ -62,4 +62,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index ad08d8004..acafdaf75 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -54,4 +54,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/aws-machinedeployment.yaml b/examples/aws-machinedeployment.yaml index ce04e8aef..8f56c6892 100644 --- a/examples/aws-machinedeployment.yaml +++ b/examples/aws-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/azure-machinedeployment.yaml b/examples/azure-machinedeployment.yaml index 63166665e..6612809f7 100644 --- a/examples/azure-machinedeployment.yaml +++ b/examples/azure-machinedeployment.yaml @@ -91,4 +91,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/digitalocean-machinedeployment.yaml b/examples/digitalocean-machinedeployment.yaml index f914a89ef..bdf1db222 100644 --- a/examples/digitalocean-machinedeployment.yaml +++ b/examples/digitalocean-machinedeployment.yaml @@ -57,4 +57,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/equinixmetal-machinedeployment.yaml b/examples/equinixmetal-machinedeployment.yaml index c46b17f5d..20e4733b4 100644 --- a/examples/equinixmetal-machinedeployment.yaml +++ b/examples/equinixmetal-machinedeployment.yaml @@ -51,4 +51,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index cdef9d4cd..dca6466f8 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -90,4 +90,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/hetzner-machinedeployment.yaml b/examples/hetzner-machinedeployment.yaml index 57ce82784..07a7324fe 100644 --- a/examples/hetzner-machinedeployment.yaml +++ b/examples/hetzner-machinedeployment.yaml @@ -67,4 +67,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index 81b71ed4a..d510c9490 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -69,4 +69,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/linode-machinedeployment.yaml b/examples/linode-machinedeployment.yaml index cf7beb50d..a08d60acf 100644 --- a/examples/linode-machinedeployment.yaml +++ b/examples/linode-machinedeployment.yaml @@ -53,4 +53,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/nutanix-machinedeployment.yaml b/examples/nutanix-machinedeployment.yaml index e6978b0a6..862bb15bb 100644 --- a/examples/nutanix-machinedeployment.yaml +++ b/examples/nutanix-machinedeployment.yaml @@ -82,4 +82,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/opennebula-machinedeployment.yaml b/examples/opennebula-machinedeployment.yaml index 0bc101881..768135a99 100644 --- a/examples/opennebula-machinedeployment.yaml +++ b/examples/opennebula-machinedeployment.yaml @@ -67,4 +67,4 @@ spec: # use cloud-init for flatcar as ignition doesn't know anything about OpenNebula yet provisioningUtility: "cloud-init" versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index f3401d3cd..ccffe0096 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -166,4 +166,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/scaleway-machinedeployment.yaml b/examples/scaleway-machinedeployment.yaml index da66040ff..9a14eb3cc 100644 --- a/examples/scaleway-machinedeployment.yaml +++ b/examples/scaleway-machinedeployment.yaml @@ -61,4 +61,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/vmware-cloud-director-machinedeployment.yaml b/examples/vmware-cloud-director-machinedeployment.yaml index 15b7011f6..bdecf1769 100644 --- a/examples/vmware-cloud-director-machinedeployment.yaml +++ b/examples/vmware-cloud-director-machinedeployment.yaml @@ -83,4 +83,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml index 6acd9acd9..58d93c5ef 100644 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ b/examples/vsphere-datastore-cluster-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index 4150dc76e..368f1f16a 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/examples/vultr-machinedeployment.yaml b/examples/vultr-machinedeployment.yaml index 390c88453..3aa064daf 100644 --- a/examples/vultr-machinedeployment.yaml +++ b/examples/vultr-machinedeployment.yaml @@ -44,7 +44,7 @@ spec: name: machine-controller-vultr key: apiKey region: blr - plan: 'vhf-8c-32gb' + plan: "vhf-8c-32gb" # Required: app_id, image_id, os_id, snapshot_id, or iso_id must be provided. Currently only os_id is supported. osId: 215 # Optional @@ -57,4 +57,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.24.9 + kubelet: 1.26.6 diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index 53866c402..bf6db0909 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -102,40 +102,40 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.24.9-aws", + name: "kubelet-v1.26.6-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.9", + Kubelet: "1.26.6", }, }, }, { - name: "kubelet-v1.24.9-aws-external", + name: "kubelet-v1.26.6-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.9", + Kubelet: "1.26.6", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.24.9-vsphere", + name: "kubelet-v1.26.6-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.9", + Kubelet: "1.26.6", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.24.9-vsphere-proxy", + name: "kubelet-v1.26.6-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.9", + Kubelet: "1.26.6", }, }, cloudProviderName: stringPtr("vsphere"), @@ -145,11 +145,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.24.9-vsphere-mirrors", + name: "kubelet-v1.26.6-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.9", + Kubelet: "1.26.6", }, }, cloudProviderName: stringPtr("vsphere"), @@ -159,11 +159,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.24-aws", + name: "kubelet-v1.26-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.0", + Kubelet: "1.26.0", }, }, }, @@ -176,15 +176,6 @@ func TestUserDataGeneration(t *testing.T) { }, }, }, - { - name: "kubelet-v1.26-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.0", - }, - }, - }, { name: "kubelet-v1.27-aws", spec: clusterv1alpha1.MachineSpec{ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml deleted file mode 100644 index c71e4fccc..000000000 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws.yaml +++ /dev/null @@ -1,452 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml index 25422f0f0..b29b20207 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml @@ -143,7 +143,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml index 8d7237688..6e525dc4e 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml @@ -143,7 +143,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index 7b65738b3..8b5b3ba5f 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -156,7 +156,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index 3fa66891b..594cde2c8 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -156,7 +156,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml index 36afcc388..a0c323240 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml @@ -148,7 +148,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index fccccab59..d0c9df2a8 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -102,50 +102,50 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.24.9-aws", + name: "kubelet-v1.26.6-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.9", + Kubelet: "1.26.6", }, }, }, { - name: "kubelet-v1.24.9-nutanix", + name: "kubelet-v1.26.6-nutanix", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.9", + Kubelet: "1.26.6", }, }, cloudProviderName: stringPtr("nutanix"), }, { - name: "kubelet-v1.24.9-aws-external", + name: "kubelet-v1.26.6-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.9", + Kubelet: "1.26.6", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.24.9-vsphere", + name: "kubelet-v1.26.6-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.9", + Kubelet: "1.26.6", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.24.9-vsphere-proxy", + name: "kubelet-v1.26.6-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.9", + Kubelet: "1.26.6", }, }, cloudProviderName: stringPtr("vsphere"), @@ -155,11 +155,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.24.9-vsphere-mirrors", + name: "kubelet-v1.26.6-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.9", + Kubelet: "1.26.6", }, }, cloudProviderName: stringPtr("vsphere"), @@ -177,15 +177,6 @@ func TestUserDataGeneration(t *testing.T) { }, }, }, - { - name: "kubelet-v1.26-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.0", - }, - }, - }, { name: "kubelet-v1.27-aws", spec: clusterv1alpha1.MachineSpec{ diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml index 4704bf099..f4cdb9b1d 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml index 28c514eb6..8bc556202 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml index 78cbe025e..9cc2721da 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml @@ -156,7 +156,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index 5226b46ee..8ad4918c1 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -162,7 +162,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index 04204ee53..ca9a5ea4b 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -162,7 +162,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml index 05284d026..d796580f4 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.24.9-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml @@ -154,7 +154,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index c12fe1aa4..fc9845183 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -155,7 +155,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.24.9", + name: "ignition_v1.26.6", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -171,7 +171,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.9", + Kubelet: "1.26.6", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -186,7 +186,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.24.9", + name: "ignition_v1.26.6", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -202,7 +202,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.9", + Kubelet: "1.26.6", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -217,7 +217,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.24.0", + name: "ignition_v1.26.6", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -233,7 +233,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.0", + Kubelet: "1.26.6", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -279,7 +279,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.24.9", + name: "cloud-init_v1.26.6", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -295,7 +295,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.9", + Kubelet: "1.26.6", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -310,7 +310,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.24.0", + name: "cloud-init_v1.26.6", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -326,7 +326,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.0", + Kubelet: "1.26.6", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -353,7 +353,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.0", + Kubelet: "1.26.6", }, }, ccProvider: &fakeCloudConfigProvider{}, diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml deleted file mode 100644 index c2c598fd9..000000000 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.24.9.yaml +++ /dev/null @@ -1,519 +0,0 @@ -#cloud-config - -users: -- name: core - ssh_authorized_keys: - - ssh-rsa AAABBB - - ssh-rsa CCCDDD - - -coreos: - units: - - name: static-nic.network - content: | - [Match] - # Because of difficulty predicting specific NIC names on different cloud providers, - # we only support static addressing on VSphere. There should be a single NIC attached - # that we will match by name prefix 'en' which denotes ethernet devices. - Name=en* - - [Network] - DHCP=no - Address=192.168.81.4/24 - Gateway=192.168.81.1 - DNS=8.8.8.8 - - - name: update-engine.service - command: stop - mask: true - - name: locksmithd.service - command: stop - mask: true - - name: download-script.service - enable: true - command: start - content: | - [Unit] - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/download.sh - [Install] - WantedBy=multi-user.target - - - name: kubelet-healthcheck.service - enable: true - command: start - drop-ins: - - name: 40-download.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - - - name: nodeip.service - enable: true - command: start - content: | - [Unit] - Description=Setup Kubelet Node IP Env - Requires=network-online.target - After=network-online.target - - [Service] - ExecStart=/opt/bin/setup_net_env.sh - RemainAfterExit=yes - Type=oneshot - [Install] - WantedBy=multi-user.target - - - name: kubelet.service - enable: true - command: start - drop-ins: - - name: 10-nodeip.conf - content: | - [Service] - EnvironmentFile=/etc/kubernetes/nodeip.conf - - name: resolv.conf - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - - name: 40-download.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - - name: 50-rpc-statd.conf - content: | - [Unit] - Wants=rpc-statd.service - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=anexia \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - - - name: apply-sysctl-settings.service - enable: true - command: start - content: | - [Unit] - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - ExecStart=/opt/bin/apply_sysctl_settings.sh - [Install] - WantedBy=multi-user.target - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - permissions: "0644" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/etc/kubernetes/kubelet.conf" - permissions: "0644" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /opt/load-kernel-modules.sh - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: /etc/sysctl.d/k8s.conf - permissions: "0644" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" - permissions: "0755" - content: | - # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar - # so we enable IPv6 Router Advertisement here. - # See for details https://github.com/flatcar-linux/Flatcar/issues/384 - [Network] - IPv6AcceptRA=true - -- path: /etc/kubernetes/bootstrap-kubelet.conf - permissions: "0400" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: /etc/kubernetes/cloud-config - permissions: "0400" - content: | - {anexia-config:true} - -- path: /etc/kubernetes/pki/ca.crt - permissions: "0644" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - -- path: /etc/hostname - permissions: "0600" - content: 'node1' - -- path: /etc/ssh/sshd_config - permissions: "0600" - user: root - content: | - # Use most defaults for sshd configuration. - Subsystem sftp internal-sftp - ClientAliveInterval 180 - UseDNS no - UsePAM yes - PrintLastLog no # handled by PAM - PrintMotd no # handled by PAM - PasswordAuthentication no - ChallengeResponseAuthentication no - -- path: /opt/bin/download.sh - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.2.0}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") - cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat < /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml index 6bd394ef3..dd50f43ee 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml index 6220c331d..451e55065 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml index 1d25a9ce1..6997f46cd 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index 03e7e7b37..f6af69a8b 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -157,7 +157,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index 42348f33d..49829999d 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -157,7 +157,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml index 64f29c5c4..584c43377 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 351d9c909..513206b94 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -129,10 +129,9 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.24.13"), - semver.MustParse("v1.25.9"), - semver.MustParse("v1.26.4"), - semver.MustParse("v1.27.1"), + semver.MustParse("v1.25.11"), + semver.MustParse("v1.26.6"), + semver.MustParse("v1.27.3"), } var tests []userDataTestCase @@ -603,7 +602,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.24.9", + Kubelet: "1.26.6", }, }, ccProvider: &fakeCloudConfigProvider{ diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 26c10bc00..14b64823a 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -153,7 +153,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml b/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml deleted file mode 100644 index d1b970bef..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.24.9.yaml +++ /dev/null @@ -1,458 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.24.13.yaml b/pkg/userdata/ubuntu/testdata/version-1.25.11.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.24.13.yaml rename to pkg/userdata/ubuntu/testdata/version-1.25.11.yaml index 45876c521..11364d00d 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.24.13.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.25.11.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.13}" + KUBE_VERSION="${KUBE_VERSION:-v1.25.11}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.26.4.yaml b/pkg/userdata/ubuntu/testdata/version-1.26.6.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.26.4.yaml rename to pkg/userdata/ubuntu/testdata/version-1.26.6.yaml index e35832c2d..5c3a67922 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.26.4.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.26.6.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.4}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.27.1.yaml b/pkg/userdata/ubuntu/testdata/version-1.27.3.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.27.1.yaml rename to pkg/userdata/ubuntu/testdata/version-1.27.3.yaml index 86dd288d9..cc5235c70 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.27.1.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.27.3.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.27.1}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.3}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 92d299bb8..3e1898919 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -342,7 +342,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.26. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.26.4", "1.27.1"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.26.6", "1.27.3"))) runScenarios(t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -422,7 +422,7 @@ func TestAWSProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := Not(VersionSelector("1.27.1")) + selector := Not(VersionSelector("1.27.3")) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -476,7 +476,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.1"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.3"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -498,7 +498,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.1"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.3"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 4df03c90e..8c4231085 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,10 +33,9 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.24.13"), - semver.MustParse("v1.25.9"), - semver.MustParse("v1.26.4"), - semver.MustParse("v1.27.1"), + semver.MustParse("v1.25.11"), + semver.MustParse("v1.26.6"), + semver.MustParse("v1.27.3"), } operatingSystems = []providerconfigtypes.OperatingSystem{ diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml index 0af6c42a0..db1768715 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml @@ -55,4 +55,4 @@ spec: rhelOrganizationName: "" rhelActivationKey: "" versions: - kubelet: 1.24.9 + kubelet: 1.26.6 From 885284c29d4f2a74bedb035aa5a92dcc3f4c2deb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Jul 2023 18:42:30 +0200 Subject: [PATCH 336/489] Bump google.golang.org/grpc from 1.51.0 to 1.53.0 (#1676) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.51.0 to 1.53.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.51.0...v1.53.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 31c8eaed5..741d50750 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( golang.org/x/oauth2 v0.5.0 gomodules.xyz/jsonpatch/v2 v2.2.0 google.golang.org/api v0.105.0 - google.golang.org/grpc v1.51.0 + google.golang.org/grpc v1.53.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.26.4 @@ -70,7 +70,7 @@ require ( require ( cloud.google.com/go v0.107.0 // indirect - cloud.google.com/go/compute v1.14.0 // indirect + cloud.google.com/go/compute v1.15.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/longrunning v0.3.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect @@ -175,7 +175,7 @@ require ( golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.3 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect + google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index df2100f12..a4de3e560 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,8 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.14.0 h1:hfm2+FfxVmnRlh6LpB7cg1ZNU+5edAHmW679JePztk0= -cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= +cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= +cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= @@ -1137,8 +1137,8 @@ google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef h1:uQ2vjV/sHTsWSqdKeLqmwitzgvjMl7o4IdtHwUDXSJY= -google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= +google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1157,8 +1157,8 @@ google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= -google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= +google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= google.golang.org/grpc/examples v0.0.0-20210728214646-ad0a2a847cdf/go.mod h1:bF8wuZSAZTcbF7ZPKrDI/qY52toTP/yxLpRRY4Eu9Js= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From cb07ce83702b3224ea16f4bdb8c54f6677d5f7a7 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 7 Jul 2023 12:40:31 +0500 Subject: [PATCH 337/489] VSphere: Fine tune multiple NIC support and add E2E tests (#1678) Signed-off-by: Waleed Malik --- .prow/provider-vsphere.yaml | 32 +++++++++++ pkg/cloudprovider/provider/vsphere/network.go | 8 ++- test/e2e/provisioning/all_e2e_test.go | 12 ++++ ...achinedeployment-vsphere-multiple-nic.yaml | 57 +++++++++++++++++++ .../machinedeployment-vsphere-static-ip.yaml | 2 +- 5 files changed, 107 insertions(+), 4 deletions(-) create mode 100644 test/e2e/provisioning/testdata/machinedeployment-vsphere-multiple-nic.yaml diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 35becb03d..28e7a7310 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -108,3 +108,35 @@ presubmits: cpu: 2 limits: memory: 7Gi + + - name: pull-machine-controller-e2e-vsphere-multiple-networks + always_run: false + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-vsphere: "true" + preset-rhel: "true" + preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" + spec: + containers: + - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + command: + - "./hack/ci/run-e2e-tests.sh" + args: + - "TestVsphereMultipleNICProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: vsphere + securityContext: + privileged: true + resources: + requests: + memory: 7Gi + cpu: 2 + limits: + memory: 7Gi diff --git a/pkg/cloudprovider/provider/vsphere/network.go b/pkg/cloudprovider/provider/vsphere/network.go index e1d50b60a..2f3b6dc3d 100644 --- a/pkg/cloudprovider/provider/vsphere/network.go +++ b/pkg/cloudprovider/provider/vsphere/network.go @@ -28,9 +28,9 @@ const ( ethCardType = "vmxnet3" ) -// Based on https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/pkg/cloud/vsphere/services/govmomi/vcenter/clone.go#L158 +// Based on https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/v1.7.0/pkg/services/govmomi/vcenter/clone.go#L372 func GetNetworkSpecs(ctx context.Context, session *Session, devices object.VirtualDeviceList, network string, networks []string) ([]types.BaseVirtualDeviceConfigSpec, error) { - var deviceSpecs []types.BaseVirtualDeviceConfigSpec + deviceSpecs := []types.BaseVirtualDeviceConfigSpec{} // Remove any existing NICs. for _, dev := range devices.SelectByType((*types.VirtualEthernetCard)(nil)) { @@ -46,6 +46,7 @@ func GetNetworkSpecs(ctx context.Context, session *Session, devices object.Virtu } // Add NICs for each network. + deviceKey := int32(-100) for _, net := range networks { // Add new NICs based on the machine config. ref, err := session.Finder.Network(ctx, net) @@ -68,12 +69,13 @@ func GetNetworkSpecs(ctx context.Context, session *Session, devices object.Virtu // Assign a temporary device key to ensure that a unique one will be // generated when the device is created. - nic.Key = devices.NewKey() + nic.Key = deviceKey deviceSpecs = append(deviceSpecs, &types.VirtualDeviceConfigSpec{ Device: dev, Operation: types.VirtualDeviceConfigSpecOperationAdd, }) + deviceKey-- } return deviceSpecs, nil } diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 3e1898919..bfcbe76ff 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -66,6 +66,7 @@ const ( LinodeManifest = "./testdata/machinedeployment-linode.yaml" VMwareCloudDirectorManifest = "./testdata/machinedeployment-vmware-cloud-director.yaml" VSPhereManifest = "./testdata/machinedeployment-vsphere.yaml" + VSPhereMultipleNICManifest = "./testdata/machinedeployment-vsphere-multiple-nic.yaml" VSPhereDSCManifest = "./testdata/machinedeployment-vsphere-datastore-cluster.yaml" VSPhereResourcePoolManifest = "./testdata/machinedeployment-vsphere-resource-pool.yaml" ScalewayManifest = "./testdata/machinedeployment-scaleway.yaml" @@ -842,6 +843,17 @@ func TestVsphereProvisioningE2E(t *testing.T) { runScenarios(t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) } +// TestVsphereMultipleNICProvisioning - is the same as the TestVsphereProvisioning suit but has multiple networks attached to the VMs. +// by requesting nodes with different combination of container runtime type, container runtime version and the OS flavour. +func TestVsphereMultipleNICProvisioningE2E(t *testing.T) { + t.Parallel() + + selector := OsSelector("ubuntu") + params := getVSphereTestParams(t) + + runScenarios(t, selector, params, VSPhereMultipleNICManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) +} + // TestVsphereDatastoreClusterProvisioning - is the same as the TestVsphereProvisioning suite but specifies a DatastoreCluster // instead of the Datastore in the provider specs. func TestVsphereDatastoreClusterProvisioningE2E(t *testing.T) { diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-multiple-nic.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-multiple-nic.yaml new file mode 100644 index 000000000..4be08e39c --- /dev/null +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-multiple-nic.yaml @@ -0,0 +1,57 @@ +apiVersion: "cluster.k8s.io/v1alpha1" +kind: MachineDeployment +metadata: + name: << MACHINE_NAME >> + namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + name: << MACHINE_NAME >> + template: + metadata: + labels: + name: << MACHINE_NAME >> + spec: + providerSpec: + value: + sshPublicKeys: + - "<< YOUR_PUBLIC_KEY >>" + cloudProvider: "vsphere" + cloudProviderSpec: + templateVMName: "<< OS_Image_Template >>" + username: "<< VSPHERE_USERNAME >>" + vsphereURL: "<< VSPHERE_ADDRESS >>" + datacenter: "Hamburg" + folder: "/Hamburg/vm/Kubermatic-ci" + password: << VSPHERE_PASSWORD >> + # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically + cluster: Kubermatic + vmAntiAffinity: true + networks: + - /Hamburg/network/Default Network + - /Hamburg/network/Management + datastore: vsan + cpus: 2 + MemoryMB: 4096 + diskSizeGB: << DISK_SIZE >> + allowInsecure: true + operatingSystem: "<< OS_NAME >>" + operatingSystemSpec: + distUpgradeOnBoot: false + disableAutoUpdate: true + attachSubscription: false + # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` + rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" + # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` + rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" + rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" + versions: + kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml index 548ab6786..ad4dcfda1 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml @@ -32,7 +32,7 @@ spec: datacenter: 'Hamburg' folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> - # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically3 + # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically cluster: Kubermatic vmAntiAffinity: true datastore: vsan From e5f238ce308488a2acb25f4240214bcefad05955 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 18 Jul 2023 13:30:46 +0500 Subject: [PATCH 338/489] Propagate datastore cluster for vSphere to cloud-config (#1682) Signed-off-by: Waleed Malik --- pkg/cloudprovider/provider/vsphere/provider.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index bf8b0f53b..93f9bf552 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -618,6 +618,11 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri workingDir = fmt.Sprintf("/%s/vm", c.Datacenter) } + datastore := c.Datastore + if datastore == "" { + datastore = c.DatastoreCluster + } + cc := &vspheretypes.CloudConfig{ Global: vspheretypes.GlobalOpts{ User: c.Username, @@ -631,7 +636,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri Workspace: vspheretypes.WorkspaceOpts{ Datacenter: c.Datacenter, VCenterIP: u.Hostname(), - DefaultDatastore: c.Datastore, + DefaultDatastore: datastore, Folder: workingDir, }, VirtualCenter: map[string]*vspheretypes.VirtualCenterConfig{ From 1c127159380bf0efffbd68fecfb40a3f1fa765bb Mon Sep 17 00:00:00 2001 From: Eng Zer Jun Date: Thu, 20 Jul 2023 22:10:19 +0800 Subject: [PATCH 339/489] Replace `github.com/ghodss/yaml` with `sigs.k8s.io/yaml` (#1683) At the time of making this commit, the package `github.com/ghodss/yaml` is no longer actively maintained. `sigs.k8s.io/yaml` is a permanent fork of `ghodss/yaml` and is actively maintained by Kubernetes SIG. Signed-off-by: Eng Zer Jun --- go.mod | 1 - go.sum | 1 - pkg/apis/cluster/v1alpha1/conversions/conversions_test.go | 2 +- .../v1alpha1/conversions/providerconfig_to_providerspec_test.go | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 741d50750..5fba6b994 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,6 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/digitalocean/godo v1.93.0 github.com/flatcar/container-linux-config-transpiler v0.9.4 - github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v1.2.4 github.com/go-logr/zapr v1.2.3 github.com/go-test/deep v1.0.8 diff --git a/go.sum b/go.sum index a4de3e560..de32ba4dd 100644 --- a/go.sum +++ b/go.sum @@ -240,7 +240,6 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= -github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= diff --git a/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go b/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go index fe87ba1b9..4c84f36c8 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go +++ b/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go @@ -23,7 +23,7 @@ import ( "os" "testing" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" machinesv1alpha1 "github.com/kubermatic/machine-controller/pkg/machines/v1alpha1" diff --git a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go b/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go index 7f290e02a..be61daf8a 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go +++ b/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go @@ -22,7 +22,7 @@ import ( "os" "testing" - "github.com/ghodss/yaml" + "sigs.k8s.io/yaml" testhelper "github.com/kubermatic/machine-controller/pkg/test" ) From d285b8ff85664d9e6e5035c1446d7ea3fee36f73 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 21 Jul 2023 19:31:18 +0500 Subject: [PATCH 340/489] VCD: Fix compute and sizing policies (#1685) Signed-off-by: Waleed Malik --- ...ware-cloud-director-machinedeployment.yaml | 6 +++++ .../provider/vmwareclouddirector/helper.go | 26 +++++++++---------- .../provider/vmwareclouddirector/provider.go | 2 ++ 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/examples/vmware-cloud-director-machinedeployment.yaml b/examples/vmware-cloud-director-machinedeployment.yaml index bdecf1769..eb342dceb 100644 --- a/examples/vmware-cloud-director-machinedeployment.yaml +++ b/examples/vmware-cloud-director-machinedeployment.yaml @@ -79,6 +79,12 @@ spec: diskBusType: "paravirtual" diskIOPS: 0 storageProfile: "*" + # Optional: SizingPolicy is the sizing policy to be used for machines created by this machine deployment. + # If left empty, default sizing policy if specified at OVDC/organization level is used. + sizingPolicy: "" + # Optional: PlacementPolicy is the placement policy to be used for machines created by this machine deployment. + # If left empty, default placement policy if specified at OVDC/organization level is used. + placementPolicy: "" operatingSystem: "ubuntu" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go index 99b5064a6..6c1980960 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go @@ -77,7 +77,7 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * } // 2. Retrieve Sizing and Placement Compute Policy if required. - computePolicy := vcdapitypes.ComputePolicy{} + var computePolicy *types.ComputePolicy if c.SizingPolicy != nil || c.PlacementPolicy != nil { allPolicies, err := org.GetAllVdcComputePolicies(url.Values{}) if err != nil { @@ -89,6 +89,9 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * if sizingPolicy == nil { return fmt.Errorf("sizing policy '%s' doesn't exist", *c.SizingPolicy) } + if computePolicy == nil { + computePolicy = &types.ComputePolicy{} + } computePolicy.VmSizingPolicy = &vcdapitypes.Reference{ HREF: sizingPolicy.VdcComputePolicy.ID, } @@ -99,6 +102,9 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * if placementPolicy == nil { return fmt.Errorf("placement policy '%s' doesn't exist", *c.PlacementPolicy) } + if computePolicy == nil { + computePolicy = &types.ComputePolicy{} + } computePolicy.VmPlacementPolicy = &vcdapitypes.Reference{ HREF: placementPolicy.VdcComputePolicy.ID, } @@ -106,15 +112,15 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * } // 3. Retrieve Storage Profile - storageProfileRef := vcdapitypes.Reference{} + var storageProfile *types.Reference if c.StorageProfile != nil && *c.StorageProfile != defaultStorageProfile { for _, sp := range vdc.Vdc.VdcStorageProfiles.VdcStorageProfile { if sp.Name == *c.StorageProfile || sp.ID == *c.StorageProfile { - storageProfileRef = vcdapitypes.Reference{HREF: sp.HREF, Name: sp.Name, ID: sp.ID} + storageProfile = sp break } } - if storageProfileRef.HREF == "" { + if storageProfile == nil { if err != nil { return fmt.Errorf("failed to get storage profile '%s': %w", *c.StorageProfile, err) } @@ -154,20 +160,12 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * }, }, }, + StorageProfile: storageProfile, + ComputePolicy: computePolicy, }, AllEULAsAccepted: true, } - // Add storage profile - if storageProfileRef.HREF != "" { - vAppRecomposition.SourcedItem.StorageProfile = &storageProfileRef - } - - // Add compute policy - if computePolicy.HREF != "" { - vAppRecomposition.SourcedItem.ComputePolicy = &computePolicy - } - apiEndpoint, err := url.Parse(vapp.VApp.HREF) if err != nil { return fmt.Errorf("error getting vapp href '%s': %w", c.Auth.URL, err) diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go index 5da253d8c..c67169efc 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -419,6 +419,8 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c.DiskBusType = rawConfig.DiskBusType c.StorageProfile = rawConfig.StorageProfile c.Metadata = rawConfig.Metadata + c.SizingPolicy = rawConfig.SizingPolicy + c.PlacementPolicy = rawConfig.PlacementPolicy return &c, pconfig, rawConfig, err } From 46144bd8e43cf4b86684e4df9b662481184589dd Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 24 Jul 2023 17:32:25 +0500 Subject: [PATCH 341/489] Update Kubernetes patch versions for E2E tests (#1691) Signed-off-by: Waleed Malik --- pkg/userdata/helper/common_test.go | 6 +- ...lden => download_binaries_v1.25.12.golden} | 2 +- .../testdata/download_binaries_v1.25.9.golden | 17 - .../testdata/download_binaries_v1.26.0.golden | 17 - ...olden => download_binaries_v1.26.7.golden} | 2 +- .../testdata/download_binaries_v1.27.3.golden | 17 - ...olden => download_binaries_v1.27.4.golden} | 2 +- ...emd_unit_version-v1.25.12-external.golden} | 0 ...blet_systemd_unit_version-v1.25.12.golden} | 0 ...stemd_unit_version-v1.26.0-external.golden | 36 -- ...kublet_systemd_unit_version-v1.26.0.golden | 35 -- ...stemd_unit_version-v1.26.4-external.golden | 36 -- ...kublet_systemd_unit_version-v1.26.4.golden | 35 -- ...stemd_unit_version-v1.26.6-external.golden | 36 -- ...kublet_systemd_unit_version-v1.26.6.golden | 35 -- ...temd_unit_version-v1.26.7-external.golden} | 0 ...ublet_systemd_unit_version-v1.26.7.golden} | 0 ...stemd_unit_version-v1.27.1-external.golden | 36 -- ...kublet_systemd_unit_version-v1.27.1.golden | 35 -- ...stemd_unit_version-v1.27.3-external.golden | 36 -- ...kublet_systemd_unit_version-v1.27.3.golden | 35 -- ...temd_unit_version-v1.27.4-external.golden} | 0 ...ublet_systemd_unit_version-v1.27.4.golden} | 0 pkg/userdata/ubuntu/provider_test.go | 6 +- ...sion-1.25.11.yaml => version-1.25.12.yaml} | 2 +- .../ubuntu/testdata/version-1.25.5.yaml | 458 ------------------ .../ubuntu/testdata/version-1.26.0.yaml | 458 ------------------ ...ersion-1.26.6.yaml => version-1.26.7.yaml} | 2 +- .../ubuntu/testdata/version-1.27.3.yaml | 457 ----------------- ...ersion-1.25.9.yaml => version-1.27.4.yaml} | 2 +- test/e2e/provisioning/all_e2e_test.go | 8 +- test/e2e/provisioning/helper.go | 6 +- 32 files changed, 19 insertions(+), 1798 deletions(-) rename pkg/userdata/helper/testdata/{download_binaries_v1.25.11.golden => download_binaries_v1.25.12.golden} (92%) delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.25.9.golden delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.26.0.golden rename pkg/userdata/helper/testdata/{download_binaries_v1.26.6.golden => download_binaries_v1.26.7.golden} (92%) delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.27.3.golden rename pkg/userdata/helper/testdata/{download_binaries_v1.25.5.golden => download_binaries_v1.27.4.golden} (92%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.25.11-external.golden => kublet_systemd_unit_version-v1.25.12-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.25.11.golden => kublet_systemd_unit_version-v1.25.12.golden} (100%) delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.0-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.0.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.4-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.4.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.6-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.6.golden rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.25.5-external.golden => kublet_systemd_unit_version-v1.26.7-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.25.5.golden => kublet_systemd_unit_version-v1.26.7.golden} (100%) delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.1-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.1.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.3-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.3.golden rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.25.9-external.golden => kublet_systemd_unit_version-v1.27.4-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.25.9.golden => kublet_systemd_unit_version-v1.27.4.golden} (100%) rename pkg/userdata/ubuntu/testdata/{version-1.25.11.yaml => version-1.25.12.yaml} (99%) delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.25.5.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.26.0.yaml rename pkg/userdata/ubuntu/testdata/{version-1.26.6.yaml => version-1.26.7.yaml} (99%) delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.27.3.yaml rename pkg/userdata/ubuntu/testdata/{version-1.25.9.yaml => version-1.27.4.yaml} (99%) diff --git a/pkg/userdata/helper/common_test.go b/pkg/userdata/helper/common_test.go index 970d058ed..5d1eaa2ca 100644 --- a/pkg/userdata/helper/common_test.go +++ b/pkg/userdata/helper/common_test.go @@ -26,8 +26,8 @@ var update = flag.Bool("update", false, "update testdata files") var ( versions = []*semver.Version{ - semver.MustParse("v1.25.11"), - semver.MustParse("v1.26.6"), - semver.MustParse("v1.27.3"), + semver.MustParse("v1.25.12"), + semver.MustParse("v1.26.7"), + semver.MustParse("v1.27.4"), } ) diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.25.11.golden b/pkg/userdata/helper/testdata/download_binaries_v1.25.12.golden similarity index 92% rename from pkg/userdata/helper/testdata/download_binaries_v1.25.11.golden rename to pkg/userdata/helper/testdata/download_binaries_v1.25.12.golden index 3930c6fce..4786c596e 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.25.11.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.25.12.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.25.11/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.25.12/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.25.9.golden b/pkg/userdata/helper/testdata/download_binaries_v1.25.9.golden deleted file mode 100644 index 864f2924f..000000000 --- a/pkg/userdata/helper/testdata/download_binaries_v1.25.9.golden +++ /dev/null @@ -1,17 +0,0 @@ -mkdir -p /opt/bin/ -mkdir -p /var/lib/calico -mkdir -p /etc/kubernetes/manifests -mkdir -p /etc/cni/net.d -mkdir -p /opt/cni/bin -if [ ! -f /opt/cni/bin/loopback ]; then - curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - -fi -if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.25.9/bin/linux/amd64/kubelet - chmod +x /opt/bin/kubelet -fi - -if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh -fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.26.0.golden b/pkg/userdata/helper/testdata/download_binaries_v1.26.0.golden deleted file mode 100644 index 2d3ea0401..000000000 --- a/pkg/userdata/helper/testdata/download_binaries_v1.26.0.golden +++ /dev/null @@ -1,17 +0,0 @@ -mkdir -p /opt/bin/ -mkdir -p /var/lib/calico -mkdir -p /etc/kubernetes/manifests -mkdir -p /etc/cni/net.d -mkdir -p /opt/cni/bin -if [ ! -f /opt/cni/bin/loopback ]; then - curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - -fi -if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.26.0/bin/linux/amd64/kubelet - chmod +x /opt/bin/kubelet -fi - -if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh -fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.26.6.golden b/pkg/userdata/helper/testdata/download_binaries_v1.26.7.golden similarity index 92% rename from pkg/userdata/helper/testdata/download_binaries_v1.26.6.golden rename to pkg/userdata/helper/testdata/download_binaries_v1.26.7.golden index 2e6cd2e12..c72528aeb 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.26.6.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.26.7.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.26.6/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.26.7/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.27.3.golden b/pkg/userdata/helper/testdata/download_binaries_v1.27.3.golden deleted file mode 100644 index 24403aa28..000000000 --- a/pkg/userdata/helper/testdata/download_binaries_v1.27.3.golden +++ /dev/null @@ -1,17 +0,0 @@ -mkdir -p /opt/bin/ -mkdir -p /var/lib/calico -mkdir -p /etc/kubernetes/manifests -mkdir -p /etc/cni/net.d -mkdir -p /opt/cni/bin -if [ ! -f /opt/cni/bin/loopback ]; then - curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - -fi -if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.27.3/bin/linux/amd64/kubelet - chmod +x /opt/bin/kubelet -fi - -if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh -fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.25.5.golden b/pkg/userdata/helper/testdata/download_binaries_v1.27.4.golden similarity index 92% rename from pkg/userdata/helper/testdata/download_binaries_v1.25.5.golden rename to pkg/userdata/helper/testdata/download_binaries_v1.27.4.golden index c824f0189..602bf7e26 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.25.5.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.27.4.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.25.5/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.27.4/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.11-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.12-external.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.11-external.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.12-external.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.11.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.12.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.11.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.12.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.0-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.0-external.golden deleted file mode 100644 index 50f4f5138..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.0-external.golden +++ /dev/null @@ -1,36 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.0.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.0.golden deleted file mode 100644 index e70567560..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.0.golden +++ /dev/null @@ -1,35 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.4-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.4-external.golden deleted file mode 100644 index 50f4f5138..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.4-external.golden +++ /dev/null @@ -1,36 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.4.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.4.golden deleted file mode 100644 index e70567560..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.4.golden +++ /dev/null @@ -1,35 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.6-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.6-external.golden deleted file mode 100644 index 50f4f5138..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.6-external.golden +++ /dev/null @@ -1,36 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.6.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.6.golden deleted file mode 100644 index e70567560..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.6.golden +++ /dev/null @@ -1,35 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.5-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.7-external.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.5-external.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.7-external.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.5.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.7.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.5.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.7.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.1-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.1-external.golden deleted file mode 100644 index 50f4f5138..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.1-external.golden +++ /dev/null @@ -1,36 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.1.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.1.golden deleted file mode 100644 index e70567560..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.1.golden +++ /dev/null @@ -1,35 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.3-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.3-external.golden deleted file mode 100644 index 50f4f5138..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.3-external.golden +++ /dev/null @@ -1,36 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.3.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.3.golden deleted file mode 100644 index e70567560..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.3.golden +++ /dev/null @@ -1,35 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.9-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.4-external.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.9-external.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.4-external.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.9.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.4.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.9.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.4.golden diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 513206b94..d07e31eae 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -129,9 +129,9 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.25.11"), - semver.MustParse("v1.26.6"), - semver.MustParse("v1.27.3"), + semver.MustParse("v1.25.12"), + semver.MustParse("v1.26.7"), + semver.MustParse("v1.27.4"), } var tests []userDataTestCase diff --git a/pkg/userdata/ubuntu/testdata/version-1.25.11.yaml b/pkg/userdata/ubuntu/testdata/version-1.25.12.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.25.11.yaml rename to pkg/userdata/ubuntu/testdata/version-1.25.12.yaml index 11364d00d..f9527dd19 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.25.11.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.25.12.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.11}" + KUBE_VERSION="${KUBE_VERSION:-v1.25.12}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml deleted file mode 100644 index 6fbbae81d..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.25.5.yaml +++ /dev/null @@ -1,458 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml deleted file mode 100644 index e284cdb1b..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.26.0.yaml +++ /dev/null @@ -1,458 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.26.6.yaml b/pkg/userdata/ubuntu/testdata/version-1.26.7.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.26.6.yaml rename to pkg/userdata/ubuntu/testdata/version-1.26.7.yaml index 5c3a67922..e604cff52 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.26.6.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.26.7.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.7}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.27.3.yaml b/pkg/userdata/ubuntu/testdata/version-1.27.3.yaml deleted file mode 100644 index cc5235c70..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.27.3.yaml +++ /dev/null @@ -1,457 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.25.9.yaml b/pkg/userdata/ubuntu/testdata/version-1.27.4.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.25.9.yaml rename to pkg/userdata/ubuntu/testdata/version-1.27.4.yaml index f1d40f1f9..2d1ebffa1 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.25.9.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.27.4.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.4}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index bfcbe76ff..d9356ccfe 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -343,7 +343,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.26. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.26.6", "1.27.3"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.26.7", "1.27.4"))) runScenarios(t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -423,7 +423,7 @@ func TestAWSProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := Not(VersionSelector("1.27.3")) + selector := Not(VersionSelector("1.27.4")) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -477,7 +477,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.3"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.4"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -499,7 +499,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.3"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.4"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 8c4231085..25a56e696 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,9 +33,9 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.25.11"), - semver.MustParse("v1.26.6"), - semver.MustParse("v1.27.3"), + semver.MustParse("v1.25.12"), + semver.MustParse("v1.26.7"), + semver.MustParse("v1.27.4"), } operatingSystems = []providerconfigtypes.OperatingSystem{ From 5a9f7387aedd3d25076068d0e3c8d3840afcb276 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 24 Jul 2023 23:23:25 +0500 Subject: [PATCH 342/489] Remove support for docker (#1692) * Remove support for docker Signed-off-by: Waleed Malik * Refactored code Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- cmd/machine-controller/main.go | 6 +- examples/machine-controller.yaml | 4 +- hack/ci/setup-machine-controller-in-kind.sh | 4 +- hack/run-machine-controller.sh | 5 +- pkg/containerruntime/config.go | 4 + pkg/containerruntime/containerruntime.go | 46 +--- pkg/containerruntime/docker.go | 197 ------------------ pkg/userdata/amzn2/provider.go | 8 - pkg/userdata/centos/provider.go | 8 - pkg/userdata/flatcar/provider.go | 18 -- pkg/userdata/rhel/provider.go | 8 - pkg/userdata/rockylinux/provider.go | 8 - pkg/userdata/ubuntu/provider.go | 8 - .../kubelet-version-without-v-prefix.yaml | 44 ++-- .../ubuntu/testdata/multiple-ssh-keys.yaml | 44 ++-- .../openstack-overwrite-cloud-config.yaml | 44 ++-- .../ubuntu/testdata/vsphere-mirrors.yaml | 44 ++-- .../ubuntu/testdata/vsphere-proxy.yaml | 51 +++-- pkg/userdata/ubuntu/testdata/vsphere.yaml | 44 ++-- 19 files changed, 205 insertions(+), 390 deletions(-) delete mode 100644 pkg/containerruntime/docker.go diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 1e591ff7a..f663e2aa3 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -177,7 +177,7 @@ func main() { flag.StringVar(&nodeRegistryMirrors, "node-registry-mirrors", "", "Comma separated list of Docker image mirrors") flag.StringVar(&nodePauseImage, "node-pause-image", "", "Image for the pause container including tag. If not set, the kubelet default will be used: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/") flag.String("node-kubelet-repository", "quay.io/kubermatic/kubelet", "[NO-OP] Repository for the kubelet container. Has no effects.") - flag.StringVar(&nodeContainerRuntime, "node-container-runtime", "docker", "container-runtime to deploy") + flag.StringVar(&nodeContainerRuntime, "node-container-runtime", "containerd", "container-runtime to deploy") flag.StringVar(&nodeContainerdVersion, "node-containerd-version", "", "version of containerd to deploy") flag.Var(&nodeContainerdRegistryMirrors, "node-containerd-registry-mirrors", "Configure registry mirrors endpoints. Can be used multiple times to specify multiple mirrors") flag.StringVar(&caBundleFile, "ca-bundle", "", "path to a file containing all PEM-encoded CA certificates (will be used instead of the host's certificates if set)") @@ -195,6 +195,10 @@ func main() { log.Fatalf("Invalid options: %v", err) } + if nodeContainerRuntime != "containerd" { + log.Fatalf("%s not supported; containerd is the only supported container runtime", nodeContainerRuntime) + } + rawLog := machinecontrollerlog.New(logFlags.Debug, logFlags.Format) log := rawLog.Sugar() diff --git a/examples/machine-controller.yaml b/examples/machine-controller.yaml index e3848f2fc..4ccb06b56 100644 --- a/examples/machine-controller.yaml +++ b/examples/machine-controller.yaml @@ -240,7 +240,7 @@ spec: - -cluster-dns=10.10.10.10 - -metrics-address=0.0.0.0:8080 - -health-probe-address=0.0.0.0:8085 - - -use-osm=true + - -use-external-bootstrap=true # Machines that fail to join the cluster within this timeout and # are owned by a MachineSet will get deleted so the MachineSet # controller re-creates them @@ -284,7 +284,7 @@ spec: # on debug level, full Machine objects with inline credentials might be logged, beware! - -log-debug=false - -log-format=json # json or console - - -use-osm=true + - -use-external-bootstrap=true - -namespace=kube-system - -listen-address=0.0.0.0:9876 volumeMounts: diff --git a/hack/ci/setup-machine-controller-in-kind.sh b/hack/ci/setup-machine-controller-in-kind.sh index 29cb6cf90..b1aea2db2 100755 --- a/hack/ci/setup-machine-controller-in-kind.sh +++ b/hack/ci/setup-machine-controller-in-kind.sh @@ -59,9 +59,9 @@ if [ ! -f machine-controller-deployed ]; then url="-override-bootstrap-kubelet-apiserver=$MASTER_URL" sed -i "s;-node-csr-approver=true;$url;g" examples/machine-controller.yaml - # Ensure that we update `use-osm` flag if OSM is disabled + # Ensure that we update `use-external-bootstrap` flag if OSM is disabled if [[ "$OPERATING_SYSTEM_MANAGER" == "false" ]]; then - sed -i "s;-use-osm=true;-use-osm=false;g" examples/machine-controller.yaml + sed -i "s;-use-external-bootstrap=true;-use-external-bootstrap=false;g" examples/machine-controller.yaml fi # e2e tests logs are primarily read by humans, if ever diff --git a/hack/run-machine-controller.sh b/hack/run-machine-controller.sh index 2eaff7d45..fd8bb620c 100755 --- a/hack/run-machine-controller.sh +++ b/hack/run-machine-controller.sh @@ -21,7 +21,7 @@ set -e MC_KUBECONFIG=${MC_KUBECONFIG:-$(dirname $0)/../.kubeconfig} # If you want to use the default kubeconfig `export MC_KUBECONFIG=$KUBECONFIG` -# `-use-osm` flag can be removed to use legacy userdata that is generated by machine-controller. +# `-use-external-bootstrap` flag can be removed to use legacy userdata that is generated by machine-controller. make -C $(dirname $0)/.. build-machine-controller $(dirname $0)/../machine-controller \ @@ -32,5 +32,4 @@ $(dirname $0)/../machine-controller \ -enable-profiling \ -metrics-address=0.0.0.0:8080 \ -health-probe-address=0.0.0.0:8085 \ - -use-osm=true \ - -node-container-runtime=containerd + -use-external-bootstrap=true diff --git a/pkg/containerruntime/config.go b/pkg/containerruntime/config.go index f93a54d9f..65b8815be 100644 --- a/pkg/containerruntime/config.go +++ b/pkg/containerruntime/config.go @@ -39,6 +39,10 @@ type Opts struct { ContainerdRegistryMirrors RegistryMirrorsFlags } +type DockerCfgJSON struct { + Auths map[string]AuthConfig `json:"auths,omitempty"` +} + func BuildConfig(opts Opts) (Config, error) { var insecureRegistries []string for _, registry := range strings.Split(opts.InsecureRegistries, ",") { diff --git a/pkg/containerruntime/containerruntime.go b/pkg/containerruntime/containerruntime.go index b7fe47979..a6c3c5c9a 100644 --- a/pkg/containerruntime/containerruntime.go +++ b/pkg/containerruntime/containerruntime.go @@ -23,7 +23,6 @@ import ( ) const ( - dockerName = "docker" containerdName = "containerd" ) @@ -65,18 +64,7 @@ func withContainerdVersion(version string) Opt { func get(containerRuntimeName string, opts ...Opt) Config { cfg := Config{} - - switch containerRuntimeName { - case dockerName: - cfg.Docker = &Docker{} - cfg.Containerd = nil - case containerdName: - cfg.Containerd = &Containerd{} - cfg.Docker = nil - default: - cfg.Docker = &Docker{} - cfg.Containerd = nil - } + cfg.Containerd = &Containerd{} for _, o := range opts { o(&cfg) @@ -86,7 +74,6 @@ func get(containerRuntimeName string, opts ...Opt) Config { } type Config struct { - Docker *Docker `json:",omitempty"` Containerd *Containerd `json:",omitempty"` InsecureRegistries []string `json:",omitempty"` RegistryMirrors map[string][]string `json:",omitempty"` @@ -113,26 +100,10 @@ type AuthConfig struct { } func (cfg Config) String() string { - switch { - case cfg.Containerd != nil: - return containerdName - case cfg.Docker != nil: - return dockerName - } - - return dockerName + return containerdName } func (cfg Config) Engine(kubeletVersion *semver.Version) Engine { - docker := &Docker{ - insecureRegistries: cfg.InsecureRegistries, - registryMirrors: cfg.RegistryMirrors["docker.io"], - containerLogMaxFiles: cfg.ContainerLogMaxFiles, - containerLogMaxSize: cfg.ContainerLogMaxSize, - registryCredentials: cfg.RegistryCredentials, - containerdVersion: cfg.ContainerdVersion, - } - containerd := &Containerd{ insecureRegistries: cfg.InsecureRegistries, registryMirrors: cfg.RegistryMirrors, @@ -140,16 +111,5 @@ func (cfg Config) Engine(kubeletVersion *semver.Version) Engine { registryCredentials: cfg.RegistryCredentials, version: cfg.ContainerdVersion, } - - moreThan124, _ := semver.NewConstraint(">= 1.24") - - switch { - case moreThan124.Check(kubeletVersion) || cfg.Containerd != nil: - // docker support has been removed in Kubernetes 1.24 - return containerd - case cfg.Docker != nil: - return docker - } - - return docker + return containerd } diff --git a/pkg/containerruntime/docker.go b/pkg/containerruntime/docker.go deleted file mode 100644 index 801e8e0ea..000000000 --- a/pkg/containerruntime/docker.go +++ /dev/null @@ -1,197 +0,0 @@ -/* -Copyright 2020 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package containerruntime - -import ( - "encoding/json" - "fmt" - "strings" - "text/template" - - "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - "github.com/kubermatic/machine-controller/pkg/userdata/helper" -) - -const ( - LegacyDockerContainerdVersion = "1.4*" - DefaultDockerContainerdVersion = "1.6*" - DefaultDockerVersion = "20.10" - LegacyDockerVersion = "19.03" -) - -type Docker struct { - insecureRegistries []string - registryMirrors []string - containerLogMaxFiles string - containerLogMaxSize string - registryCredentials map[string]AuthConfig - containerdVersion string -} - -type DockerCfgJSON struct { - Auths map[string]AuthConfig `json:"auths,omitempty"` -} - -func (eng *Docker) Config() (string, error) { - return helper.DockerConfig(eng.insecureRegistries, eng.registryMirrors, eng.containerLogMaxFiles, eng.containerLogMaxSize) -} - -func (eng *Docker) ConfigFileName() string { - return "/etc/docker/daemon.json" -} - -func (eng *Docker) AuthConfig() (string, error) { - if eng.registryCredentials == nil { - return "", nil - } - - cfg := DockerCfgJSON{ - Auths: eng.registryCredentials, - } - b, err := json.MarshalIndent(cfg, "", " ") - - return string(b), err -} - -func (eng *Docker) AuthConfigFileName() string { - return "/root/.docker/config.json" -} - -func (eng *Docker) KubeletFlags() []string { - return []string{ - "--container-runtime=docker", - "--container-runtime-endpoint=unix:///var/run/dockershim.sock", - } -} - -func (eng *Docker) ScriptFor(os types.OperatingSystem) (string, error) { - var buf strings.Builder - - args := struct { - DockerVersion string - ContainerdVersion string - }{ - DockerVersion: DefaultDockerVersion, - ContainerdVersion: DefaultDockerContainerdVersion, - } - - if eng.containerdVersion != "" { - args.ContainerdVersion = eng.containerdVersion - } - - switch os { - case types.OperatingSystemAmazonLinux2: - args.ContainerdVersion = LegacyDockerContainerdVersion - err := dockerAmazonTemplate.Execute(&buf, args) - return buf.String(), err - case types.OperatingSystemCentOS, types.OperatingSystemRHEL, types.OperatingSystemRockyLinux: - err := dockerYumTemplate.Execute(&buf, args) - return buf.String(), err - case types.OperatingSystemUbuntu: - err := dockerAptTemplate.Execute(&buf, args) - return buf.String(), err - case types.OperatingSystemFlatcar: - err := dockerFlatcarTemplate.Execute(&buf, args) - return buf.String(), err - } - - return "", fmt.Errorf("unknown OS: %s", os) -} - -var ( - dockerFlatcarTemplate = template.Must(template.New("docker-flatcar").Parse(` -systemctl daemon-reload -systemctl enable --now docker -`)) - - dockerAmazonTemplate = template.Must(template.New("docker-yum-amzn2").Parse(` -mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - -cat < Date: Wed, 26 Jul 2023 14:02:48 +0300 Subject: [PATCH 343/489] Fix version of k8s.io/client-go in go.mod (#1693) Signed-off-by: Artiom Diomin --- go.mod | 5 +-- go.sum | 99 +++++++++++++++++++++++++++++++++++++++++----------------- 2 files changed, 71 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 5fba6b994..9e3e35186 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( k8s.io/api v0.26.4 k8s.io/apiextensions-apiserver v0.26.4 k8s.io/apimachinery v0.26.4 - k8s.io/client-go v12.0.0+incompatible + k8s.io/client-go v0.26.4 k8s.io/cloud-provider v0.26.4 k8s.io/klog v1.0.0 k8s.io/kubelet v0.26.4 @@ -188,6 +188,3 @@ require ( sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) - -// TODO: Upgrade to 0.27.x once sigs.k8s.io/controller-runtime 0.14.7 is released. For more details https://github.com/kubernetes/client-go/issues/1245#issuecomment-1523434471 -replace k8s.io/client-go => k8s.io/client-go v0.26.4 diff --git a/go.sum b/go.sum index de32ba4dd..7e5011d53 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,11 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww= cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= @@ -50,9 +55,11 @@ github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9Eb github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= @@ -170,6 +177,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= @@ -212,13 +220,12 @@ github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkg github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= @@ -234,6 +241,8 @@ github.com/flatcar/container-linux-config-transpiler v0.9.4/go.mod h1:LxanhPvXkW github.com/flatcar/ignition v0.36.2 h1:xGHgScUe0P4Fkprjqv7L2CE58emiQgP833OCCn9z2v4= github.com/flatcar/ignition v0.36.2/go.mod h1:uk1tpzLFRXus4RrvzgMI+IqmmB8a/RGFSBlI+tMTbbA= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -277,7 +286,6 @@ github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaL github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= @@ -327,6 +335,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -342,6 +351,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= @@ -375,6 +385,7 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -383,6 +394,10 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= @@ -431,7 +446,7 @@ github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= @@ -555,9 +570,6 @@ github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042 github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= -github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= -github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/ginkgo/v2 v2.10.0 h1:sfUl4qgLdvkChZrWCYndY2EAu9BRIw1YphNAzy1VNWs= github.com/onsi/ginkgo/v2 v2.10.0/go.mod h1:UDQOh5wbQUlMnkLfVaIUMtQ1Vus92oM+P2JX1aulgcE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -566,10 +578,6 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= -github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= -github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= -github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -739,6 +747,8 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= @@ -784,6 +794,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -793,7 +804,6 @@ golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= @@ -819,6 +829,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -828,11 +839,11 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -867,8 +878,12 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= @@ -882,11 +897,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -894,8 +906,14 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -953,10 +971,18 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -974,21 +1000,16 @@ golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -996,11 +1017,11 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1058,15 +1079,20 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200815165600-90abf76919f3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1091,6 +1117,11 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.105.0 h1:t6P9Jj+6XTn4U9I2wycQai6Q/Kz7iOT+QzjJ3G2V4x8= google.golang.org/api v0.105.0/go.mod h1:qh7eD5FJks5+BcE+cjBIm6Gz8vioK7EHvnlniqXBnqI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -1133,7 +1164,17 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= @@ -1150,9 +1191,13 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= @@ -1173,7 +1218,6 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= @@ -1233,6 +1277,7 @@ k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2U k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apimachinery v0.26.4 h1:rZccKdBLg9vP6J09JD+z8Yr99Ce8gk3Lbi9TCx05Jzs= k8s.io/apimachinery v0.26.4/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= +k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= k8s.io/client-go v0.26.4 h1:/7P/IbGBuT73A+G97trf44NTPSNqvuBREpOfdLbHvD4= k8s.io/client-go v0.26.4/go.mod h1:6qOItWm3EwxJdl/8p5t7FWtWUOwyMdA8N9ekbW4idpI= k8s.io/cloud-provider v0.26.4 h1:mqN4vhC4mRoMi+ujI92ImkIOuYS7ZS55FvXB10d6Wp4= @@ -1251,13 +1296,11 @@ k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= -k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a h1:gmovKNur38vgoWfGtP5QOGNOA7ki4n6qNYoFAgMlNvg= k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= k8s.io/kubelet v0.26.4 h1:SEQPfjN4lu4uL9O8NdeN7Aum3liQ4kOnp/yC3jMRMUo= @@ -1266,7 +1309,6 @@ k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v0.58.0 h1:qeNeRtD6AIJ5WVJuRXajmmXtnrO5dYchy+hpCm6QwhE= @@ -1283,7 +1325,6 @@ sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92 sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= -sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= From 6a870df173ad007fdea8ec4f14fcf3daf6791089 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 15 Aug 2023 16:40:52 +0500 Subject: [PATCH 344/489] Upgrade to Go 1.21 (#1698) * Upgrade to Go 1.21 Signed-off-by: Waleed Malik * Update fixtures and yamllint config Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- .golangci.yml | 8 +++ .prow/e2e-features.yaml | 8 +-- .prow/postsubmits.yaml | 4 +- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 16 +++--- .prow/provider-azure.yaml | 6 +- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 +- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 8 +-- .prow/verify.yaml | 10 ++-- .yamllint.conf | 3 + Dockerfile | 2 +- Makefile | 2 +- cmd/machine-controller/main.go | 2 +- go.mod | 2 +- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- pkg/admission/admission.go | 2 +- .../v1alpha1/conversions/conversions_test.go | 3 +- .../providerconfig_to_providerspec_test.go | 4 +- .../cluster/v1alpha1/migrations/migrations.go | 6 +- .../provider/alibaba/provider.go | 6 +- .../provider/anexia/helper_test.go | 3 +- pkg/cloudprovider/provider/aws/provider.go | 4 +- .../provider/azure/create_delete_resources.go | 4 +- pkg/cloudprovider/provider/azure/provider.go | 8 +-- .../provider/digitalocean/provider.go | 6 +- .../provider/equinixmetal/provider.go | 6 +- pkg/cloudprovider/provider/gce/provider.go | 4 +- .../provider/hetzner/provider.go | 6 +- .../provider/kubevirt/provider.go | 12 ++-- .../provider/kubevirt/provider_test.go | 2 +- pkg/cloudprovider/provider/linode/provider.go | 6 +- pkg/cloudprovider/provider/nutanix/client.go | 3 +- .../provider/nutanix/provider.go | 12 ++-- .../provider/opennebula/provider.go | 7 +-- .../provider/openstack/helper.go | 4 +- .../provider/openstack/provider.go | 4 +- .../provider/scaleway/provider.go | 8 +-- .../provider/vmwareclouddirector/provider.go | 10 ++-- pkg/cloudprovider/provider/vsphere/helper.go | 8 +-- .../provider/vsphere/helper_test.go | 8 +-- .../provider/vsphere/provider.go | 5 +- pkg/cloudprovider/provider/vultr/provider.go | 6 +- pkg/containerruntime/containerruntime.go | 6 +- pkg/controller/machine/controller.go | 12 ++-- .../machinedeployment/controller.go | 57 +------------------ pkg/controller/machinedeployment/rolling.go | 5 +- pkg/controller/machinedeployment/sync.go | 37 +++--------- pkg/userdata/amzn2/provider.go | 4 +- pkg/userdata/amzn2/provider_test.go | 2 +- pkg/userdata/centos/provider.go | 4 +- pkg/userdata/centos/provider_test.go | 2 +- pkg/userdata/flatcar/provider.go | 6 +- pkg/userdata/flatcar/provider_test.go | 2 +- pkg/userdata/helper/kubelet.go | 2 +- pkg/userdata/helper/template_functions.go | 4 +- pkg/userdata/rhel/provider.go | 4 +- pkg/userdata/rhel/provider_test.go | 2 +- pkg/userdata/rockylinux/provider.go | 4 +- pkg/userdata/rockylinux/provider_test.go | 2 +- pkg/userdata/ubuntu/provider.go | 4 +- pkg/userdata/ubuntu/provider_test.go | 2 +- test/e2e/provisioning/helper.go | 16 +++--- test/e2e/provisioning/migrateuidscenario.go | 2 +- 74 files changed, 178 insertions(+), 259 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index cc14d4190..222c27acb 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -44,6 +44,14 @@ linters: - whitespace disable-all: true +linters-settings: + depguard: + rules: + main: + deny: + - { pkg: io/ioutil, desc: https://go.dev/doc/go1.16#ioutil } + - { pkg: github.com/ghodss/yaml, desc: use sigs.k8s.io/yaml instead } + issues: exclude: - should have comment or be unexported diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index ad796bc18..c4ea60b65 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -91,7 +91,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -118,7 +118,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 920e15064..8e7df95ec 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index a458e8333..ec5a62fe7 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 8695fe2cc..926dd2e5c 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index b784d50e2..511fa1636 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -156,7 +156,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -187,7 +187,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -218,7 +218,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -249,7 +249,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 0e99aaf01..d93235398 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index f1b16e888..a40f42393 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index e27aa4a98..dca573aa3 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index d71c90168..03565e3c7 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 2fbe5ea24..3ca308140 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 3da41e259..3f8ced0c8 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 9163a84c6..f8bccf074 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 876ad33ec..63f8059c6 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index b504a0ecf..f89a151ba 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index ea094e84e..789eeee80 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 11d74385a..7ef994489 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 28e7a7310..748d199a2 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 033e183e0..b29c74dda 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.20.5 + - image: golang:1.21.0 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.20.5 + - image: golang:1.21.0 command: - make args: @@ -63,7 +63,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golangci/golangci-lint:v1.50.1 + - image: golangci/golangci-lint:v1.54.1 command: - make args: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.20.5 + - image: golang:1.21.0 command: - make args: diff --git a/.yamllint.conf b/.yamllint.conf index dda206acc..3c79f7897 100644 --- a/.yamllint.conf +++ b/.yamllint.conf @@ -5,3 +5,6 @@ rules: document-start: disable comments: disable line-length: disable + +ignore: | + .golangci.yml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 07f311f23..04ad940d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.20.5 +ARG GO_VERSION=1.21.0 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index a409c2cf5..1697eb852 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.20.5 +GO_VERSION ?= 1.21.0 GOOS ?= $(shell go env GOOS) diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index f663e2aa3..a3c1761f4 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -407,7 +407,7 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { } // Migrate MachinesV1Alpha1Machine to ClusterV1Alpha1Machine. - if err := migrations.MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary(ctx, bs.opt.log, client, bs.opt.kubeClient, providerData); err != nil { + if err := migrations.MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary(ctx, bs.opt.log, client, providerData); err != nil { return fmt.Errorf("migration to clusterv1alpha1 failed: %w", err) } diff --git a/go.mod b/go.mod index 9e3e35186..3a350354c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/kubermatic/machine-controller -go 1.19 +go 1.20 require ( cloud.google.com/go/logging v1.6.1 diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 437cc1e4e..2147af2b0 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.20.5 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.21.0 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index bffa632f1..cc8787f18 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.20-node-18-kind-0.18-7 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 containerize ./hack/verify-licenses.sh go mod vendor diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go index d972fd9af..bf563bb6e 100644 --- a/pkg/admission/admission.go +++ b/pkg/admission/admission.go @@ -92,7 +92,7 @@ func (build Builder) Build() (*http.Server, error) { }, nil } -func healthZHandler(w http.ResponseWriter, r *http.Request) { +func healthZHandler(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) } diff --git a/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go b/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go index 4c84f36c8..3ba9ce84e 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go +++ b/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go @@ -23,12 +23,11 @@ import ( "os" "testing" - "sigs.k8s.io/yaml" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" machinesv1alpha1 "github.com/kubermatic/machine-controller/pkg/machines/v1alpha1" kyaml "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/yaml" ) var update = flag.Bool("update", false, "update .testdata files") diff --git a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go b/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go index be61daf8a..902a11cd2 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go +++ b/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go @@ -22,9 +22,9 @@ import ( "os" "testing" - "sigs.k8s.io/yaml" - testhelper "github.com/kubermatic/machine-controller/pkg/test" + + "sigs.k8s.io/yaml" ) func Test_Convert_MachineDeployment_ProviderConfig_To_ProviderSpec(t *testing.T) { diff --git a/pkg/apis/cluster/v1alpha1/migrations/migrations.go b/pkg/apis/cluster/v1alpha1/migrations/migrations.go index 84cab3a91..a65cb91af 100644 --- a/pkg/apis/cluster/v1alpha1/migrations/migrations.go +++ b/pkg/apis/cluster/v1alpha1/migrations/migrations.go @@ -45,7 +45,6 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" dynamicclient "k8s.io/client-go/dynamic" - "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/cache" @@ -148,7 +147,6 @@ func MigrateProviderConfigToProviderSpecIfNecessary(ctx context.Context, log *za func MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary( ctx context.Context, log *zap.SugaredLogger, client ctrlruntimeclient.Client, - kubeClient kubernetes.Interface, providerData *cloudprovidertypes.ProviderData) error { var ( cachePopulatingInterval = 15 * time.Second @@ -192,7 +190,7 @@ func MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary( return fmt.Errorf("error when checking for existence of 'machines.cluster.k8s.io' crd: %w", err) } - if err := migrateMachines(ctx, log, client, kubeClient, providerData); err != nil { + if err := migrateMachines(ctx, log, client, providerData); err != nil { return fmt.Errorf("failed to migrate machines: %w", err) } crdLog.Info("Attempting to delete CRD") @@ -203,7 +201,7 @@ func MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary( return nil } -func migrateMachines(ctx context.Context, log *zap.SugaredLogger, client ctrlruntimeclient.Client, kubeClient kubernetes.Interface, providerData *cloudprovidertypes.ProviderData) error { +func migrateMachines(ctx context.Context, log *zap.SugaredLogger, client ctrlruntimeclient.Client, providerData *cloudprovidertypes.ProviderData) error { log.Info("Starting migration for machine.machines.k8s.io/v1alpha1 to machine.cluster.k8s.io/v1alpha1") // Get machinesv1Alpha1Machines diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index 54ff507ca..f66e7b848 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -155,7 +155,7 @@ func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, machineSpec return nil } -func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -200,7 +200,7 @@ func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *cluster return nil, fmt.Errorf("instance %v is not ready", foundInstance.InstanceId) } -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } @@ -342,7 +342,7 @@ func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, machine * return nil } -func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/anexia/helper_test.go b/pkg/cloudprovider/provider/anexia/helper_test.go index c6256fa20..c4d41e02a 100644 --- a/pkg/cloudprovider/provider/anexia/helper_test.go +++ b/pkg/cloudprovider/provider/anexia/helper_test.go @@ -20,8 +20,6 @@ import ( "encoding/json" "testing" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/gophercloud/gophercloud/testhelper" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" @@ -31,6 +29,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/providerconfig/types" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 6a61988e4..55449c4f7 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -535,7 +535,7 @@ func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.Machin return spec, err } -func (p *provider) Validate(ctx context.Context, log *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { config, pc, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -665,7 +665,7 @@ func areVpcDNSHostnamesEnabled(ctx context.Context, client *ec2.Client, id strin return *out.EnableDnsHostnames.Value, nil } -func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/azure/create_delete_resources.go b/pkg/cloudprovider/provider/azure/create_delete_resources.go index 4d0d80129..242fd765f 100644 --- a/pkg/cloudprovider/provider/azure/create_delete_resources.go +++ b/pkg/cloudprovider/provider/azure/create_delete_resources.go @@ -152,7 +152,7 @@ func deleteDisksByMachineUID(ctx context.Context, c *config, machineUID types.UI return fmt.Errorf("failed to get disks client: %w", err) } - matchingDisks, err := getDisksByMachineUID(ctx, disksClient, c, machineUID) + matchingDisks, err := getDisksByMachineUID(ctx, disksClient, machineUID) if err != nil { return err } @@ -171,7 +171,7 @@ func deleteDisksByMachineUID(ctx context.Context, c *config, machineUID types.UI return nil } -func getDisksByMachineUID(ctx context.Context, disksClient *compute.DisksClient, c *config, UID types.UID) ([]compute.Disk, error) { +func getDisksByMachineUID(ctx context.Context, disksClient *compute.DisksClient, UID types.UID) ([]compute.Disk, error) { list, err := disksClient.List(ctx) if err != nil { return nil, fmt.Errorf("failed to list disks: %w", err) diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 819f7465a..461693401 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -970,7 +970,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri return s, "azure", nil } -func validateDiskSKUs(ctx context.Context, c *config, sku compute.ResourceSku) error { +func validateDiskSKUs(_ context.Context, c *config, sku compute.ResourceSku) error { if c.OSDiskSKU != nil || c.DataDiskSKU != nil { if c.OSDiskSKU != nil { if _, ok := osDiskSKUs[*c.OSDiskSKU]; !ok { @@ -1002,7 +1002,7 @@ func validateDiskSKUs(ctx context.Context, c *config, sku compute.ResourceSku) e return nil } -func validateSKUCapabilities(ctx context.Context, c *config, sku compute.ResourceSku) error { +func validateSKUCapabilities(_ context.Context, c *config, sku compute.ResourceSku) error { if c.EnableAcceleratedNetworking != nil && *c.EnableAcceleratedNetworking { if !SKUHasCapability(sku, capabilityAcceleratedNetworking) { return fmt.Errorf("VM size %q does not support accelerated networking", c.VMSize) @@ -1169,7 +1169,7 @@ func (p *provider) MigrateUID(ctx context.Context, log *zap.SugaredLogger, machi return fmt.Errorf("failed to get disks client: %w", err) } - disks, err := getDisksByMachineUID(ctx, disksClient, config, machine.UID) + disks, err := getDisksByMachineUID(ctx, disksClient, machine.UID) if err != nil { return fmt.Errorf("failed to get disks: %w", err) } @@ -1217,7 +1217,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } -func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index d6a2c554a..0f686f2e1 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -276,7 +276,7 @@ func uploadRandomSSHPublicKey(ctx context.Context, service godo.KeysService) (st return newDoKey.Fingerprint, nil } -func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -477,7 +477,7 @@ func (p *provider) MigrateUID(ctx context.Context, _ *zap.SugaredLogger, machine return nil } -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } @@ -558,6 +558,6 @@ func doStatusAndErrToTerminalError(status int, err error) error { } } -func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index 2bbcb6aa5..2832be59a 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -240,7 +240,7 @@ func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec cluste return nil } -func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, _, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -367,7 +367,7 @@ func (p *provider) MigrateUID(_ context.Context, log *zap.SugaredLogger, machine return nil } -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } @@ -383,7 +383,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } -func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 62190dbb1..1e1cf46ae 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -304,7 +304,7 @@ func (p *Provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * } // Cleanup deletes the instance associated with the machine and all associated resources. -func (p *Provider) Cleanup(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *Provider) Cleanup(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { // Read configuration. cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) if err != nil { @@ -397,7 +397,7 @@ func (p *Provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, machine * } // SetMetricsForMachines allows providers to provide provider-specific metrics. -func (p *Provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { +func (p *Provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 83521dccf..697cdf6d4 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -260,7 +260,7 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus return nil } -func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -523,7 +523,7 @@ func (p *provider) MigrateUID(ctx context.Context, log *zap.SugaredLogger, machi return nil } -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } @@ -634,6 +634,6 @@ func (p *provider) publicIPsAssignment(rawConfig *hetznertypes.RawConfig) (bool, return assignIPv4, assignIPv6, nil } -func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 4d23f2c98..b9face9b0 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -585,7 +585,7 @@ func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *cl userDataSecretName := fmt.Sprintf("userdata-%s-%s", machine.Name, strconv.Itoa(int(time.Now().Unix()))) virtualMachine, err := p.newVirtualMachine(ctx, c, pc, machine, userDataSecretName, userdata, - machineDeploymentNameAndRevisionForMachineGetter(ctx, machine, data.Client), randomMacAddressGetter, sigClient) + machineDeploymentNameAndRevisionForMachineGetter(ctx, machine, data.Client), randomMacAddressGetter) if err != nil { return nil, fmt.Errorf("could not create a VirtualMachine manifest %w", err) } @@ -608,8 +608,8 @@ func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *cl return &kubeVirtServer{}, nil } -func (p *provider) newVirtualMachine(ctx context.Context, c *Config, pc *providerconfigtypes.Config, machine *clusterv1alpha1.Machine, - userdataSecretName, userdata string, mdNameGetter machineDeploymentNameGetter, macAddressGetter macAddressGetter, sigClient client.Client) (*kubevirtv1.VirtualMachine, error) { +func (p *provider) newVirtualMachine(_ context.Context, c *Config, pc *providerconfigtypes.Config, machine *clusterv1alpha1.Machine, + userdataSecretName, userdata string, mdNameGetter machineDeploymentNameGetter, macAddressGetter macAddressGetter) (*kubevirtv1.VirtualMachine, error) { // We add the timestamp because the secret name must be different when we recreate the VMI // because its pod got deleted // The secret has an ownerRef on the VMI so garbace collection will take care of cleaning up. @@ -685,7 +685,7 @@ func (p *provider) newVirtualMachine(ctx context.Context, c *Config, pc *provide }, Resources: resourceRequirements, }, - Affinity: getAffinity(c, machineDeploymentLabelKey, labels[machineDeploymentLabelKey]), + Affinity: getAffinity(c), TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, Volumes: getVMVolumes(c, dataVolumeName, userdataSecretName), DNSPolicy: c.DNSPolicy, @@ -738,7 +738,7 @@ func parseResources(cpus, memory string) (*corev1.ResourceList, error) { }, nil } -func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } @@ -873,7 +873,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. return dataVolumeTemplates } -func getAffinity(config *Config, matchKey, matchValue string) *corev1.Affinity { +func getAffinity(config *Config) *corev1.Affinity { affinity := &corev1.Affinity{} expressions := []corev1.NodeSelectorRequirement{ diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index 41de8082f..989296bab 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -256,7 +256,7 @@ func TestNewVirtualMachine(t *testing.T) { c.Namespace = testNamespace // Check the created VirtualMachine - vm, _ := p.newVirtualMachine(context.TODO(), c, pc, machine, "udsn", userdata, fakeMachineDeploymentNameAndRevisionForMachineGetter(), fixedMacAddressGetter, fakeclient) + vm, _ := p.newVirtualMachine(context.TODO(), c, pc, machine, "udsn", userdata, fakeMachineDeploymentNameAndRevisionForMachineGetter(), fixedMacAddressGetter) vm.TypeMeta.APIVersion, vm.TypeMeta.Kind = kubevirtv1.VirtualMachineGroupVersionKind.ToAPIVersionAndKind() if !equality.Semantic.DeepEqual(vm, expectedVms[tt.name]) { diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index c3ef9ebe3..5ac0a7c3b 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -212,7 +212,7 @@ func createRandomPassword() (string, error) { return rootPass, nil } -func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -376,7 +376,7 @@ func (p *provider) MigrateUID(ctx context.Context, _ *zap.SugaredLogger, machine return nil } -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } @@ -457,6 +457,6 @@ func linodeStatusAndErrToTerminalError(err error) error { } } -func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/nutanix/client.go b/pkg/cloudprovider/provider/nutanix/client.go index 332769823..9ec878238 100644 --- a/pkg/cloudprovider/provider/nutanix/client.go +++ b/pkg/cloudprovider/provider/nutanix/client.go @@ -32,7 +32,6 @@ import ( cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" nutanixtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/nutanix/types" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -93,7 +92,7 @@ func GetClientSet(config *Config) (*ClientSet, error) { }, nil } -func createVM(ctx context.Context, client *ClientSet, name string, conf Config, os providerconfigtypes.OperatingSystem, userdata string) (instance.Instance, error) { +func createVM(ctx context.Context, client *ClientSet, name string, conf Config, userdata string) (instance.Instance, error) { cluster, err := getClusterByName(ctx, client, conf.ClusterName) if err != nil { return nil, err diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index c93816d58..77baf0eaa 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -270,7 +270,7 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * } func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { - config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) + config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ Reason: common.InvalidConfigurationMachineError, @@ -286,14 +286,14 @@ func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, } } - return createVM(ctx, client, machine.Name, *config, pc.OperatingSystem, userdata) + return createVM(ctx, client, machine.Name, *config, userdata) } func (p *provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { return p.cleanup(ctx, machine, data) } -func (p *provider) cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, cloudprovidererrors.TerminalError{ @@ -354,7 +354,7 @@ func (p *provider) cleanup(ctx context.Context, machine *clusterv1alpha1.Machine return true, nil } -func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -424,7 +424,7 @@ func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *cluste } // GetCloudConfig returns an empty cloud configuration for Nutanix as no CCM exists. -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } @@ -442,6 +442,6 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, nil } -func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/opennebula/provider.go b/pkg/cloudprovider/provider/opennebula/provider.go index 1108bb8b6..728ca9e63 100644 --- a/pkg/cloudprovider/provider/opennebula/provider.go +++ b/pkg/cloudprovider/provider/opennebula/provider.go @@ -29,7 +29,6 @@ import ( "github.com/OpenNebula/one/src/oca/go/src/goca/schemas/shared" "github.com/OpenNebula/one/src/oca/go/src/goca/schemas/vm" "github.com/OpenNebula/one/src/oca/go/src/goca/schemas/vm/keys" - "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" @@ -166,7 +165,7 @@ func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string return "", "", nil } -func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -250,7 +249,7 @@ func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, machine *clus return &openNebulaInstance{vm}, nil } -func (p *provider) Cleanup(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { instance, err := p.get(machine) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { @@ -345,7 +344,7 @@ func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.Machin return spec, nil } -func (p *provider) MigrateUID(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return cloudprovidererrors.TerminalError{ diff --git a/pkg/cloudprovider/provider/openstack/helper.go b/pkg/cloudprovider/provider/openstack/helper.go index d6bb0b8ab..2c2a5ec3b 100644 --- a/pkg/cloudprovider/provider/openstack/helper.go +++ b/pkg/cloudprovider/provider/openstack/helper.go @@ -98,7 +98,7 @@ func getNewComputeV2(client *gophercloud.ProviderClient, c *Config) (*gopherclou return computeClient, nil } -func getAvailabilityZones(computeClient *gophercloud.ServiceClient, c *Config) ([]osavailabilityzones.AvailabilityZone, error) { +func getAvailabilityZones(computeClient *gophercloud.ServiceClient) ([]osavailabilityzones.AvailabilityZone, error) { allPages, err := osavailabilityzones.List(computeClient).AllPages() if err != nil { return nil, err @@ -107,7 +107,7 @@ func getAvailabilityZones(computeClient *gophercloud.ServiceClient, c *Config) ( } func getAvailabilityZone(computeClient *gophercloud.ServiceClient, c *Config) (*osavailabilityzones.AvailabilityZone, error) { - zones, err := getAvailabilityZones(computeClient, c) + zones, err := getAvailabilityZones(computeClient) if err != nil { return nil, err } diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 65a6096dc..c4dcea347 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -386,7 +386,7 @@ func (p *provider) AddDefaults(log *zap.SugaredLogger, spec clusterv1alpha1.Mach if c.AvailabilityZone == "" { log.Debug("Trying to default availability zone for machine...") - availabilityZones, err := getAvailabilityZones(computeClient, c) + availabilityZones, err := getAvailabilityZones(computeClient) if err != nil { return spec, osErrorToTerminalError(log, err, "failed to get availability zones") } @@ -1110,6 +1110,6 @@ func assignFloatingIPToInstance(instanceLog *zap.SugaredLogger, machineUpdater c return nil } -func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go index 2916b671e..f91af1deb 100644 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ b/pkg/cloudprovider/provider/scaleway/provider.go @@ -139,7 +139,7 @@ func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.Machin return spec, nil } -func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -172,7 +172,7 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus return nil } -func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (cloudInstance.Instance, error) { +func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (cloudInstance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -351,7 +351,7 @@ func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, machine * return nil } -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } @@ -449,6 +449,6 @@ func scalewayErrToTerminalError(err error) error { return err } -func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go index c67169efc..404c62185 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -156,7 +156,7 @@ func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.Machin return spec, err } -func (p *provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { +func (p *provider) Cleanup(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return false, fmt.Errorf("failed to parse config: %w", err) @@ -209,7 +209,7 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * return vm, nil } -func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { +func (p *provider) create(_ context.Context, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { c, providerConfig, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("failed to parse config: %w", err) @@ -287,7 +287,7 @@ func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, return p.getInstance(vm) } -func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("failed to parse config: %w", err) @@ -306,7 +306,7 @@ func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clust return p.getInstance(vm) } -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } @@ -474,7 +474,7 @@ func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *cluste return nil } -func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go index 9e79dfebc..9680c4d80 100644 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ b/pkg/cloudprovider/provider/vsphere/helper.go @@ -34,8 +34,6 @@ import ( "github.com/vmware/govmomi/vim25/types" "go.uber.org/zap" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" ) @@ -45,7 +43,7 @@ const ( local-hostname: {{ .Hostname }}` ) -func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, config *Config, session *Session, os providerconfigtypes.OperatingSystem, containerLinuxUserdata string) (*object.VirtualMachine, error) { +func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, config *Config, session *Session, containerLinuxUserdata string) (*object.VirtualMachine, error) { tpl, err := session.Finder.VirtualMachine(ctx, config.TemplateVMName) if err != nil { return nil, fmt.Errorf("failed to get template vm: %w", err) @@ -87,7 +85,7 @@ func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, return nil, fmt.Errorf("failed to resolve datastore: %w", err) } - resourcepoolref, err := resolveResourcePoolRef(ctx, config, session, tpl) + resourcepoolref, err := resolveResourcePoolRef(ctx, config, session) if err != nil { return nil, fmt.Errorf("failed to resolve resourcePool: %w", err) } @@ -449,7 +447,7 @@ func getDatastoreFromVM(ctx context.Context, session *Session, vmRef *object.Vir return session.Finder.Datastore(ctx, datastorePathObj.Datastore) } -func resolveResourcePoolRef(ctx context.Context, config *Config, session *Session, vm *object.VirtualMachine) (*types.ManagedObjectReference, error) { +func resolveResourcePoolRef(ctx context.Context, config *Config, session *Session) (*types.ManagedObjectReference, error) { if config.ResourcePool != "" { targetResourcePool, err := session.Finder.ResourcePool(ctx, config.ResourcePool) if err != nil { diff --git a/pkg/cloudprovider/provider/vsphere/helper_test.go b/pkg/cloudprovider/provider/vsphere/helper_test.go index 783e92d24..257339db4 100644 --- a/pkg/cloudprovider/provider/vsphere/helper_test.go +++ b/pkg/cloudprovider/provider/vsphere/helper_test.go @@ -131,7 +131,7 @@ type CustomStorageResourceManager struct { } // RecommendDatastores always return a recommendation for the purposes of the test. -func (c *CustomStorageResourceManager) RecommendDatastores(req *types.RecommendDatastores) soap.HasFault { +func (c *CustomStorageResourceManager) RecommendDatastores(_ *types.RecommendDatastores) soap.HasFault { body := &methods.RecommendDatastoresBody{} res := &types.RecommendDatastoresResponse{} ds := c.ds.Reference() @@ -222,11 +222,7 @@ func TestResolveResourcePoolRef(t *testing.T) { t.Fatalf("error creating session: %v", err) } - // Obtain a VM from the simulator - obj := simulator.Map.Any("VirtualMachine").(*simulator.VirtualMachine) - vm := object.NewVirtualMachine(session.Client.Client, obj.Reference()) - - got, err := resolveResourcePoolRef(ctx, tt.config, session, vm) + got, err := resolveResourcePoolRef(ctx, tt.config, session) if (err != nil) != tt.wantErr { t.Errorf("error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 93f9bf552..30e751f03 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -370,7 +370,6 @@ func (p *provider) create(ctx context.Context, log *zap.SugaredLogger, machine * machine.Spec.Name, config, session, - pc.OperatingSystem, containerLinuxUserdata, ) if err != nil { @@ -527,7 +526,7 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine return true, nil } -func (p *provider) Get(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (p *provider) Get(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("failed to parse config: %w", err) @@ -669,7 +668,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } -func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/cloudprovider/provider/vultr/provider.go b/pkg/cloudprovider/provider/vultr/provider.go index 3f4e63d5a..09e1f9eca 100644 --- a/pkg/cloudprovider/provider/vultr/provider.go +++ b/pkg/cloudprovider/provider/vultr/provider.go @@ -209,11 +209,11 @@ func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clust return p.get(ctx, machine) } -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { +func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } -func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -370,6 +370,6 @@ func vltErrorToTerminalError(err error, msg string) error { return err } -func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } diff --git a/pkg/containerruntime/containerruntime.go b/pkg/containerruntime/containerruntime.go index a6c3c5c9a..3bdf41188 100644 --- a/pkg/containerruntime/containerruntime.go +++ b/pkg/containerruntime/containerruntime.go @@ -17,8 +17,6 @@ limitations under the License. package containerruntime import ( - "github.com/Masterminds/semver/v3" - "github.com/kubermatic/machine-controller/pkg/providerconfig/types" ) @@ -62,7 +60,7 @@ func withContainerdVersion(version string) Opt { } } -func get(containerRuntimeName string, opts ...Opt) Config { +func get(_ string, opts ...Opt) Config { cfg := Config{} cfg.Containerd = &Containerd{} @@ -103,7 +101,7 @@ func (cfg Config) String() string { return containerdName } -func (cfg Config) Engine(kubeletVersion *semver.Version) Engine { +func (cfg Config) Engine() Engine { containerd := &Containerd{ insecureRegistries: cfg.InsecureRegistries, registryMirrors: cfg.RegistryMirrors, diff --git a/pkg/controller/machine/controller.go b/pkg/controller/machine/controller.go index d952ec113..335d901f8 100644 --- a/pkg/controller/machine/controller.go +++ b/pkg/controller/machine/controller.go @@ -67,7 +67,6 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/tools/reference" "k8s.io/client-go/util/retry" - ccmapi "k8s.io/cloud-provider/api" "sigs.k8s.io/controller-runtime/pkg/client" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -245,7 +244,6 @@ func Add( machinesList := &clusterv1alpha1.MachineList{} if err := mgr.GetClient().List(ctx, machinesList); err != nil { utilruntime.HandleError(fmt.Errorf("failed to list machines in lister: %w", err)) - return } var ownerUIDString string @@ -264,7 +262,7 @@ func Add( Name: machine.Name}}) } } - return + return result } for _, machine := range machinesList.Items { @@ -276,7 +274,7 @@ func Add( }}} } } - return + return result }), predicate.Funcs{UpdateFunc: func(e event.UpdateEvent) bool { oldNode := e.ObjectOld.(*corev1.Node) @@ -363,7 +361,7 @@ func (r *Reconciler) updateMachineErrorIfTerminalError(machine *clusterv1alpha1. func (r *Reconciler) createProviderInstance(ctx context.Context, log *zap.SugaredLogger, prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine, userdata string) (instance.Instance, error) { // Ensure finalizer is there. - _, err := r.ensureDeleteFinalizerExists(log, machine) + _, err := r.ensureDeleteFinalizerExists(machine) if err != nil { return nil, fmt.Errorf("failed to add %q finalizer: %w", FinalizerDeleteInstance, err) } @@ -934,7 +932,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( return nil, fmt.Errorf("failed to get instance from provider: %w", err) } // Instance exists, so ensure finalizer does as well - machine, err = r.ensureDeleteFinalizerExists(log, machine) + machine, err = r.ensureDeleteFinalizerExists(machine) if err != nil { return nil, fmt.Errorf("failed to add %q finalizer: %w", FinalizerDeleteInstance, err) } @@ -1229,7 +1227,7 @@ func (r *Reconciler) ReadinessChecks(ctx context.Context) map[string]healthcheck } } -func (r *Reconciler) ensureDeleteFinalizerExists(log *zap.SugaredLogger, machine *clusterv1alpha1.Machine) (*clusterv1alpha1.Machine, error) { +func (r *Reconciler) ensureDeleteFinalizerExists(machine *clusterv1alpha1.Machine) (*clusterv1alpha1.Machine, error) { finalizers := sets.NewString(machine.Finalizers...) length := finalizers.Len() diff --git a/pkg/controller/machinedeployment/controller.go b/pkg/controller/machinedeployment/controller.go index 9f45b1fe0..c1f8bad32 100644 --- a/pkg/controller/machinedeployment/controller.go +++ b/pkg/controller/machinedeployment/controller.go @@ -33,7 +33,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -197,22 +196,17 @@ func (r *ReconcileMachineDeployment) reconcile(ctx context.Context, log *zap.Sug return reconcile.Result{}, err } - machineMap, err := r.getMachineMapForDeployment(ctx, d, msList) - if err != nil { - return reconcile.Result{}, err - } - if d.DeletionTimestamp != nil { - return reconcile.Result{}, r.sync(ctx, log, d, msList, machineMap) + return reconcile.Result{}, r.sync(ctx, log, d, msList) } if d.Spec.Paused { - return reconcile.Result{}, r.sync(ctx, log, d, msList, machineMap) + return reconcile.Result{}, r.sync(ctx, log, d, msList) } switch d.Spec.Strategy.Type { case common.RollingUpdateMachineDeploymentStrategyType: - return reconcile.Result{}, r.rolloutRolling(ctx, log, d, msList, machineMap) + return reconcile.Result{}, r.rolloutRolling(ctx, log, d, msList) } return reconcile.Result{}, errors.Errorf("unexpected deployment strategy type: %s", d.Spec.Strategy.Type) @@ -274,51 +268,6 @@ func (r *ReconcileMachineDeployment) adoptOrphan(ctx context.Context, deployment return r.Client.Update(ctx, machineSet) } -// getMachineMapForDeployment returns the Machines managed by a Deployment. -// -// It returns a map from MachineSet UID to a list of Machines controlled by that MachineSet, -// according to the Machine's ControllerRef. -func (r *ReconcileMachineDeployment) getMachineMapForDeployment(ctx context.Context, d *v1alpha1.MachineDeployment, msList []*v1alpha1.MachineSet) (map[types.UID]*v1alpha1.MachineList, error) { - // TODO(droot): double check if previous selector maps correctly to new one. - // _, err := metav1.LabelSelectorAsSelector(&d.Spec.Selector) - - // Get all Machines that potentially belong to this Deployment. - selector, err := metav1.LabelSelectorAsMap(&d.Spec.Selector) - if err != nil { - return nil, err - } - - machines := &v1alpha1.MachineList{} - listOptions := &client.ListOptions{Namespace: d.Namespace} - if err = r.Client.List(ctx, machines, listOptions, client.MatchingLabels(selector)); err != nil { - return nil, err - } - - // Group Machines by their controller (if it's in msList). - machineMap := make(map[types.UID]*v1alpha1.MachineList, len(msList)) - for _, ms := range msList { - machineMap[ms.UID] = &v1alpha1.MachineList{} - } - - for idx := range machines.Items { - machine := &machines.Items[idx] - - // Do not ignore inactive Machines because Recreate Deployments need to verify that no - // Machines from older versions are running before spinning up new Machines. - controllerRef := metav1.GetControllerOf(machine) - if controllerRef == nil { - continue - } - - // Only append if we care about this UID. - if machineList, ok := machineMap[controllerRef.UID]; ok { - machineList.Items = append(machineList.Items, *machine) - } - } - - return machineMap, nil -} - // getMachineDeploymentsForMachineSet returns a list of MachineDeployments that could potentially match a MachineSet. func (r *ReconcileMachineDeployment) getMachineDeploymentsForMachineSet(ctx context.Context, log *zap.SugaredLogger, ms *v1alpha1.MachineSet) []*v1alpha1.MachineDeployment { if len(ms.Labels) == 0 { diff --git a/pkg/controller/machinedeployment/rolling.go b/pkg/controller/machinedeployment/rolling.go index 11eb84d08..f4cb42676 100644 --- a/pkg/controller/machinedeployment/rolling.go +++ b/pkg/controller/machinedeployment/rolling.go @@ -26,14 +26,13 @@ import ( "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" dutil "github.com/kubermatic/machine-controller/pkg/controller/util" - "k8s.io/apimachinery/pkg/types" "k8s.io/utils/integer" "sigs.k8s.io/controller-runtime/pkg/client" ) // rolloutRolling implements the logic for rolling a new machine set. -func (r *ReconcileMachineDeployment) rolloutRolling(ctx context.Context, log *zap.SugaredLogger, d *v1alpha1.MachineDeployment, msList []*v1alpha1.MachineSet, machineMap map[types.UID]*v1alpha1.MachineList) error { - newMS, oldMSs, err := r.getAllMachineSetsAndSyncRevision(ctx, log, d, msList, machineMap, true) +func (r *ReconcileMachineDeployment) rolloutRolling(ctx context.Context, log *zap.SugaredLogger, d *v1alpha1.MachineDeployment, msList []*v1alpha1.MachineSet) error { + newMS, oldMSs, err := r.getAllMachineSetsAndSyncRevision(ctx, log, d, msList, true) if err != nil { return err } diff --git a/pkg/controller/machinedeployment/sync.go b/pkg/controller/machinedeployment/sync.go index 9fef99055..9313d950a 100644 --- a/pkg/controller/machinedeployment/sync.go +++ b/pkg/controller/machinedeployment/sync.go @@ -41,8 +41,8 @@ import ( // sync is responsible for reconciling deployments on scaling events or when they // are paused. -func (r *ReconcileMachineDeployment) sync(ctx context.Context, log *zap.SugaredLogger, d *clusterv1alpha1.MachineDeployment, msList []*clusterv1alpha1.MachineSet, machineMap map[types.UID]*clusterv1alpha1.MachineList) error { - newMS, oldMSs, err := r.getAllMachineSetsAndSyncRevision(ctx, log, d, msList, machineMap, false) +func (r *ReconcileMachineDeployment) sync(ctx context.Context, log *zap.SugaredLogger, d *clusterv1alpha1.MachineDeployment, msList []*clusterv1alpha1.MachineSet) error { + newMS, oldMSs, err := r.getAllMachineSetsAndSyncRevision(ctx, log, d, msList, false) if err != nil { return err } @@ -54,7 +54,7 @@ func (r *ReconcileMachineDeployment) sync(ctx context.Context, log *zap.SugaredL } // - // // TODO: Clean up the deployment when it's paused and no rollback is in flight. + // TODO: Clean up the deployment when it's paused and no rollback is in flight. // allMSs := append(oldMSs, newMS) return r.syncDeploymentStatus(ctx, allMSs, newMS, d) @@ -72,7 +72,7 @@ func (r *ReconcileMachineDeployment) sync(ctx context.Context, log *zap.SugaredL // // Note that currently the deployment controller is using caches to avoid querying the server for reads. // This may lead to stale reads of machine sets, thus incorrect deployment status. -func (r *ReconcileMachineDeployment) getAllMachineSetsAndSyncRevision(ctx context.Context, log *zap.SugaredLogger, d *clusterv1alpha1.MachineDeployment, msList []*clusterv1alpha1.MachineSet, machineMap map[types.UID]*clusterv1alpha1.MachineList, createIfNotExisted bool) (*clusterv1alpha1.MachineSet, []*clusterv1alpha1.MachineSet, error) { +func (r *ReconcileMachineDeployment) getAllMachineSetsAndSyncRevision(ctx context.Context, log *zap.SugaredLogger, d *clusterv1alpha1.MachineDeployment, msList []*clusterv1alpha1.MachineSet, createIfNotExisted bool) (*clusterv1alpha1.MachineSet, []*clusterv1alpha1.MachineSet, error) { _, allOldMSs := dutil.FindOldMachineSets(d, msList) // Get new machine set with the updated revision number @@ -269,21 +269,6 @@ func (r *ReconcileMachineDeployment) scale(ctx context.Context, log *zap.Sugared // machine sets. deploymentReplicasToAdd := allowedSize - totalMSReplicas - // The additional replicas should be distributed proportionally amongst the active - // machine sets from the larger to the smaller in size machine set. Scaling direction - // drives what happens in case we are trying to scale machine sets of the same size. - // In such a case when scaling up, we should scale up newer machine sets first, and - // when scaling down, we should scale down older machine sets first. - var scalingOperation string - switch { - case deploymentReplicasToAdd > 0: - sort.Sort(dutil.MachineSetsBySizeNewer(allMSs)) - scalingOperation = "up" - case deploymentReplicasToAdd < 0: - sort.Sort(dutil.MachineSetsBySizeOlder(allMSs)) - scalingOperation = "down" - } - // Iterate over all active machine sets and estimate proportions for each of them. // The absolute value of deploymentReplicasAdded should never exceed the absolute // value of deploymentReplicasToAdd. @@ -321,7 +306,7 @@ func (r *ReconcileMachineDeployment) scale(ctx context.Context, log *zap.Sugared } // TODO: Use transactions when we have them. - if _, err := r.scaleMachineSetOperation(ctx, ms, nameToSize[ms.Name], deployment, scalingOperation); err != nil { + if _, err := r.scaleMachineSetOperation(ctx, ms, nameToSize[ms.Name], deployment); err != nil { // Return as soon as we fail, the deployment is requeued return err } @@ -376,18 +361,10 @@ func (r *ReconcileMachineDeployment) scaleMachineSet(ctx context.Context, ms *cl if *(ms.Spec.Replicas) == newScale { return false, nil } - - var scalingOperation string - if *(ms.Spec.Replicas) < newScale { - scalingOperation = "up" - } else { - scalingOperation = "down" - } - - return r.scaleMachineSetOperation(ctx, ms, newScale, deployment, scalingOperation) + return r.scaleMachineSetOperation(ctx, ms, newScale, deployment) } -func (r *ReconcileMachineDeployment) scaleMachineSetOperation(ctx context.Context, ms *clusterv1alpha1.MachineSet, newScale int32, deployment *clusterv1alpha1.MachineDeployment, scaleOperation string) (bool, error) { +func (r *ReconcileMachineDeployment) scaleMachineSetOperation(ctx context.Context, ms *clusterv1alpha1.MachineSet, newScale int32, deployment *clusterv1alpha1.MachineDeployment) (bool, error) { if ms.Spec.Replicas == nil { return false, errors.Errorf("spec replicas for machine set %v is nil, this is unexpected", ms.Name) } diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index 1085ec7ff..e2b838328 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -77,7 +77,7 @@ func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) ( return "", fmt.Errorf("error extracting cacert: %w", err) } - crEngine := req.ContainerRuntime.Engine(kubeletVersion) + crEngine := req.ContainerRuntime.Engine() crScript, err := crEngine.ScriptFor(providerconfigtypes.OperatingSystemAmazonLinux2) if err != nil { return "", fmt.Errorf("failed to generate container runtime install script: %w", err) @@ -276,7 +276,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - path: "/etc/kubernetes/pki/ca.crt" content: | diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index bf6db0909..eb2af6bce 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -76,7 +76,7 @@ type fakeCloudConfigProvider struct { err error } -func (p *fakeCloudConfigProvider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { +func (p *fakeCloudConfigProvider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return p.config, p.name, p.err } diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go index 94f5790a5..0f24ee2c8 100644 --- a/pkg/userdata/centos/provider.go +++ b/pkg/userdata/centos/provider.go @@ -77,7 +77,7 @@ func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) ( return "", fmt.Errorf("error extracting cacert: %w", err) } - crEngine := req.ContainerRuntime.Engine(kubeletVersion) + crEngine := req.ContainerRuntime.Engine() crScript, err := crEngine.ScriptFor(providerconfigtypes.OperatingSystemCentOS) if err != nil { return "", fmt.Errorf("failed to generate container runtime install script: %w", err) @@ -294,7 +294,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - path: "/etc/kubernetes/pki/ca.crt" content: | diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index d0c9df2a8..88e1260a3 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -76,7 +76,7 @@ type fakeCloudConfigProvider struct { err error } -func (p *fakeCloudConfigProvider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { +func (p *fakeCloudConfigProvider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return p.config, p.name, p.err } diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go index 6e9457f16..a8de7033c 100644 --- a/pkg/userdata/flatcar/provider.go +++ b/pkg/userdata/flatcar/provider.go @@ -83,7 +83,7 @@ func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) ( flatcarConfig.DisableUpdateEngine = true } - crEngine := req.ContainerRuntime.Engine(kubeletVersion) + crEngine := req.ContainerRuntime.Engine() crScript, err := crEngine.ScriptFor(providerconfigtypes.OperatingSystemFlatcar) if err != nil { return "", fmt.Errorf("failed to generate container runtime install script: %w", err) @@ -331,7 +331,7 @@ storage: mode: 0644 contents: inline: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 10 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 10 }} - path: /opt/load-kernel-modules.sh filesystem: root @@ -693,7 +693,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" permissions: "0644" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - path: /opt/load-kernel-modules.sh permissions: "0755" diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index fc9845183..1ac154916 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -96,7 +96,7 @@ type fakeCloudConfigProvider struct { err error } -func (p *fakeCloudConfigProvider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { +func (p *fakeCloudConfigProvider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return p.config, p.name, p.err } diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index 9a64e81dc..67d5405bd 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -200,7 +200,7 @@ func KubeletSystemdUnit(log *zap.SugaredLogger, containerRuntime, kubeletVersion } // kubeletConfiguration returns marshaled kubelet.config.k8s.io/v1beta1 KubeletConfiguration. -func kubeletConfiguration(log *zap.SugaredLogger, clusterDomain string, clusterDNS []net.IP, featureGates map[string]bool, kubeletConfigs map[string]string, containerRuntime string) (string, error) { +func kubeletConfiguration(log *zap.SugaredLogger, clusterDomain string, clusterDNS []net.IP, featureGates map[string]bool, kubeletConfigs map[string]string) (string, error) { clusterDNSstr := make([]string, 0, len(clusterDNS)) for _, ip := range clusterDNS { clusterDNSstr = append(clusterDNSstr, ip.String()) diff --git a/pkg/userdata/helper/template_functions.go b/pkg/userdata/helper/template_functions.go index 2b7fe2ee9..ab449414f 100644 --- a/pkg/userdata/helper/template_functions.go +++ b/pkg/userdata/helper/template_functions.go @@ -47,8 +47,8 @@ func TxtFuncMap(log *zap.SugaredLogger) template.FuncMap { return KubeletSystemdUnit(log, containerRuntime, kubeletVersion, cloudProvider, hostname, dnsIPs, external, ipFamily, pauseImage, initialTaints, extraKubeletFlags, disableSwap) } - funcMap["kubeletConfiguration"] = func(clusterDomain string, clusterDNS []net.IP, featureGates map[string]bool, kubeletConfigs map[string]string, containerRuntime string) (string, error) { - return kubeletConfiguration(log, clusterDomain, clusterDNS, featureGates, kubeletConfigs, containerRuntime) + funcMap["kubeletConfiguration"] = func(clusterDomain string, clusterDNS []net.IP, featureGates map[string]bool, kubeletConfigs map[string]string) (string, error) { + return kubeletConfiguration(log, clusterDomain, clusterDNS, featureGates, kubeletConfigs) } funcMap["kubeletFlags"] = func(version, cloudProvider, hostname string, dnsIPs []net.IP, external bool, ipFamily util.IPFamily, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string) (string, error) { diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go index d0395cb6a..218c8b3ec 100644 --- a/pkg/userdata/rhel/provider.go +++ b/pkg/userdata/rhel/provider.go @@ -77,7 +77,7 @@ func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) ( return "", fmt.Errorf("error extracting cacert: %w", err) } - crEngine := req.ContainerRuntime.Engine(kubeletVersion) + crEngine := req.ContainerRuntime.Engine() crScript, err := crEngine.ScriptFor(providerconfigtypes.OperatingSystemRHEL) if err != nil { return "", fmt.Errorf("failed to generate container runtime install script: %w", err) @@ -301,7 +301,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - path: "/etc/kubernetes/pki/ca.crt" content: | diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index 0ca2b1e91..6f101211f 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -76,7 +76,7 @@ type fakeCloudConfigProvider struct { err error } -func (p *fakeCloudConfigProvider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { +func (p *fakeCloudConfigProvider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return p.config, p.name, p.err } diff --git a/pkg/userdata/rockylinux/provider.go b/pkg/userdata/rockylinux/provider.go index e42948b35..06f2633de 100644 --- a/pkg/userdata/rockylinux/provider.go +++ b/pkg/userdata/rockylinux/provider.go @@ -77,7 +77,7 @@ func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) ( return "", fmt.Errorf("error extracting cacert: %w", err) } - crEngine := req.ContainerRuntime.Engine(kubeletVersion) + crEngine := req.ContainerRuntime.Engine() crScript, err := crEngine.ScriptFor(providerconfigtypes.OperatingSystemRockyLinux) if err != nil { return "", fmt.Errorf("failed to generate container runtime install script: %w", err) @@ -298,7 +298,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - path: "/etc/kubernetes/pki/ca.crt" content: | diff --git a/pkg/userdata/rockylinux/provider_test.go b/pkg/userdata/rockylinux/provider_test.go index 5556fd4c2..2ec2339c6 100644 --- a/pkg/userdata/rockylinux/provider_test.go +++ b/pkg/userdata/rockylinux/provider_test.go @@ -76,7 +76,7 @@ type fakeCloudConfigProvider struct { err error } -func (p *fakeCloudConfigProvider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { +func (p *fakeCloudConfigProvider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return p.config, p.name, p.err } diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index 4a12d92a9..47dbc0107 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -77,7 +77,7 @@ func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) ( return "", fmt.Errorf("error extracting cacert: %w", err) } - crEngine := req.ContainerRuntime.Engine(kubeletVersion) + crEngine := req.ContainerRuntime.Engine() crScript, err := crEngine.ScriptFor(providerconfigtypes.OperatingSystemUbuntu) if err != nil { return "", fmt.Errorf("failed to generate container runtime install script: %w", err) @@ -320,7 +320,7 @@ write_files: - path: "/etc/kubernetes/kubelet.conf" content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }} +{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index d07e31eae..fa956441f 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -103,7 +103,7 @@ type fakeCloudConfigProvider struct { err error } -func (p *fakeCloudConfigProvider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { +func (p *fakeCloudConfigProvider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return p.config, p.name, p.err } diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 25a56e696..510d2a330 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -278,14 +278,14 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar gopath := os.Getenv("GOPATH") projectDir := filepath.Join(gopath, "src/github.com/kubermatic/machine-controller") kubeConfig := filepath.Join(projectDir, ".kubeconfig") - - if _, err := os.Stat(kubeConfig); err == nil { - // it exists at hardcoded path - } else if os.IsNotExist(err) { - // it doesn't exist, fall back to $KUBECONFIG - kubeConfig = os.Getenv("KUBECONFIG") - } else { - t.Fatal(err) + _, err := os.Stat(kubeConfig) + if err != nil { + if os.IsNotExist(err) { + // it doesn't exist, fall back to $KUBECONFIG + kubeConfig = os.Getenv("KUBECONFIG") + } else { + t.Fatal(err) + } } // the golang test runtime waits for individual subtests to complete before reporting the status. diff --git a/test/e2e/provisioning/migrateuidscenario.go b/test/e2e/provisioning/migrateuidscenario.go index 89fcf3a0c..54beb379f 100644 --- a/test/e2e/provisioning/migrateuidscenario.go +++ b/test/e2e/provisioning/migrateuidscenario.go @@ -40,7 +40,7 @@ import ( fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -func verifyMigrateUID(kubeConfig, manifestPath string, parameters []string, timeout time.Duration) error { +func verifyMigrateUID(_, manifestPath string, parameters []string, _ time.Duration) error { log := zap.NewNop().Sugar() // prepare the manifest From a55b9aeda60e78d54d78b1d0612708a37ed6b9eb Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 16 Aug 2023 20:07:53 +0500 Subject: [PATCH 345/489] Support for Kubernetes v1.28 (#1699) Signed-off-by: Waleed Malik --- pkg/userdata/helper/common_test.go | 1 + .../testdata/download_binaries_v1.28.0.golden | 17 + ...stemd_unit_version-v1.28.0-external.golden | 36 ++ ...kublet_systemd_unit_version-v1.28.0.golden | 35 ++ pkg/userdata/ubuntu/provider_test.go | 1 + .../ubuntu/testdata/version-1.28.0.yaml | 457 ++++++++++++++++++ test/e2e/provisioning/all_e2e_test.go | 8 +- test/e2e/provisioning/helper.go | 1 + 8 files changed, 552 insertions(+), 4 deletions(-) create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.28.0.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0-external.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0.golden create mode 100644 pkg/userdata/ubuntu/testdata/version-1.28.0.yaml diff --git a/pkg/userdata/helper/common_test.go b/pkg/userdata/helper/common_test.go index 5d1eaa2ca..3b09f8e92 100644 --- a/pkg/userdata/helper/common_test.go +++ b/pkg/userdata/helper/common_test.go @@ -29,5 +29,6 @@ var ( semver.MustParse("v1.25.12"), semver.MustParse("v1.26.7"), semver.MustParse("v1.27.4"), + semver.MustParse("v1.28.0"), } ) diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.28.0.golden b/pkg/userdata/helper/testdata/download_binaries_v1.28.0.golden new file mode 100644 index 000000000..e438f0e67 --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.28.0.golden @@ -0,0 +1,17 @@ +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - +fi +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.28.0/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0-external.golden new file mode 100644 index 000000000..50f4f5138 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0-external.golden @@ -0,0 +1,36 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0.golden new file mode 100644 index 000000000..e70567560 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0.golden @@ -0,0 +1,35 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index fa956441f..37a4cd04f 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -132,6 +132,7 @@ func simpleVersionTests() []userDataTestCase { semver.MustParse("v1.25.12"), semver.MustParse("v1.26.7"), semver.MustParse("v1.27.4"), + semver.MustParse("v1.28.0"), } var tests []userDataTestCase diff --git a/pkg/userdata/ubuntu/testdata/version-1.28.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.28.0.yaml new file mode 100644 index 000000000..b9a4b3daa --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.28.0.yaml @@ -0,0 +1,457 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index d9356ccfe..fe892dd90 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -343,7 +343,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.26. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.26.7", "1.27.4"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.26.7", "1.27.4", "1.28.0"))) runScenarios(t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -423,7 +423,7 @@ func TestAWSProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := Not(VersionSelector("1.27.4")) + selector := Not(VersionSelector("1.27.4", "1.28.0")) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -477,7 +477,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.4"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.4", "1.28.0"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -499,7 +499,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.4"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.4", "1.28.0"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 510d2a330..dc8c3412b 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -36,6 +36,7 @@ var ( semver.MustParse("v1.25.12"), semver.MustParse("v1.26.7"), semver.MustParse("v1.27.4"), + semver.MustParse("v1.28.0"), } operatingSystems = []providerconfigtypes.OperatingSystem{ From 8957c018176be11f98fcd6ed6ba0fb097c89183d Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 17 Aug 2023 13:52:54 +0500 Subject: [PATCH 346/489] Update dependencies (#1700) * Update k8s APIs to v1.28 Signed-off-by: Waleed Malik * Satisfy fake clients requirement of finalizers with DeletionTimeStamp Fake client won't create objects after https://github.com/kubernetes-sigs/controller-runtime/pull/2316 if they have DeletionTimeStamp without finalizers Signed-off-by: Waleed Malik * Revert change in all_e2e_test.go Signed-off-by: Waleed Malik * Refactored code Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- go.mod | 181 ++++---- go.sum | 405 +++++++++--------- .../cluster/v1alpha1/migrations/migrations.go | 6 +- pkg/cloudprovider/provider/aws/provider.go | 28 +- pkg/cloudprovider/provider/azure/provider.go | 40 +- .../provider/digitalocean/provider.go | 2 +- pkg/cloudprovider/provider/gce/provider.go | 26 +- pkg/cloudprovider/provider/gce/service.go | 14 +- .../provider/kubevirt/provider.go | 6 +- pkg/cloudprovider/provider/nutanix/client.go | 41 +- .../provider/nutanix/provider.go | 4 +- .../provider/openstack/provider.go | 14 +- .../provider/openstack/provider_test.go | 8 +- .../openstack/types/cloudconfig_test.go | 6 +- .../provider/vmwareclouddirector/helper.go | 8 +- .../provider/vmwareclouddirector/provider.go | 6 +- .../provider/vsphere/provider_test.go | 18 +- pkg/controller/machine/controller.go | 83 ++-- pkg/controller/machine/controller_test.go | 8 + .../machinedeployment/controller.go | 67 +-- pkg/controller/machineset/controller.go | 70 +-- pkg/controller/nodecsrapprover/controller.go | 2 +- pkg/node/nodemanager/node_manager.go | 2 +- pkg/providerconfig/types/types_test.go | 26 +- .../amzn2/testdata/kubelet-v1.25-aws.yaml | 1 + .../amzn2/testdata/kubelet-v1.26-aws.yaml | 1 + .../kubelet-v1.26.6-aws-external.yaml | 1 + .../amzn2/testdata/kubelet-v1.26.6-aws.yaml | 1 + .../kubelet-v1.26.6-vsphere-mirrors.yaml | 1 + .../kubelet-v1.26.6-vsphere-proxy.yaml | 1 + .../testdata/kubelet-v1.26.6-vsphere.yaml | 1 + .../amzn2/testdata/kubelet-v1.27-aws.yaml | 1 + .../centos/testdata/kubelet-v1.25-aws.yaml | 1 + .../kubelet-v1.26.6-aws-external.yaml | 1 + .../centos/testdata/kubelet-v1.26.6-aws.yaml | 1 + .../testdata/kubelet-v1.26.6-nutanix.yaml | 1 + .../kubelet-v1.26.6-vsphere-mirrors.yaml | 1 + .../kubelet-v1.26.6-vsphere-proxy.yaml | 1 + .../testdata/kubelet-v1.26.6-vsphere.yaml | 1 + .../centos/testdata/kubelet-v1.27-aws.yaml | 1 + .../flatcar/testdata/cloud-init_v1.25.0.yaml | 1 + .../flatcar/testdata/cloud-init_v1.26.6.yaml | 1 + pkg/userdata/flatcar/testdata/containerd.yaml | 1 + .../flatcar/testdata/ignition_v1.25.0.json | 2 +- .../flatcar/testdata/ignition_v1.26.6.json | 2 +- pkg/userdata/helper/kubelet.go | 10 +- .../rhel/testdata/kubelet-v1.25-aws.yaml | 1 + .../rhel/testdata/kubelet-v1.25-nutanix.yaml | 1 + .../testdata/kubelet-v1.26-aws-external.yaml | 1 + .../rhel/testdata/kubelet-v1.26-aws.yaml | 1 + .../kubelet-v1.26.6-aws-external.yaml | 1 + .../rhel/testdata/kubelet-v1.26.6-aws.yaml | 1 + .../kubelet-v1.26.6-vsphere-mirrors.yaml | 1 + .../kubelet-v1.26.6-vsphere-proxy.yaml | 1 + .../testdata/kubelet-v1.26.6-vsphere.yaml | 1 + .../rhel/testdata/pod-cidr-azure-rhel.yaml | 1 + .../testdata/kubelet-v1.25-aws.yaml | 1 + .../kubelet-v1.26.6-aws-external.yaml | 1 + .../testdata/kubelet-v1.26.6-aws.yaml | 1 + .../testdata/kubelet-v1.26.6-nutanix.yaml | 1 + .../kubelet-v1.26.6-vsphere-mirrors.yaml | 1 + .../kubelet-v1.26.6-vsphere-proxy.yaml | 1 + .../testdata/kubelet-v1.26.6-vsphere.yaml | 1 + pkg/userdata/ubuntu/testdata/containerd.yaml | 1 + .../digitalocean-dualstack-IPv6+IPv4.yaml | 1 + .../testdata/digitalocean-dualstack.yaml | 1 + .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 1 + pkg/userdata/ubuntu/testdata/docker.yaml | 1 + .../kubelet-version-without-v-prefix.yaml | 1 + .../ubuntu/testdata/multiple-dns-servers.yaml | 1 + .../ubuntu/testdata/multiple-ssh-keys.yaml | 1 + pkg/userdata/ubuntu/testdata/nutanix.yaml | 1 + .../openstack-dualstack-IPv6+IPv4.yaml | 1 + .../ubuntu/testdata/openstack-dualstack.yaml | 1 + .../openstack-overwrite-cloud-config.yaml | 1 + pkg/userdata/ubuntu/testdata/openstack.yaml | 1 + .../ubuntu/testdata/version-1.25.12.yaml | 1 + .../ubuntu/testdata/version-1.26.7.yaml | 1 + .../ubuntu/testdata/version-1.27.4.yaml | 1 + .../ubuntu/testdata/version-1.28.0.yaml | 1 + .../ubuntu/testdata/vsphere-mirrors.yaml | 1 + .../ubuntu/testdata/vsphere-proxy.yaml | 1 + pkg/userdata/ubuntu/testdata/vsphere.yaml | 1 + test/e2e/provisioning/all_e2e_test.go | 2 +- test/e2e/provisioning/deploymentscenario.go | 24 +- test/e2e/provisioning/verify.go | 24 +- 86 files changed, 636 insertions(+), 555 deletions(-) diff --git a/go.mod b/go.mod index 3a350354c..c15abc74d 100644 --- a/go.mod +++ b/go.mod @@ -3,135 +3,136 @@ module github.com/kubermatic/machine-controller go 1.20 require ( - cloud.google.com/go/logging v1.6.1 - cloud.google.com/go/monitoring v1.9.1 + cloud.google.com/go/logging v1.8.1 + cloud.google.com/go/monitoring v1.15.1 github.com/Azure/azure-sdk-for-go v65.0.0+incompatible - github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 + github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 github.com/Azure/go-autorest/autorest/to v0.4.0 - github.com/BurntSushi/toml v1.2.1 - github.com/Masterminds/semver/v3 v3.2.0 + github.com/BurntSushi/toml v1.3.2 + github.com/Masterminds/semver/v3 v3.2.1 github.com/Masterminds/sprig/v3 v3.2.3 - github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20220908162715-b27302cc7db5 - github.com/aliyun/alibaba-cloud-sdk-go v1.62.112 - github.com/aws/aws-sdk-go-v2 v1.17.3 - github.com/aws/aws-sdk-go-v2/config v1.18.7 - github.com/aws/aws-sdk-go-v2/credentials v1.13.7 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0 - github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 - github.com/aws/smithy-go v1.13.5 + github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20230725113508-e18d1b6d4ff8 + github.com/aliyun/alibaba-cloud-sdk-go v1.62.512 + github.com/aws/aws-sdk-go-v2 v1.20.1 + github.com/aws/aws-sdk-go-v2/config v1.18.33 + github.com/aws/aws-sdk-go-v2/credentials v1.13.32 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.112.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.21.2 + github.com/aws/smithy-go v1.14.1 github.com/davecgh/go-spew v1.1.1 - github.com/digitalocean/godo v1.93.0 + github.com/digitalocean/godo v1.102.0 github.com/flatcar/container-linux-config-transpiler v0.9.4 github.com/go-logr/logr v1.2.4 - github.com/go-logr/zapr v1.2.3 + github.com/go-logr/zapr v1.2.4 github.com/go-test/deep v1.0.8 github.com/google/uuid v1.3.0 - github.com/gophercloud/gophercloud v1.1.1 + github.com/gophercloud/gophercloud v1.5.0 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb github.com/hetznercloud/hcloud-go v1.39.0 - github.com/linode/linodego v1.10.0 + github.com/linode/linodego v1.20.1 github.com/nutanix-cloud-native/prism-go-client v0.3.4 - github.com/packethost/packngo v0.29.0 + github.com/packethost/packngo v0.30.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 - github.com/prometheus/client_golang v1.15.0 - github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10 + github.com/prometheus/client_golang v1.16.0 + github.com/scaleway/scaleway-sdk-go v1.0.0-beta.20 github.com/sethvargo/go-password v0.2.0 github.com/spf13/pflag v1.0.5 github.com/tinkerbell/tink v0.8.0 - github.com/vmware/go-vcloud-director/v2 v2.19.0 - github.com/vmware/govmomi v0.30.0 + github.com/vmware/go-vcloud-director/v2 v2.21.0 + github.com/vmware/govmomi v0.30.7 github.com/vultr/govultr/v2 v2.17.2 go.anx.io/go-anxcloud v0.5.3 - go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.10.0 - golang.org/x/oauth2 v0.5.0 - gomodules.xyz/jsonpatch/v2 v2.2.0 - google.golang.org/api v0.105.0 - google.golang.org/grpc v1.53.0 + go.uber.org/zap v1.25.0 + golang.org/x/crypto v0.12.0 + golang.org/x/oauth2 v0.11.0 + gomodules.xyz/jsonpatch/v2 v2.4.0 + google.golang.org/api v0.137.0 + google.golang.org/grpc v1.57.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.26.4 - k8s.io/apiextensions-apiserver v0.26.4 - k8s.io/apimachinery v0.26.4 - k8s.io/client-go v0.26.4 - k8s.io/cloud-provider v0.26.4 + k8s.io/api v0.28.0 + k8s.io/apiextensions-apiserver v0.28.0 + k8s.io/apimachinery v0.28.0 + k8s.io/client-go v0.28.0 + k8s.io/cloud-provider v0.28.0 k8s.io/klog v1.0.0 - k8s.io/kubelet v0.26.4 - k8s.io/utils v0.0.0-20230209194617-a36077c30491 - kubevirt.io/api v0.58.0 - kubevirt.io/containerized-data-importer-api v1.55.2 - sigs.k8s.io/controller-runtime v0.14.6 + k8s.io/kubelet v0.28.0 + k8s.io/utils v0.0.0-20230726121419-3b25d923346b + kubevirt.io/api v1.0.0 + kubevirt.io/containerized-data-importer-api v1.57.0 + sigs.k8s.io/controller-runtime v0.15.1 sigs.k8s.io/yaml v1.3.0 ) require ( - cloud.google.com/go v0.107.0 // indirect - cloud.google.com/go/compute v1.15.1 // indirect + cloud.google.com/go v0.110.7 // indirect + cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/longrunning v0.3.0 // indirect + cloud.google.com/go/longrunning v0.5.1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.28 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.21 // indirect + github.com/Azure/go-autorest/autorest v0.11.29 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/PaesslerAG/gval v1.2.1 // indirect + github.com/PaesslerAG/gval v1.2.2 // indirect github.com/PaesslerAG/jsonpath v0.1.1 // indirect github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.13.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/emicklei/go-restful/v3 v3.10.1 // indirect + github.com/emicklei/go-restful/v3 v3.10.2 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/flatcar/ignition v0.36.2 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.1 // indirect - github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-openapi/jsonpointer v0.20.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect github.com/go-resty/resty/v2 v2.7.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.4.3 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/gnostic v0.6.9 // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/google/s2a-go v0.1.5 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-retryablehttp v0.7.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.4 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect - github.com/imdario/mergo v0.3.13 // indirect + github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kolo/xmlrpc v0.0.0-20190717152603-07c4ee3fd181 // indirect + github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -145,46 +146,48 @@ require ( github.com/onsi/ginkgo/v2 v2.10.0 // indirect github.com/onsi/gomega v1.27.8 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 // indirect + github.com/openshift/api v0.0.0-20230815201604-a2362cf53230 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect - github.com/packethost/pkg v0.0.0-20211110202003-387414657e83 // indirect + github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046 // indirect github.com/peterhellberg/link v1.2.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.1 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect - github.com/spf13/cast v1.5.0 // indirect - github.com/spf13/cobra v1.6.1 // indirect + github.com/spf13/cast v1.5.1 // indirect + github.com/spf13/cobra v1.7.0 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 // indirect - go.opentelemetry.io/otel v1.11.2 // indirect - go.opentelemetry.io/otel/metric v0.34.0 // indirect - go.opentelemetry.io/otel/trace v1.11.2 // indirect - go.uber.org/atomic v1.10.0 // indirect - go.uber.org/multierr v1.9.0 // indirect - go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sync v0.2.0 // indirect - golang.org/x/sys v0.9.0 // indirect - golang.org/x/term v0.9.0 // indirect - golang.org/x/text v0.10.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 // indirect + go.opentelemetry.io/otel v1.16.0 // indirect + go.opentelemetry.io/otel/metric v1.16.0 // indirect + go.opentelemetry.io/otel/trace v1.16.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go4.org v0.0.0-20230225012048-214862532bf5 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sync v0.3.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/term v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.3 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/genproto v0.0.0-20230815205213-6bfd019c3878 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/component-base v0.26.4 // indirect - k8s.io/klog/v2 v2.90.1 // indirect - k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a // indirect + k8s.io/component-base v0.28.0 // indirect + k8s.io/klog/v2 v2.100.1 // indirect + k8s.io/kube-openapi v0.0.0-20230811205723-7ac0aad8c58d // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect ) diff --git a/go.sum b/go.sum index 7e5011d53..5b8cb0c49 100644 --- a/go.sum +++ b/go.sum @@ -18,27 +18,27 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww= -cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= +cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= -cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= -cloud.google.com/go/logging v1.6.1 h1:ZBsZK+JG+oCDT+vaxwqF2egKNRjz8soXiS6Xv79benI= -cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= -cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= -cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= -cloud.google.com/go/monitoring v1.9.1 h1:y9g09cWAQaX3ZYscR/nfaFUXtuyRqD2+i0jTOw0BZFI= -cloud.google.com/go/monitoring v1.9.1/go.mod h1:iFzRDMSDMvvf/z30Ge1jwtuEe/jlPPAFusmvCkUdo+o= +cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= +cloud.google.com/go/logging v1.8.1 h1:26skQWPeYhvIasWKm48+Eq7oUqdcdbwsCVwz5Ys0FvU= +cloud.google.com/go/logging v1.8.1/go.mod h1:TJjR+SimHwuC8MZ9cjByQulAMgni+RkXeI3wwctHJEI= +cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= +cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= +cloud.google.com/go/monitoring v1.15.1 h1:65JhLMd+JiYnXr6j5Z63dUYCuOg770p8a/VC+gil/58= +cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -57,14 +57,15 @@ github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= -github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= -github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= +github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= +github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= -github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= -github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 h1:P6bYXFoao05z5uhOQzbC3Qd8JqF3jUoocoTeIxkp2cA= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.11/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= +github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= +github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8= +github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.12/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc= github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= @@ -82,22 +83,23 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= -github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20220908162715-b27302cc7db5 h1:3SAiuS+PAdJHhDlTdnqseo9mZAcolUgDRS1PYEDUaFY= -github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20220908162715-b27302cc7db5/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= +github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20230725113508-e18d1b6d4ff8 h1:hzczEtHROO5bxA9QWva112sguv3REVxKWq5oTY5biOE= +github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20230725113508-e18d1b6d4ff8/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= -github.com/PaesslerAG/gval v1.2.1 h1:Ggwtej1xCyt1994VuDCSjycybIDo3duDCDghK/xc/A0= -github.com/PaesslerAG/gval v1.2.1/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac= +github.com/PaesslerAG/gval v1.2.2 h1:Y7iBzhgE09IGTt5QgGQ2IdaYYYOU134YGHBThD+wm9E= +github.com/PaesslerAG/gval v1.2.2/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac= github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8= github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk= github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY= @@ -117,8 +119,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.112 h1:49S6VGQeYyk2KIw85CHbAVaVF2lSgi8xrWDwSw0GCBM= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.112/go.mod h1:Api2AkmMgGaSUAhmk76oaFObkoeCPc/bKAqcyplPODs= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.512 h1:1r0pVpVs5XrgibXeBW7SwD9kU9ceTEq+LHjPEwxkKU0= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.512/go.mod h1:Api2AkmMgGaSUAhmk76oaFObkoeCPc/bKAqcyplPODs= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -129,32 +131,32 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/aws/aws-sdk-go v1.42.23/go.mod h1:gyRszuZ/icHmHAVE4gc/r+cfCmhA1AD+vqfWbgI+eHs= -github.com/aws/aws-sdk-go-v2 v1.17.3 h1:shN7NlnVzvDUgPQ+1rLMSxY8OWRNDRYtiqe0p/PgrhY= -github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= -github.com/aws/aws-sdk-go-v2/config v1.18.7 h1:V94lTcix6jouwmAsgQMAEBozVAGJMFhVj+6/++xfe3E= -github.com/aws/aws-sdk-go-v2/config v1.18.7/go.mod h1:OZYsyHFL5PB9UpyS78NElgKs11qI/B5KJau2XOJDXHA= -github.com/aws/aws-sdk-go-v2/credentials v1.13.7 h1:qUUcNS5Z1092XBFT66IJM7mYkMwgZ8fcC8YDIbEwXck= -github.com/aws/aws-sdk-go-v2/credentials v1.13.7/go.mod h1:AdCcbZXHQCjJh6NaH3pFaw8LUeBFn5+88BZGMVGuBT8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 h1:j9wi1kQ8b+e0FBVHxCqCGo4kxDU175hoDHcWAi0sauU= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21/go.mod h1:ugwW57Z5Z48bpvUyZuaPy4Kv+vEfJWnIrky7RmkBvJg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 h1:I3cakv2Uy1vNmmhRQmFptYDxOvBnwCdNwyw63N0RaRU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 h1:5NbbMrIzmUn/TXFqAle6mgrH5m9cOvMLRGL7pnG8tRE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9ebfHEqlhrMirFjSW0v0C9fI+KN5vk2kE= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 h1:KeTxcGdNnQudb46oOl4d90f2I33DF/c6q3RnZAmvQdQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28/go.mod h1:yRZVr/iT0AqyHeep00SZ4YfBAKojXz08w3XMBscdi0c= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0 h1:m6HYlpZlTWb9vHuuRHpWRieqPHWlS0mvQ90OJNrG/Nk= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0/go.mod h1:mV0E7631M1eXdB+tlGFIw6JxfsC7Pz7+7Aw15oLVhZw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 h1:5C6XgTViSb0bunmU57b3CT+MhxULqHH2721FVA+/kDM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21/go.mod h1:lRToEJsn+DRA9lW4O9L9+/3hjTkUzlzyzHqn8MTds5k= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 h1:gItLq3zBYyRDPmqAClgzTH8PBjDQGeyptYGHIwtYYNA= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.28/go.mod h1:wo/B7uUm/7zw/dWhBJ4FXuw1sySU5lyIhVg1Bu2yL9A= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 h1:KCacyVSs/wlcPGx37hcbT3IGYO8P8Jx+TgSDhAXtQMY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11/go.mod h1:TZSH7xLO7+phDtViY/KUp9WGCJMQkLJ/VpgkTFd5gh8= -github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 h1:9Mtq1KM6nD8/+HStvWcvYnixJ5N85DX+P+OY3kI3W2k= -github.com/aws/aws-sdk-go-v2/service/sts v1.17.7/go.mod h1:+lGbb3+1ugwKrNTWcf2RT05Xmp543B06zDFTwiTLp7I= -github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= -github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/aws-sdk-go-v2 v1.20.1 h1:rZBf5DWr7YGrnlTK4kgDQGn1ltqOg5orCYb/UhOFZkg= +github.com/aws/aws-sdk-go-v2 v1.20.1/go.mod h1:NU06lETsFm8fUC6ZjhgDpVBcGZTFQ6XM+LZWZxMI4ac= +github.com/aws/aws-sdk-go-v2/config v1.18.33 h1:JKcw5SFxFW/rpM4mOPjv0VQ11E2kxW13F3exWOy7VZU= +github.com/aws/aws-sdk-go-v2/config v1.18.33/go.mod h1:hXO/l9pgY3K5oZJldamP0pbZHdPqqk+4/maa7DSD3cA= +github.com/aws/aws-sdk-go-v2/credentials v1.13.32 h1:lIH1eKPcCY1ylR4B6PkBGRWMHO3aVenOKJHWiS4/G2w= +github.com/aws/aws-sdk-go-v2/credentials v1.13.32/go.mod h1:lL8U3v/Y79YRG69WlAho0OHIKUXCyFvSXaIvfo81sls= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8 h1:DK/9C+UN/X+1+Wm8pqaDksQr2tSLzq+8X1/rI/ZxKEQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8/go.mod h1:ce7BgLQfYr5hQFdy67oX2svto3ufGtm6oBvmsHScI1Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 h1:c8ed/T9T2K5I+h/JzmF5tpI46+OODQ74dzmdo+QnaMg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38/go.mod h1:qggunOChCMu9ZF/UkAfhTz25+U2rLVb3ya0Ua6TTfCA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 h1:hNeAAymUY5gu11WrrmFb3CVIp9Dar9hbo44yzzcQpzA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32/go.mod h1:0ZXSqrty4FtQ7p8TEuRde/SZm9X05KT18LAUlR40Ln0= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 h1:fc0ukRAiP1syoSGZYu+DaE+FulSYhTiJ8WpVu5jElU4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39/go.mod h1:WLAW8PT7+JhjZfLSWe7WEJaJu0GNo0cKc2Zyo003RBs= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.112.0 h1:8I4NQ9BfrQATHzXKtBuu+jBdOVd2mBANqhbMOXfSIdA= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.112.0/go.mod h1:Ie0Kp61cLk223argiS+t8vO29SpbFIphzlPflIvYcv0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32 h1:dGAseBFEYxth10V23b5e2mAS+tX7oVbfYHD6dnDdAsg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32/go.mod h1:4jwAWKEkCR0anWk5+1RbfSg1R5Gzld7NLiuaq5bTR/Y= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.2 h1:A2RlEMo4SJSwbNoUUgkxTAEMduAy/8wG3eB2b2lP4gY= +github.com/aws/aws-sdk-go-v2/service/sso v1.13.2/go.mod h1:ju+nNXUunfIFamXUIZQiICjnO/TPlOmWcYhZcSy7xaE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2 h1:OJELEgyaT2kmaBGZ+myyZbTTLobfe3ox3FSh5eYK9Qs= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2/go.mod h1:ubDBBaDFs1GHijSOTi8ljppML15GLG0HxhILtbjNNYQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.21.2 h1:ympg1+Lnq33XLhcK/xTG4yZHPs1Oyxu+6DEWbl7qOzA= +github.com/aws/aws-sdk-go-v2/service/sts v1.21.2/go.mod h1:FQ/DQcOfESELfJi5ED+IPPAjI5xC6nxtSolVVB773jM= +github.com/aws/smithy-go v1.14.1 h1:EFKMUmH/iHMqLiwoEDx2rRjRQpI1YCn5jTysoaDujFs= +github.com/aws/smithy-go v1.14.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -165,7 +167,6 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bnkamalesh/webgo/v4 v4.1.11/go.mod h1:taIAonQTzao8G5rnB22WgKmQuIOWHpQ0n/YLAidBXlM= github.com/bnkamalesh/webgo/v6 v6.2.2/go.mod h1:2Y+dEdTp1xC/ra+3PAVZV6hh4sCI+iPK7mcHt+t9bfM= -github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -184,9 +185,11 @@ github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/coreos/go-semver v0.1.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -196,23 +199,18 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46t github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creasty/defaults v1.5.2 h1:/VfB6uxpyp6h0fr7SPp7n8WJBoV8jfxQXPCnkVSjyls= github.com/creasty/defaults v1.5.2/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY= -github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= -github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= -github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= -github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= -github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= -github.com/digitalocean/godo v1.93.0 h1:N0K9z2yssZVP7nBHQ32P1Wemd5yeiJdH4ROg+7ySRxY= -github.com/digitalocean/godo v1.93.0/go.mod h1:NRpFznZFvhHjBoqZAaOD3khVzsJ3EibzKqFL4R60dmA= +github.com/digitalocean/godo v1.102.0 h1:iXkLEHmrBi5n9TAbRcKM0182NwXYrp1xCj//7vMRFs4= +github.com/digitalocean/godo v1.102.0/go.mod h1:SaUYccN7r+CO1QtsbXGypAsgobDrmSfVMJESEfXgoEg= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= @@ -220,8 +218,8 @@ github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkg github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= -github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE= +github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -230,7 +228,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/envoyproxy/protoc-gen-validate v0.10.1 h1:c0g45+xCJhdgFGw7a5QAfdS4byAbud7miNWJ1WwEVf8= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -240,10 +238,9 @@ github.com/flatcar/container-linux-config-transpiler v0.9.4 h1:yXQ0NB8PeNrKJPrZv github.com/flatcar/container-linux-config-transpiler v0.9.4/go.mod h1:LxanhPvXkWgHG9PrkT4rX/p7YhUPdDGGsUdkNpV3L5U= github.com/flatcar/ignition v0.36.2 h1:xGHgScUe0P4Fkprjqv7L2CE58emiQgP833OCCn9z2v4= github.com/flatcar/ignition v0.36.2/go.mod h1:uk1tpzLFRXus4RrvzgMI+IqmmB8a/RGFSBlI+tMTbbA= -github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -277,22 +274,25 @@ github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= +github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= +github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= +github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= -github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= @@ -320,8 +320,8 @@ github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptG github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= -github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -360,8 +360,8 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= -github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= -github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -386,7 +386,6 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -402,21 +401,23 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.5 h1:8IYp3w9nysqv3JH+NJgXJzGbDHzLOTj43BmSkp+O7qg= +github.com/google/s2a-go v0.1.5/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.1 h1:RY7tHKZcRlk788d5WSo/e83gOyyy742E8GSs771ySpg= -github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= +github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= -github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gophercloud/gophercloud v1.1.1 h1:MuGyqbSxiuVBqkPZ3+Nhbytk1xZxhmfCB2Rg1cJWFWM= -github.com/gophercloud/gophercloud v1.1.1/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= +github.com/gophercloud/gophercloud v1.5.0 h1:cDN6XFCLKiiqvYpjQLq9AiM7RDRbIC9450WpPH+yvXo= +github.com/gophercloud/gophercloud v1.5.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -429,8 +430,8 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= -github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= +github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -448,9 +449,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -485,8 +485,9 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/kolo/xmlrpc v0.0.0-20190717152603-07c4ee3fd181 h1:TrxPzApUukas24OMMVDUMlCs1XCExJtnGaDEiIAR4oQ= github.com/kolo/xmlrpc v0.0.0-20190717152603-07c4ee3fd181/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= +github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00= +github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= @@ -511,8 +512,8 @@ github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++ github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= github.com/lestrrat-go/jwx v1.2.20/go.mod h1:tLE1XszaFgd7zaS5wHe4NxA+XVhu7xgdRvDpNyi3kNM= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/linode/linodego v1.10.0 h1:nH/BffTBQEZr48q/9UszuB5dhWpGKuVuJs/uE9Nweuc= -github.com/linode/linodego v1.10.0/go.mod h1:lRWOfS3HmRV63U6Rt+llKziobIwpySYGlCdTIHoIgps= +github.com/linode/linodego v1.20.1 h1:IW3SrZjRzrclYZnzFd80f8lSkTYAM7gVTJW0t7HnFKQ= +github.com/linode/linodego v1.20.1/go.mod h1:ggoWnJXssx9wPWNnR3x7WaOpOBOEhsPB/HO7iflF5qY= github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -549,6 +550,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -582,18 +584,17 @@ github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 h1:+ShYlGoPriGahTTFTjQ0RtNXW0srxDodk2STdc238Rk= -github.com/openshift/api v0.0.0-20211217221424-8779abfbd571/go.mod h1:F/eU6jgr6Q2VhMu1mSpMmygxAELd7+BUxs3NHZ25jV4= -github.com/openshift/build-machinery-go v0.0.0-20211213093930-7e33a7eb4ce3/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE= +github.com/openshift/api v0.0.0-20230815201604-a2362cf53230 h1:PY2JBJdSkzTxathfsYMa/Mb2Xcw9YphTZ6IcvURayKk= +github.com/openshift/api v0.0.0-20230815201604-a2362cf53230/go.mod h1:yimSGmjsI+XF1mr+AKBs2//fSXIOhhetHGbMlBEfXbs= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= -github.com/packethost/packngo v0.29.0 h1:gRIhciVZQ/zLNrIdIdbOUyB/Tw5IgoaXyhP4bvE+D2s= -github.com/packethost/packngo v0.29.0/go.mod h1:/UHguFdPs6Lf6FOkkSEPnRY5tgS0fsVM+Zv/bvBrmt0= -github.com/packethost/pkg v0.0.0-20211110202003-387414657e83 h1:uhBvTY/Hnm7rLz7gPkA83JU4EQf4A2YZUBry6+Gyn9g= -github.com/packethost/pkg v0.0.0-20211110202003-387414657e83/go.mod h1:iF7Mj6XXQ6O+bCfrBCrsJrIGxG7ptrZwb0bW91+wzm8= +github.com/packethost/packngo v0.30.0 h1:JVeTwbXXETsLTDQncUbYwIFpkOp/xevXrffM2HrFECI= +github.com/packethost/packngo v0.30.0/go.mod h1:BT/XcdwLVmeMtGPbovnxCpnI1s9ylSE1cs/7pq007NE= +github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046 h1:zF+CUhv8LMpqTFFpECX6WF+yUWS2Bd1Nc1W+AczzqbY= +github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046/go.mod h1:W/xTaqgJ2kJCwayvm3BF3bOj9ku0F5DjjYnZaioxnOk= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= @@ -614,27 +615,27 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= -github.com/prometheus/client_golang v1.15.0/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.31.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -642,18 +643,17 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rollbar/rollbar-go v1.4.2/go.mod h1:kLQ9gP3WCRGrvJmF0ueO3wK9xWocej8GRX98D8sa39w= github.com/rollbar/rollbar-go/errors v0.0.0-20210929193720-32947096267e/go.mod h1:Ie0xEc1Cyj+T4XMO8s0Vf7pMfvSAAy1sb4AYc8aJsao= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10 h1:wsfMs0iv+MJiViM37qh5VEKISi3/ZUq2nNKNdqmumAs= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.20 h1:a9hSJdJcd16e0HoMsnFvaHvxB3pxSD+SC7+CISp7xY0= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.20/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI= github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE= @@ -671,10 +671,10 @@ github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= -github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= -github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= +github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= @@ -691,8 +691,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinkerbell/lint-install v0.0.0-20211012174934-5ee5ab01db76/go.mod h1:0h2KsALaQLNkoVeV+G+HjBWWCnp0COFYhJdRd5WCQPM= github.com/tinkerbell/tink v0.8.0 h1:qgl/rglpO5Rvq6UKZd29O6X9mDgZZYgf841+Y0IYWak= @@ -713,10 +714,10 @@ github.com/vektah/gqlparser/v2 v2.2.0 h1:bAc3slekAAJW6sZTi07aGq0OrfaCjj4jxARAaC7 github.com/vektah/gqlparser/v2 v2.2.0/go.mod h1:i3mQIGIrbK2PD1RrCeMTlVbkF2FJ6WkU1KJlJlC+3F4= github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= -github.com/vmware/go-vcloud-director/v2 v2.19.0 h1:A9p95VLn50dm7JbXqg5q+VmQxu3RxoMH6OD5ZeLK9EQ= -github.com/vmware/go-vcloud-director/v2 v2.19.0/go.mod h1:QPxGFgrUcSyzy9IlpwDE4UNT3tsOy2047tJOPEJ4nlw= -github.com/vmware/govmomi v0.30.0 h1:Fm8ugPnnlMSTSceDKY9goGvjmqc6eQLPUSUeNXdpeXA= -github.com/vmware/govmomi v0.30.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= +github.com/vmware/go-vcloud-director/v2 v2.21.0 h1:zIONrJpM+Fj+rDyXmsRfMAn1sP5WAP87USL0T9GS4DY= +github.com/vmware/go-vcloud-director/v2 v2.21.0/go.mod h1:QPxGFgrUcSyzy9IlpwDE4UNT3tsOy2047tJOPEJ4nlw= +github.com/vmware/govmomi v0.30.7 h1:YO8CcDpLJzmq6PK5/CBQbXyV21iCMh8SbdXt+xNkXp8= +github.com/vmware/govmomi v0.30.7/go.mod h1:epgoslm97rLECMV4D+08ORzUBEU7boFSepKjt7AYVGg= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs= @@ -724,9 +725,6 @@ github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -752,41 +750,42 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 h1:+uFejS4DCfNH6d3xODVIGsdhzgzhh45p9gpbHQMbdZI= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0/go.mod h1:HSmzQvagH8pS2/xrK7ScWsk0vAMtRTGbMFgInXCi8Tc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 h1:ZOLJc06r4CB42laIXg/7udr0pbZyuAihN10A/XuiQRY= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0/go.mod h1:5z+/ZWJQKXa9YT34fQNx5K8Hd1EoIhvtUygUQPqEOgQ= go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= -go.opentelemetry.io/otel v1.11.2 h1:YBZcQlsVekzFsFbjygXMOXSs6pialIZxcjfO/mBDmR0= -go.opentelemetry.io/otel v1.11.2/go.mod h1:7p4EUV+AqgdlNV9gL97IgUZiVR3yrFXYo53f9BM3tRI= -go.opentelemetry.io/otel/metric v0.34.0 h1:MCPoQxcg/26EuuJwpYN1mZTeCYAUGx8ABxfW07YkjP8= -go.opentelemetry.io/otel/metric v0.34.0/go.mod h1:ZFuI4yQGNCupurTXCwkeD/zHBt+C2bR7bw5JqUm/AP8= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= -go.opentelemetry.io/otel/trace v1.11.2 h1:Xf7hWSF2Glv0DE3MH7fBHvtpSBsjcBUe5MYAmZM/+y0= -go.opentelemetry.io/otel/trace v1.11.2/go.mod h1:4N+yC7QEz7TTsG9BSRLNAa63eg5E06ObSbKPmxQ/pKA= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= -go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= +go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= go4.org v0.0.0-20160314031811-03efcb870d84/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= -go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= -golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= +go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= +go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -801,12 +800,14 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= -golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -889,7 +890,6 @@ golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -899,8 +899,10 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -914,8 +916,8 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s= -golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -928,10 +930,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1004,14 +1005,16 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= -golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1021,9 +1024,11 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1070,7 +1075,6 @@ golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -1099,8 +1103,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= -gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1122,8 +1126,8 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.105.0 h1:t6P9Jj+6XTn4U9I2wycQai6Q/Kz7iOT+QzjJ3G2V4x8= -google.golang.org/api v0.105.0/go.mod h1:qh7eD5FJks5+BcE+cjBIm6Gz8vioK7EHvnlniqXBnqI= +google.golang.org/api v0.137.0 h1:QrKX6uNvzJLr0Fd3vWVqcyrcmFoYi036VUAsZbiF4+s= +google.golang.org/api v0.137.0/go.mod h1:4xyob8CxC+0GChNBvEUAk8VBKNvYOTWM9T3v3UfRxuY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1176,9 +1180,12 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230815205213-6bfd019c3878 h1:Iveh6tGCJkHAjJgEqUQYGDGgbwmhjoAOz8kO/ajxefY= +google.golang.org/genproto v0.0.0-20230815205213-6bfd019c3878/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878 h1:WGq4lvB/mlicysM/dUT3SBvijH4D3sm/Ny1A4wmt2CI= +google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 h1:lv6/DhyiFFGsmzxbsUUTOkN29II+zeWHxvT8Lpdxsv0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1201,8 +1208,9 @@ google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc/examples v0.0.0-20210728214646-ad0a2a847cdf/go.mod h1:bF8wuZSAZTcbF7ZPKrDI/qY52toTP/yxLpRRY4Eu9Js= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1218,8 +1226,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1238,7 +1246,6 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= @@ -1254,7 +1261,6 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1264,28 +1270,25 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= -k8s.io/api v0.26.4 h1:qSG2PmtcD23BkYiWfoYAcak870eF/hE7NNYBYavTT94= -k8s.io/api v0.26.4/go.mod h1:WwKEXU3R1rgCZ77AYa7DFksd9/BAIKyOmRlbVxgvjCk= -k8s.io/apiextensions-apiserver v0.26.4 h1:9D2RTxYGxrG5uYg6D7QZRcykXvavBvcA59j5kTaedQI= -k8s.io/apiextensions-apiserver v0.26.4/go.mod h1:cd4uGFGIgzEqUghWpRsr9KE8j2KNTjY8Ji8pnMMazyw= -k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc= +k8s.io/api v0.28.0 h1:3j3VPWmN9tTDI68NETBWlDiA9qOiGJ7sdKeufehBYsM= +k8s.io/api v0.28.0/go.mod h1:0l8NZJzB0i/etuWnIXcwfIv+xnDOhL3lLW919AWYDuY= +k8s.io/apiextensions-apiserver v0.28.0 h1:CszgmBL8CizEnj4sj7/PtLGey6Na3YgWyGCPONv7E9E= +k8s.io/apiextensions-apiserver v0.28.0/go.mod h1:uRdYiwIuu0SyqJKriKmqEN2jThIJPhVmOWETm8ud1VE= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apimachinery v0.26.4 h1:rZccKdBLg9vP6J09JD+z8Yr99Ce8gk3Lbi9TCx05Jzs= -k8s.io/apimachinery v0.26.4/go.mod h1:ats7nN1LExKHvJ9TmwootT00Yz05MuYqPXEXaVeOy5I= +k8s.io/apimachinery v0.28.0 h1:ScHS2AG16UlYWk63r46oU3D5y54T53cVI5mMJwwqFNA= +k8s.io/apimachinery v0.28.0/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= -k8s.io/client-go v0.26.4 h1:/7P/IbGBuT73A+G97trf44NTPSNqvuBREpOfdLbHvD4= -k8s.io/client-go v0.26.4/go.mod h1:6qOItWm3EwxJdl/8p5t7FWtWUOwyMdA8N9ekbW4idpI= -k8s.io/cloud-provider v0.26.4 h1:mqN4vhC4mRoMi+ujI92ImkIOuYS7ZS55FvXB10d6Wp4= -k8s.io/cloud-provider v0.26.4/go.mod h1:F9xY0PvBuZDuGIHOM28dNiPLHxQnWfsiUuCSUikHevo= -k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE= +k8s.io/client-go v0.28.0 h1:ebcPRDZsCjpj62+cMk1eGNX1QkMdRmQ6lmz5BLoFWeM= +k8s.io/client-go v0.28.0/go.mod h1:0Asy9Xt3U98RypWJmU1ZrRAGKhP6NqDPmptlAzK2kMc= +k8s.io/cloud-provider v0.28.0 h1:BTIW7b757T+VXB5yqJeajPXsNOmeooopUgfzQueiWvk= +k8s.io/cloud-provider v0.28.0/go.mod h1:u0MGqdlutkTmCJyNrCzIMJ+OhrwQE9x5X8mBTN0R7us= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.26.4 h1:Bg2xzyXNKL3eAuiTEu3XE198d6z22ENgFgGQv2GGOUk= -k8s.io/component-base v0.26.4/go.mod h1:lTuWL1Xz/a4e80gmIC3YZG2JCO4xNwtKWHJWeJmsq20= +k8s.io/component-base v0.28.0 h1:HQKy1enJrOeJlTlN4a6dU09wtmXaUvThC0irImfqyxI= +k8s.io/component-base v0.28.0/go.mod h1:Yyf3+ZypLfMydVzuLBqJ5V7Kx6WwDr/5cN+dFjw1FNk= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= @@ -1296,42 +1299,40 @@ k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= -k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= -k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a h1:gmovKNur38vgoWfGtP5QOGNOA7ki4n6qNYoFAgMlNvg= -k8s.io/kube-openapi v0.0.0-20230308215209-15aac26d736a/go.mod h1:y5VtZWM9sHHc2ZodIH/6SHzXj+TPU5USoA8lcIeKEKY= -k8s.io/kubelet v0.26.4 h1:SEQPfjN4lu4uL9O8NdeN7Aum3liQ4kOnp/yC3jMRMUo= -k8s.io/kubelet v0.26.4/go.mod h1:ZMPGTCnrQ5UOlC7igXhbW9cgna1LtTRWLaHub4dA2FU= +k8s.io/kube-openapi v0.0.0-20230811205723-7ac0aad8c58d h1:lGN9colvFFZIY0Guxkzdd73CoDXHIbhlQkfi7n/26Ak= +k8s.io/kube-openapi v0.0.0-20230811205723-7ac0aad8c58d/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= +k8s.io/kubelet v0.28.0 h1:H/3JAkLIungVF+WLpqrxhgJ4gzwsbN8VA8LOTYsEX3U= +k8s.io/kubelet v0.28.0/go.mod h1:i8jUg4ltbRusT3ExOhSAeqETuHdoHTZcTT2cPr9RTgc= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= -k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -kubevirt.io/api v0.58.0 h1:qeNeRtD6AIJ5WVJuRXajmmXtnrO5dYchy+hpCm6QwhE= -kubevirt.io/api v0.58.0/go.mod h1:U0CQlZR0JoJCaC+Va0wz4dMOtYDdVywJ98OT1KmOkzI= -kubevirt.io/containerized-data-importer-api v1.55.2 h1:AzYnKIUFkKwO6c0uCQZYlAIxfzbiPkJXP29hFhauaQ8= -kubevirt.io/containerized-data-importer-api v1.55.2/go.mod h1:92HiQEyzPoeMiCbgfG5Qe10JQVbtWMZOXucy56dKdGg= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +kubevirt.io/api v1.0.0 h1:RBdXP5CDhE0v5qL2OUQdrYyRrHe/F68Z91GWqBDF6nw= +kubevirt.io/api v1.0.0/go.mod h1:CJ4vZsaWhVN3jNbyc9y3lIZhw8nUHbWjap0xHABQiqc= +kubevirt.io/containerized-data-importer-api v1.57.0 h1:IpRCUyDS0x7BaVa5q5MCzuWRAfvXT54GpEnNJke5hSE= +kubevirt.io/containerized-data-importer-api v1.57.0/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 h1:fZYvD3/Vnitfkx6IJxjLAk8ugnZQ7CXVYcRfkSKmuZY= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= -sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= +sigs.k8s.io/controller-runtime v0.15.1 h1:9UvgKD4ZJGcj24vefUFgZFP3xej/3igL9BsOUTb/+4c= +sigs.k8s.io/controller-runtime v0.15.1/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/pkg/apis/cluster/v1alpha1/migrations/migrations.go b/pkg/apis/cluster/v1alpha1/migrations/migrations.go index a65cb91af..08b70244a 100644 --- a/pkg/apis/cluster/v1alpha1/migrations/migrations.go +++ b/pkg/apis/cluster/v1alpha1/migrations/migrations.go @@ -156,8 +156,8 @@ func MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary( crdLog := log.With("crd", machines.CRDName) - err := wait.Poll(cachePopulatingInterval, cachePopulatingTimeout, func() (done bool, err error) { - err = client.Get(ctx, types.NamespacedName{Name: machines.CRDName}, &apiextensionsv1.CustomResourceDefinition{}) + err := wait.PollUntilContextTimeout(ctx, cachePopulatingInterval, cachePopulatingTimeout, false, func(ctx context.Context) (bool, error) { + err := client.Get(ctx, types.NamespacedName{Name: machines.CRDName}, &apiextensionsv1.CustomResourceDefinition{}) if err != nil { if kerrors.IsNotFound(err) { noMigrationNeed = true @@ -388,7 +388,7 @@ func deleteMachinesV1Alpha1Machine(ctx context.Context, return fmt.Errorf("failed to delete machine %s: %w", machine.Name, err) } - if err := wait.Poll(500*time.Millisecond, 60*time.Second, func() (bool, error) { + if err := wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, 60*time.Second, false, func(ctx context.Context) (bool, error) { return isMachinesV1Alpha1MachineDeleted(ctx, machine.Name, client) }); err != nil { return fmt.Errorf("failed to wait for machine %s to be deleted: %w", machine.Name, err) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 55449c4f7..442d95585 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -53,7 +53,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/metrics" ) @@ -452,19 +452,19 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if err != nil { return nil, nil, nil, err } - c.SpotMaxPrice = pointer.String(maxPrice) + c.SpotMaxPrice = ptr.To(maxPrice) persistentRequest, _, err := p.configVarResolver.GetConfigVarBoolValue(rawConfig.SpotInstanceConfig.PersistentRequest) if err != nil { return nil, nil, nil, err } - c.SpotPersistentRequest = pointer.Bool(persistentRequest) + c.SpotPersistentRequest = ptr.To(persistentRequest) interruptionBehavior, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.SpotInstanceConfig.InterruptionBehavior) if err != nil { return nil, nil, nil, err } - c.SpotInterruptionBehavior = pointer.String(interruptionBehavior) + c.SpotInterruptionBehavior = ptr.To(interruptionBehavior) } assumeRoleARN, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AssumeRoleARN, "AWS_ASSUME_ROLE_ARN") if err != nil { @@ -495,7 +495,7 @@ func getAwsConfig(ctx context.Context, id, secret, token, region, assumeRoleARN, stsSvc := sts.NewFromConfig(cfg) creds := stscreds.NewAssumeRoleProvider(stsSvc, assumeRoleARN, func(o *stscreds.AssumeRoleOptions) { - o.ExternalID = pointer.String(assumeRoleExternalID) + o.ExternalID = ptr.To(assumeRoleExternalID) }, ) @@ -582,7 +582,7 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus // noop case util.IPFamilyIPv6, util.IPFamilyIPv4IPv6, util.IPFamilyIPv6IPv4: if len(vpc.Ipv6CidrBlockAssociationSet) == 0 { - return fmt.Errorf("vpc %s does not have IPv6 CIDR block", pointer.StringDeref(vpc.VpcId, "")) + return fmt.Errorf("vpc %s does not have IPv6 CIDR block", ptr.Deref(vpc.VpcId, "")) } default: return fmt.Errorf(util.ErrUnknownNetworkFamily, f) @@ -770,7 +770,7 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * DeleteOnTermination: aws.Bool(true), VolumeType: config.DiskType, Iops: config.DiskIops, - Encrypted: pointer.Bool(config.EBSVolumeEncrypted), + Encrypted: ptr.To(config.EBSVolumeEncrypted), }, }, }, @@ -994,7 +994,7 @@ func (d *awsInstance) Name() string { } func (d *awsInstance) ID() string { - return pointer.StringDeref(d.instance.InstanceId, "") + return ptr.Deref(d.instance.InstanceId, "") } func (d *awsInstance) ProviderID() string { @@ -1010,15 +1010,15 @@ func (d *awsInstance) ProviderID() string { func (d *awsInstance) Addresses() map[string]v1.NodeAddressType { addresses := map[string]v1.NodeAddressType{ - pointer.StringDeref(d.instance.PublicIpAddress, ""): v1.NodeExternalIP, - pointer.StringDeref(d.instance.PublicDnsName, ""): v1.NodeExternalDNS, - pointer.StringDeref(d.instance.PrivateIpAddress, ""): v1.NodeInternalIP, - pointer.StringDeref(d.instance.PrivateDnsName, ""): v1.NodeInternalDNS, + ptr.Deref(d.instance.PublicIpAddress, ""): v1.NodeExternalIP, + ptr.Deref(d.instance.PublicDnsName, ""): v1.NodeExternalDNS, + ptr.Deref(d.instance.PrivateIpAddress, ""): v1.NodeInternalIP, + ptr.Deref(d.instance.PrivateDnsName, ""): v1.NodeInternalDNS, } for _, netInterface := range d.instance.NetworkInterfaces { for _, addr := range netInterface.Ipv6Addresses { - ipAddr := pointer.StringDeref(addr.Ipv6Address, "") + ipAddr := ptr.Deref(addr.Ipv6Address, "") // link-local addresses not very useful in machine status // filter them out @@ -1231,7 +1231,7 @@ func filterSupportedRHELImages(images []ec2types.Image) ([]ec2types.Image, error // This happens more often in some AWS regions because some regions have // slower instance creation (e.g. us-east-1 and us-west-2). func (p *provider) waitForInstance(ctx context.Context, machine *clusterv1alpha1.Machine) error { - return wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) { + return wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, false, func(ctx context.Context) (bool, error) { _, err := p.get(ctx, machine) if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { // Retry if instance is not found diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 461693401..b6a90b70a 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -47,7 +47,7 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ) const ( @@ -175,19 +175,19 @@ var imageReferences = map[providerconfigtypes.OperatingSystem]compute.ImageRefer var osPlans = map[providerconfigtypes.OperatingSystem]*compute.Plan{ providerconfigtypes.OperatingSystemFlatcar: { - Name: pointer.String("stable"), - Publisher: pointer.String("kinvolk"), - Product: pointer.String("flatcar-container-linux"), + Name: ptr.To("stable"), + Publisher: ptr.To("kinvolk"), + Product: ptr.To("flatcar-container-linux"), }, providerconfigtypes.OperatingSystemRHEL: { - Name: pointer.String("rhel-lvm85"), - Publisher: pointer.String("redhat"), - Product: pointer.String("rhel-byos"), + Name: ptr.To("rhel-lvm85"), + Publisher: ptr.To("redhat"), + Product: ptr.To("rhel-byos"), }, providerconfigtypes.OperatingSystemRockyLinux: { - Name: pointer.String("rocky-linux-8-5"), - Publisher: pointer.String("procomputers"), - Product: pointer.String("rocky-linux-8-5"), + Name: ptr.To("rocky-linux-8-5"), + Publisher: ptr.To("procomputers"), + Product: ptr.To("rocky-linux-8-5"), }, } @@ -361,18 +361,18 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p if rawCfg.ImagePlan != nil && rawCfg.ImagePlan.Name != "" { c.ImagePlan = &compute.Plan{ - Name: pointer.String(rawCfg.ImagePlan.Name), - Publisher: pointer.String(rawCfg.ImagePlan.Publisher), - Product: pointer.String(rawCfg.ImagePlan.Product), + Name: ptr.To(rawCfg.ImagePlan.Name), + Publisher: ptr.To(rawCfg.ImagePlan.Publisher), + Product: ptr.To(rawCfg.ImagePlan.Product), } } if rawCfg.ImageReference != nil { c.ImageReference = &compute.ImageReference{ - Publisher: pointer.String(rawCfg.ImageReference.Publisher), - Offer: pointer.String(rawCfg.ImageReference.Offer), - Sku: pointer.String(rawCfg.ImageReference.Sku), - Version: pointer.String(rawCfg.ImageReference.Version), + Publisher: ptr.To(rawCfg.ImageReference.Publisher), + Offer: ptr.To(rawCfg.ImageReference.Offer), + Sku: ptr.To(rawCfg.ImageReference.Sku), + Version: ptr.To(rawCfg.ImageReference.Version), } } @@ -543,7 +543,7 @@ func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) } if config.OSDiskSize != 0 { sp.OsDisk = &compute.OSDisk{ - DiskSizeGB: pointer.Int32(config.OSDiskSize), + DiskSizeGB: ptr.To(config.OSDiskSize), CreateOption: compute.DiskCreateOptionTypesFromImage, } @@ -559,7 +559,7 @@ func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) { // this should be in range 0-63 and should be unique per datadisk, since we have only one datadisk, this should be fine Lun: new(int32), - DiskSizeGB: pointer.Int32(config.DataDiskSize), + DiskSizeGB: ptr.To(config.DataDiskSize), CreateOption: compute.DiskCreateOptionTypesEmpty, }, } @@ -706,7 +706,7 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * if config.EnableBootDiagnostics { vmSpec.DiagnosticsProfile = &compute.DiagnosticsProfile{ BootDiagnostics: &compute.BootDiagnostics{ - Enabled: pointer.Bool(config.EnableBootDiagnostics), + Enabled: ptr.To(config.EnableBootDiagnostics), }, } } diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index 0f686f2e1..928d3bc7e 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -327,7 +327,7 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * dropletLog := log.With("droplet", droplet.ID) //We need to wait until the droplet really got created as tags will be only applied when the droplet is running - err = wait.Poll(createCheckPeriod, createCheckTimeout, func() (done bool, err error) { + err = wait.PollUntilContextTimeout(ctx, createCheckPeriod, createCheckTimeout, false, func(ctx context.Context) (bool, error) { newDroplet, rsp, err := client.Droplets.Get(ctx, droplet.ID) if err != nil { tErr := doStatusAndErrToTerminalError(rsp.StatusCode, err) diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 1e1cf46ae..4c81cd274 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -144,18 +144,18 @@ func (p *Provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec cluste } // Get retrieves a node instance that is associated with the given machine. -func (p *Provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - return p.get(machine) +func (p *Provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { + return p.get(ctx, machine) } -func (p *Provider) get(machine *clusterv1alpha1.Machine) (*googleInstance, error) { +func (p *Provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (*googleInstance, error) { // Read configuration. cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, errMachineSpec, err) } // Connect to Google compute. - svc, err := connectComputeService(cfg) + svc, err := connectComputeService(ctx, cfg) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, errConnect, err) } @@ -218,7 +218,7 @@ func (p *Provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * return nil, newError(common.InvalidConfigurationMachineError, errMachineSpec, err) } // Connect to Google compute. - svc, err := connectComputeService(cfg) + svc, err := connectComputeService(ctx, cfg) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, errConnect, err) } @@ -295,7 +295,7 @@ func (p *Provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * if err != nil { return nil, newError(common.InvalidConfigurationMachineError, errInsertInstance, err) } - err = svc.waitZoneOperation(cfg, op.Name) + err = svc.waitZoneOperation(ctx, cfg, op.Name) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, errInsertInstance, err) } @@ -304,14 +304,14 @@ func (p *Provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * } // Cleanup deletes the instance associated with the machine and all associated resources. -func (p *Provider) Cleanup(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { +func (p *Provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { // Read configuration. cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) if err != nil { return false, newError(common.InvalidConfigurationMachineError, errMachineSpec, err) } // Connect to Google compute. - svc, err := connectComputeService(cfg) + svc, err := connectComputeService(ctx, cfg) if err != nil { return false, newError(common.InvalidConfigurationMachineError, errConnect, err) } @@ -326,7 +326,7 @@ func (p *Provider) Cleanup(_ context.Context, _ *zap.SugaredLogger, machine *clu } return false, newError(common.InvalidConfigurationMachineError, errDeleteInstance, err) } - err = svc.waitZoneOperation(cfg, op.Name) + err = svc.waitZoneOperation(ctx, cfg, op.Name) if err != nil { return false, newError(common.InvalidConfigurationMachineError, errDeleteInstance, err) } @@ -355,19 +355,19 @@ func (p *Provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s // MigrateUID updates the UID of an instance after the controller migrates types // and the UID of the machine object changed. -func (p *Provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { +func (p *Provider) MigrateUID(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newUID types.UID) error { // Read configuration. cfg, err := newConfig(p.resolver, machine.Spec.ProviderSpec) if err != nil { return newError(common.InvalidConfigurationMachineError, errMachineSpec, err) } // Connect to Google compute. - svc, err := connectComputeService(cfg) + svc, err := connectComputeService(ctx, cfg) if err != nil { return newError(common.InvalidConfigurationMachineError, errConnect, err) } // Retrieve instance. - inst, err := p.get(machine) + inst, err := p.get(ctx, machine) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return nil @@ -389,7 +389,7 @@ func (p *Provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, machine * if err != nil { return newError(common.InvalidConfigurationMachineError, errSetLabels, err) } - err = svc.waitZoneOperation(cfg, op.Name) + err = svc.waitZoneOperation(ctx, cfg, op.Name) if err != nil { return newError(common.InvalidConfigurationMachineError, errSetLabels, err) } diff --git a/pkg/cloudprovider/provider/gce/service.go b/pkg/cloudprovider/provider/gce/service.go index 2f451d4ab..4c1dd9570 100644 --- a/pkg/cloudprovider/provider/gce/service.go +++ b/pkg/cloudprovider/provider/gce/service.go @@ -55,11 +55,11 @@ type service struct { } // connectComputeService establishes a service connection to the Compute Engine. -func connectComputeService(cfg *config) (*service, error) { +func connectComputeService(ctx context.Context, cfg *config) (*service, error) { if cfg.clientConfig != nil && cfg.clientConfig.TokenSource != nil { - client := oauth2.NewClient(context.Background(), cfg.clientConfig.TokenSource) - svc, err := compute.NewService(context.Background(), option.WithHTTPClient(client)) + client := oauth2.NewClient(ctx, cfg.clientConfig.TokenSource) + svc, err := compute.NewService(ctx, option.WithHTTPClient(client)) if err != nil { return nil, fmt.Errorf("cannot connect to Google Cloud: %w", err) } @@ -139,18 +139,18 @@ func (svc *service) attachedDisks(cfg *config) ([]*compute.AttachedDisk, error) } // waitZoneOperation waits for a GCE operation in a zone to be completed or timed out. -func (svc *service) waitZoneOperation(cfg *config, opName string) error { - return svc.waitOperation(func() (*compute.Operation, error) { +func (svc *service) waitZoneOperation(ctx context.Context, cfg *config, opName string) error { + return svc.waitOperation(ctx, func() (*compute.Operation, error) { return svc.ZoneOperations.Get(cfg.projectID, cfg.zone, opName).Do() }) } // waitOperation waits for a GCE operation to be completed or timed out. -func (svc *service) waitOperation(refreshOperation func() (*compute.Operation, error)) error { +func (svc *service) waitOperation(ctx context.Context, refreshOperation func() (*compute.Operation, error)) error { var op *compute.Operation var err error - return wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) { + return wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, false, func(ctx context.Context) (bool, error) { op, err = refreshOperation() if err != nil { return false, err diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index b9face9b0..1f2a161c4 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -50,7 +50,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" - utilpointer "k8s.io/utils/pointer" + "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -839,7 +839,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. }, Spec: cdiv1beta1.DataVolumeSpec{ PVC: &corev1.PersistentVolumeClaimSpec{ - StorageClassName: utilpointer.String(config.StorageClassName), + StorageClassName: ptr.To(config.StorageClassName), AccessModes: []corev1.PersistentVolumeAccessMode{ "ReadWriteOnce", }, @@ -858,7 +858,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. }, Spec: cdiv1beta1.DataVolumeSpec{ PVC: &corev1.PersistentVolumeClaimSpec{ - StorageClassName: utilpointer.String(sd.StorageClassName), + StorageClassName: ptr.To(sd.StorageClassName), AccessModes: []corev1.PersistentVolumeAccessMode{ "ReadWriteOnce", }, diff --git a/pkg/cloudprovider/provider/nutanix/client.go b/pkg/cloudprovider/provider/nutanix/client.go index 9ec878238..4394c3b48 100644 --- a/pkg/cloudprovider/provider/nutanix/client.go +++ b/pkg/cloudprovider/provider/nutanix/client.go @@ -35,7 +35,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ) const ( @@ -106,7 +106,7 @@ func createVM(ctx context.Context, client *ClientSet, name string, conf Config, nicList := []*nutanixv3.VMNic{ { SubnetReference: &nutanixv3.Reference{ - Kind: pointer.String(nutanixtypes.SubnetKind), + Kind: ptr.To(nutanixtypes.SubnetKind), UUID: subnet.Metadata.UUID, }, }, @@ -119,7 +119,7 @@ func createVM(ctx context.Context, client *ClientSet, name string, conf Config, } additionalSubnetNic := &nutanixv3.VMNic{ SubnetReference: &nutanixv3.Reference{ - Kind: pointer.String(nutanixtypes.SubnetKind), + Kind: ptr.To(nutanixtypes.SubnetKind), UUID: additionalSubnet.Metadata.UUID, }, } @@ -133,41 +133,41 @@ func createVM(ctx context.Context, client *ClientSet, name string, conf Config, request := &nutanixv3.VMIntentInput{ Metadata: &nutanixv3.Metadata{ - Kind: pointer.String(nutanixtypes.VMKind), + Kind: ptr.To(nutanixtypes.VMKind), Categories: conf.Categories, }, Spec: &nutanixv3.VM{ - Name: pointer.String(name), + Name: ptr.To(name), ClusterReference: &nutanixv3.Reference{ - Kind: pointer.String(nutanixtypes.ClusterKind), + Kind: ptr.To(nutanixtypes.ClusterKind), UUID: cluster.Metadata.UUID, }, }, } resources := &nutanixv3.VMResources{ - PowerState: pointer.String("ON"), - NumSockets: pointer.Int64(conf.CPUs), - MemorySizeMib: pointer.Int64(conf.MemoryMB), + PowerState: ptr.To("ON"), + NumSockets: ptr.To(conf.CPUs), + MemorySizeMib: ptr.To(conf.MemoryMB), NicList: nicList, DiskList: []*nutanixv3.VMDisk{ { DeviceProperties: &nutanixv3.VMDiskDeviceProperties{ - DeviceType: pointer.String("DISK"), + DeviceType: ptr.To("DISK"), DiskAddress: &nutanixv3.DiskAddress{ - DeviceIndex: pointer.Int64(0), - AdapterType: pointer.String("SCSI"), + DeviceIndex: ptr.To(int64(0)), + AdapterType: ptr.To("SCSI"), }, }, DataSourceReference: &nutanixv3.Reference{ - Kind: pointer.String(nutanixtypes.ImageKind), + Kind: ptr.To(nutanixtypes.ImageKind), UUID: image.Metadata.UUID, }, }, }, GuestCustomization: &nutanixv3.GuestCustomization{ CloudInit: &nutanixv3.GuestCustomizationCloudInit{ - UserData: pointer.String(base64.StdEncoding.EncodeToString([]byte(userdata))), + UserData: ptr.To(base64.StdEncoding.EncodeToString([]byte(userdata))), }, }, } @@ -179,7 +179,7 @@ func createVM(ctx context.Context, client *ClientSet, name string, conf Config, } request.Metadata.ProjectReference = &nutanixv3.Reference{ - Kind: pointer.String(nutanixtypes.ProjectKind), + Kind: ptr.To(nutanixtypes.ProjectKind), UUID: project.Metadata.UUID, } } @@ -193,7 +193,7 @@ func createVM(ctx context.Context, client *ClientSet, name string, conf Config, } if conf.DiskSizeGB != nil { - resources.DiskList[0].DiskSizeMib = pointer.Int64(*conf.DiskSizeGB * 1024) + resources.DiskList[0].DiskSizeMib = ptr.To(*conf.DiskSizeGB * 1024) } request.Spec.Resources = resources @@ -367,7 +367,7 @@ func getVMByName(ctx context.Context, client *ClientSet, name string, projectID func getIPs(ctx context.Context, client *ClientSet, vmID string, interval time.Duration, timeout time.Duration) (map[string]corev1.NodeAddressType, error) { addresses := make(map[string]corev1.NodeAddressType) - if err := wait.Poll(interval, timeout, func() (bool, error) { + err := wait.PollUntilContextTimeout(ctx, interval, timeout, false, func(ctx context.Context) (bool, error) { vm, err := client.Prism.V3.GetVM(ctx, vmID) if err != nil { return false, wrapNutanixError(err) @@ -381,7 +381,8 @@ func getIPs(ctx context.Context, client *ClientSet, vmID string, interval time.D addresses[ip] = corev1.NodeInternalIP return true, nil - }); err != nil { + }) + if err != nil { return map[string]corev1.NodeAddressType{}, err } @@ -389,7 +390,7 @@ func getIPs(ctx context.Context, client *ClientSet, vmID string, interval time.D } func waitForCompletion(ctx context.Context, client *ClientSet, taskID string, interval time.Duration, timeout time.Duration) error { - return wait.Poll(interval, timeout, func() (bool, error) { + return wait.PollUntilContextTimeout(ctx, interval, timeout, false, func(ctx context.Context) (bool, error) { task, err := client.Prism.V3.GetTask(ctx, taskID) if err != nil { return false, wrapNutanixError(err) @@ -413,7 +414,7 @@ func waitForCompletion(ctx context.Context, client *ClientSet, taskID string, in } func waitForPowerState(ctx context.Context, client *ClientSet, vmID string, interval time.Duration, timeout time.Duration) error { - return wait.Poll(interval, timeout, func() (bool, error) { + return wait.PollUntilContextTimeout(ctx, interval, timeout, false, func(ctx context.Context) (bool, error) { vm, err := client.Prism.V3.GetVM(ctx, vmID) if err != nil { return false, wrapNutanixError(err) diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 77baf0eaa..699c778d9 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -36,7 +36,7 @@ import ( corev1 "k8s.io/api/core/v1" ktypes "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ) type Config struct { @@ -143,7 +143,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if err != nil { return nil, nil, nil, err } - c.Port = pointer.Int(portInt) + c.Port = ptr.To(portInt) } c.Username, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Username, "NUTANIX_USERNAME") diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index c4dcea347..8ff4e9dc7 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -64,7 +64,7 @@ const ( type clientGetterFunc func(c *Config) (*gophercloud.ProviderClient, error) // portReadinessWaiterFunc waits for the port with the given ID to be available. -type portReadinessWaiterFunc func(instanceLog *zap.SugaredLogger, netClient *gophercloud.ServiceClient, serverID string, networkID string, instanceReadyCheckPeriod time.Duration, instanceReadyCheckTimeout time.Duration) error +type portReadinessWaiterFunc func(ctx context.Context, instanceLog *zap.SugaredLogger, netClient *gophercloud.ServiceClient, serverID string, networkID string, instanceReadyCheckPeriod time.Duration, instanceReadyCheckTimeout time.Duration) error type provider struct { configVarResolver *providerconfig.ConfigVarResolver @@ -556,7 +556,7 @@ func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec cluste return nil } -func (p *provider) Create(_ context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { cfg, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -672,7 +672,7 @@ func (p *provider) Create(_ context.Context, log *zap.SugaredLogger, machine *cl if cfg.FloatingIPPool != "" { instanceLog := log.With("instance", server.ID) - if err := p.portReadinessWaiter(instanceLog, netClient, server.ID, network.ID, cfg.InstanceReadyCheckPeriod, cfg.InstanceReadyCheckTimeout); err != nil { + if err := p.portReadinessWaiter(ctx, instanceLog, netClient, server.ID, network.ID, cfg.InstanceReadyCheckPeriod, cfg.InstanceReadyCheckTimeout); err != nil { instanceLog.Infow("Port for instance did not became active", zap.Error(err)) } @@ -686,11 +686,11 @@ func (p *provider) Create(_ context.Context, log *zap.SugaredLogger, machine *cl return &osInstance{server: &server}, nil } -func waitForPort(instanceLog *zap.SugaredLogger, netClient *gophercloud.ServiceClient, serverID string, networkID string, checkPeriod time.Duration, checkTimeout time.Duration) error { +func waitForPort(ctx context.Context, instanceLog *zap.SugaredLogger, netClient *gophercloud.ServiceClient, serverID string, networkID string, checkPeriod time.Duration, checkTimeout time.Duration) error { started := time.Now() instanceLog.Info("Waiting for the port to become active...") - portIsReady := func() (bool, error) { + portIsReady := func(c context.Context) (bool, error) { port, err := getInstancePort(netClient, serverID, networkID) if err != nil { tErr := osErrorToTerminalError(instanceLog, err, fmt.Sprintf("failed to get current instance port %s", serverID)) @@ -705,8 +705,8 @@ func waitForPort(instanceLog *zap.SugaredLogger, netClient *gophercloud.ServiceC return port.Status == "ACTIVE", nil } - if err := wait.Poll(checkPeriod, checkTimeout, portIsReady); err != nil { - if errors.Is(err, wait.ErrWaitTimeout) { + if err := wait.PollUntilContextTimeout(ctx, checkPeriod, checkTimeout, false, portIsReady); err != nil { + if wait.Interrupted(err) { // In case we have a timeout, include the timeout details return fmt.Errorf("instance port became not active after %f seconds", checkTimeout.Seconds()) } diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index 4b7b056fd..ad650c433 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -38,7 +38,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/providerconfig" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -241,13 +241,13 @@ func TestCreateServer(t *testing.T) { }, { name: "Custom disk size", - specConf: openstackProviderSpecConf{RootDiskSizeGB: pointer.Int32(10)}, + specConf: openstackProviderSpecConf{RootDiskSizeGB: ptr.To(int32(10))}, userdata: "fake-userdata", wantServerReq: expectedBlockDeviceBootRequest, }, { name: "Custom disk type", - specConf: openstackProviderSpecConf{RootDiskSizeGB: pointer.Int32(10), RootDiskVolumeType: "ssd"}, + specConf: openstackProviderSpecConf{RootDiskSizeGB: ptr.To(int32(10)), RootDiskVolumeType: "ssd"}, userdata: "fake-userdata", wantServerReq: expectedBlockDeviceBootVolumeTypeRequest, }, @@ -282,7 +282,7 @@ func TestCreateServer(t *testing.T) { return pc.ProviderClient, nil }, // mock server readiness checker - portReadinessWaiter: func(*zap.SugaredLogger, *gophercloud.ServiceClient, string, string, time.Duration, time.Duration) error { + portReadinessWaiter: func(context.Context, *zap.SugaredLogger, *gophercloud.ServiceClient, string, string, time.Duration, time.Duration) error { return nil }, } diff --git a/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go b/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go index 86d505036..0fa109116 100644 --- a/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go +++ b/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go @@ -26,7 +26,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/ini" testhelper "github.com/kubermatic/machine-controller/pkg/test" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ) var update = flag.Bool("update", false, "update testdata files") @@ -78,7 +78,7 @@ func TestCloudConfigToString(t *testing.T) { }, LoadBalancer: LoadBalancerOpts{ ManageSecurityGroups: true, - UseOctavia: pointer.Bool(true), + UseOctavia: ptr.To(true), }, Version: "1.10.0", }, @@ -102,7 +102,7 @@ func TestCloudConfigToString(t *testing.T) { }, LoadBalancer: LoadBalancerOpts{ ManageSecurityGroups: true, - UseOctavia: pointer.Bool(false), + UseOctavia: ptr.To(false), }, Version: "1.10.0", }, diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go index 6c1980960..419e4e817 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go @@ -29,7 +29,7 @@ import ( clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ) var internalDiskBusTypes = map[string]string{ @@ -192,8 +192,8 @@ func recomposeComputeAndDisk(config *Config, vm *govcd.VM) (*govcd.VM, error) { vmSpecSection := vm.VM.VmSpecSection if config.SizingPolicy == nil || *config.SizingPolicy == "" { vmSpecSection.MemoryResourceMb.Configured = config.MemoryMB - vmSpecSection.NumCpus = pointer.Int(int(config.CPUs)) - vmSpecSection.NumCoresPerSocket = pointer.Int(int(config.CPUCores)) + vmSpecSection.NumCpus = ptr.To(int(config.CPUs)) + vmSpecSection.NumCoresPerSocket = ptr.To(int(config.CPUCores)) needsComputeRecomposition = true } @@ -207,7 +207,7 @@ func recomposeComputeAndDisk(config *Config, vm *govcd.VM) (*govcd.VM, error) { needsDiskRecomposition = true } if config.DiskIOPS != nil && *config.DiskIOPS > 0 { - vmSpecSection.DiskSection.DiskSettings[i].Iops = pointer.Int64(*config.DiskIOPS) + vmSpecSection.DiskSection.DiskSettings[i].Iops = ptr.To(*config.DiskIOPS) needsDiskRecomposition = true } if config.DiskBusType != nil && *config.DiskBusType != "" { diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go index 404c62185..c1136b79e 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -37,7 +37,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ) const ( @@ -147,10 +147,10 @@ func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.Machin // These defaults will have no effect if DiskSizeGB is not specified if rawConfig.DiskBusType == nil { - rawConfig.DiskBusType = pointer.String(defaultDiskType) + rawConfig.DiskBusType = ptr.To(defaultDiskType) } if rawConfig.DiskIOPS == nil { - rawConfig.DiskIOPS = pointer.Int64(defaultDiskIOPS) + rawConfig.DiskIOPS = ptr.To(int64(defaultDiskIOPS)) } spec.ProviderSpec.Value, err = setProviderSpec(*rawConfig, spec.ProviderSpec) return spec, err diff --git a/pkg/cloudprovider/provider/vsphere/provider_test.go b/pkg/cloudprovider/provider/vsphere/provider_test.go index f32c75275..9a30be155 100644 --- a/pkg/cloudprovider/provider/vsphere/provider_test.go +++ b/pkg/cloudprovider/provider/vsphere/provider_test.go @@ -30,7 +30,7 @@ import ( cloudprovidertesting "github.com/kubermatic/machine-controller/pkg/cloudprovider/testing" "github.com/kubermatic/machine-controller/pkg/providerconfig" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -97,7 +97,7 @@ func TestValidate(t *testing.T) { { name: "Valid Datastore", args: vsphereProviderSpecConf{ - Datastore: pointer.String("LocalDS_0"), + Datastore: ptr.To("LocalDS_0"), }, getConfigErr: nil, wantErr: false, @@ -105,8 +105,8 @@ func TestValidate(t *testing.T) { { name: "Valid Datastore end empty DatastoreCluster", args: vsphereProviderSpecConf{ - Datastore: pointer.String("LocalDS_0"), - DatastoreCluster: pointer.String(""), + Datastore: ptr.To("LocalDS_0"), + DatastoreCluster: ptr.To(""), }, getConfigErr: nil, wantErr: false, @@ -114,7 +114,7 @@ func TestValidate(t *testing.T) { { name: "Valid DatastoreCluster", args: vsphereProviderSpecConf{ - DatastoreCluster: pointer.String("DC0_POD0"), + DatastoreCluster: ptr.To("DC0_POD0"), }, getConfigErr: nil, wantErr: false, @@ -122,7 +122,7 @@ func TestValidate(t *testing.T) { { name: "Invalid Datastore", args: vsphereProviderSpecConf{ - Datastore: pointer.String("LocalDS_10"), + Datastore: ptr.To("LocalDS_10"), }, getConfigErr: nil, wantErr: true, @@ -130,7 +130,7 @@ func TestValidate(t *testing.T) { { name: "Invalid DatastoreCluster", args: vsphereProviderSpecConf{ - Datastore: pointer.String("DC0_POD10"), + Datastore: ptr.To("DC0_POD10"), }, getConfigErr: nil, wantErr: true, @@ -138,8 +138,8 @@ func TestValidate(t *testing.T) { { name: "Both Datastore and DatastoreCluster specified", args: vsphereProviderSpecConf{ - Datastore: pointer.String("DC0_POD10"), - DatastoreCluster: pointer.String("DC0_POD0"), + Datastore: ptr.To("DC0_POD10"), + DatastoreCluster: ptr.To("DC0_POD0"), }, getConfigErr: nil, wantErr: true, diff --git a/pkg/controller/machine/controller.go b/pkg/controller/machine/controller.go index 335d901f8..abc092a0b 100644 --- a/pkg/controller/machine/controller.go +++ b/pkg/controller/machine/controller.go @@ -68,7 +68,6 @@ import ( "k8s.io/client-go/tools/reference" "k8s.io/client-go/util/retry" ccmapi "k8s.io/cloud-provider/api" - "sigs.k8s.io/controller-runtime/pkg/client" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" @@ -232,50 +231,16 @@ func Add( if err != nil { return err } - if err := c.Watch(&source.Kind{Type: &clusterv1alpha1.Machine{}}, &handler.EnqueueRequestForObject{}); err != nil { + if err := c.Watch(source.Kind(mgr.GetCache(), &clusterv1alpha1.Machine{}), + &handler.EnqueueRequestForObject{}); err != nil { return err } metrics.Workers.Set(float64(numWorkers)) return c.Watch( - &source.Kind{Type: &corev1.Node{}}, - handler.EnqueueRequestsFromMapFunc(func(node client.Object) (result []reconcile.Request) { - machinesList := &clusterv1alpha1.MachineList{} - if err := mgr.GetClient().List(ctx, machinesList); err != nil { - utilruntime.HandleError(fmt.Errorf("failed to list machines in lister: %w", err)) - } - - var ownerUIDString string - var exists bool - if nodeLabels := node.GetLabels(); nodeLabels != nil { - ownerUIDString, exists = nodeLabels[NodeOwnerLabelName] - } - if !exists { - // We get triggered by node{Add,Update}, so enqeue machines if they - // have no nodeRef yet to make matching happen ASAP - for _, machine := range machinesList.Items { - if machine.Status.NodeRef == nil { - result = append(result, reconcile.Request{ - NamespacedName: types.NamespacedName{ - Namespace: machine.Namespace, - Name: machine.Name}}) - } - } - return result - } - - for _, machine := range machinesList.Items { - if string(machine.UID) == ownerUIDString { - log.Debugw("Processing node", "node", node.GetName(), "machine", ctrlruntimeclient.ObjectKeyFromObject(&machine)) - return []reconcile.Request{{NamespacedName: types.NamespacedName{ - Namespace: machine.Namespace, - Name: machine.Name, - }}} - } - } - return result - }), + source.Kind(mgr.GetCache(), &corev1.Node{}), + enqueueRequestsForNodes(ctx, log, mgr), predicate.Funcs{UpdateFunc: func(e event.UpdateEvent) bool { oldNode := e.ObjectOld.(*corev1.Node) newNode := e.ObjectNew.(*corev1.Node) @@ -301,6 +266,46 @@ func Add( ) } +func enqueueRequestsForNodes(ctx context.Context, log *zap.SugaredLogger, mgr manager.Manager) handler.EventHandler { + return handler.EnqueueRequestsFromMapFunc(func(_ context.Context, node ctrlruntimeclient.Object) []reconcile.Request { + var result []reconcile.Request + machinesList := &clusterv1alpha1.MachineList{} + if err := mgr.GetClient().List(ctx, machinesList); err != nil { + utilruntime.HandleError(fmt.Errorf("failed to list machines in lister: %w", err)) + } + + var ownerUIDString string + var exists bool + if nodeLabels := node.GetLabels(); nodeLabels != nil { + ownerUIDString, exists = nodeLabels[NodeOwnerLabelName] + } + if !exists { + // We get triggered by node{Add,Update}, so enqeue machines if they + // have no nodeRef yet to make matching happen ASAP + for _, machine := range machinesList.Items { + if machine.Status.NodeRef == nil { + result = append(result, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: machine.Namespace, + Name: machine.Name}}) + } + } + return result + } + + for _, machine := range machinesList.Items { + if string(machine.UID) == ownerUIDString { + log.Debugw("Processing node", "node", node.GetName(), "machine", ctrlruntimeclient.ObjectKeyFromObject(&machine)) + return []reconcile.Request{{NamespacedName: types.NamespacedName{ + Namespace: machine.Namespace, + Name: machine.Name, + }}} + } + } + return result + }) +} + // clearMachineError is a convenience function to remove a error on the machine if its set. // It does not return an error as it's used around the sync handler. func (r *Reconciler) clearMachineError(machine *clusterv1alpha1.Machine) { diff --git a/pkg/controller/machine/controller_test.go b/pkg/controller/machine/controller_test.go index 4ccd41762..0c98c04bc 100644 --- a/pkg/controller/machine/controller_test.go +++ b/pkg/controller/machine/controller_test.go @@ -335,6 +335,7 @@ func durationPtr(d time.Duration) *time.Duration { func TestControllerShouldEvict(t *testing.T) { threeHoursAgo := metav1.NewTime(time.Now().Add(-3 * time.Hour)) now := metav1.Now() + finalizer := "test" tests := []struct { name string @@ -354,6 +355,7 @@ func TestControllerShouldEvict(t *testing.T) { machine: &clusterv1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{ DeletionTimestamp: &threeHoursAgo, + Finalizers: []string{finalizer}, }, Status: clusterv1alpha1.MachineStatus{ NodeRef: &corev1.ObjectReference{Name: "existing-node"}, @@ -366,6 +368,7 @@ func TestControllerShouldEvict(t *testing.T) { machine: &clusterv1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{ DeletionTimestamp: &now, + Finalizers: []string{finalizer}, }, Status: clusterv1alpha1.MachineStatus{ NodeRef: nil, @@ -378,6 +381,7 @@ func TestControllerShouldEvict(t *testing.T) { machine: &clusterv1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{ DeletionTimestamp: &now, + Finalizers: []string{finalizer}, }, Status: clusterv1alpha1.MachineStatus{ NodeRef: &corev1.ObjectReference{Name: "non-existing-node"}, @@ -394,6 +398,7 @@ func TestControllerShouldEvict(t *testing.T) { machine: &clusterv1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{ DeletionTimestamp: &now, + Finalizers: []string{finalizer}, }, Status: clusterv1alpha1.MachineStatus{ NodeRef: &corev1.ObjectReference{Name: "existing-node"}, @@ -414,6 +419,7 @@ func TestControllerShouldEvict(t *testing.T) { machine: &clusterv1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{ DeletionTimestamp: &now, + Finalizers: []string{finalizer}, }, Status: clusterv1alpha1.MachineStatus{ NodeRef: &corev1.ObjectReference{Name: "existing-node"}, @@ -434,6 +440,7 @@ func TestControllerShouldEvict(t *testing.T) { machine: &clusterv1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{ DeletionTimestamp: &now, + Finalizers: []string{finalizer}, }, Status: clusterv1alpha1.MachineStatus{ NodeRef: &corev1.ObjectReference{Name: "existing-node"}, @@ -604,6 +611,7 @@ func TestControllerDeleteNodeForMachine(t *testing.T) { client := fakectrlruntimeclient.NewClientBuilder(). WithScheme(scheme.Scheme). + WithStatusSubresource(). WithObjects(objects...). Build() diff --git a/pkg/controller/machinedeployment/controller.go b/pkg/controller/machinedeployment/controller.go index c1f8bad32..745fa1e76 100644 --- a/pkg/controller/machinedeployment/controller.go +++ b/pkg/controller/machinedeployment/controller.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" + ctrlruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -71,7 +72,7 @@ func newReconciler(mgr manager.Manager, log *zap.SugaredLogger) *ReconcileMachin // Add creates a new MachineDeployment Controller and adds it to the Manager with default RBAC. func Add(mgr manager.Manager, log *zap.SugaredLogger) error { r := newReconciler(mgr, log) - return add(mgr, r, r.MachineSetToDeployments) + return add(mgr, r, r.MachineSetToDeployments()) } // add adds a new Controller to mgr with r as the reconcile.Reconciler. @@ -89,8 +90,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) err } // Watch for changes to MachineDeployment. - err = c.Watch(&source.Kind{ - Type: &v1alpha1.MachineDeployment{}}, + err = c.Watch(source.Kind(mgr.GetCache(), &v1alpha1.MachineDeployment{}), &handler.EnqueueRequestForObject{}, ) if err != nil { @@ -99,8 +99,8 @@ func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) err // Watch for changes to MachineSet and reconcile the owner MachineDeployment. err = c.Watch( - &source.Kind{Type: &v1alpha1.MachineSet{}}, - &handler.EnqueueRequestForOwner{OwnerType: &v1alpha1.MachineDeployment{}, IsController: true}, + source.Kind(mgr.GetCache(), &v1alpha1.MachineSet{}), + handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &v1alpha1.MachineDeployment{}, handler.OnlyControllerOwner()), ) if err != nil { return err @@ -110,7 +110,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) err // This watcher is required for use cases like adoption. In case a MachineSet doesn't have // a controller reference, it'll look for potential matching MachineDeployments to reconcile. err = c.Watch( - &source.Kind{Type: &v1alpha1.MachineSet{}}, + source.Kind(mgr.GetCache(), &v1alpha1.MachineSet{}), handler.EnqueueRequestsFromMapFunc(mapFn), ) if err != nil { @@ -302,39 +302,40 @@ func (r *ReconcileMachineDeployment) getMachineDeploymentsForMachineSet(ctx cont // MachineSetTodeployments is a handler.MapFunc to be used to enqeue requests for reconciliation // for MachineDeployments that might adopt an orphaned MachineSet. -func (r *ReconcileMachineDeployment) MachineSetToDeployments(o client.Object) []reconcile.Request { - result := []reconcile.Request{} - ctx := context.Background() - - ms := &v1alpha1.MachineSet{} - key := client.ObjectKey{Namespace: o.GetNamespace(), Name: o.GetName()} - if err := r.Client.Get(ctx, key, ms); err != nil { - if !apierrors.IsNotFound(err) { - r.log.Errorw("Failed to retrieve MachineSet for possible MachineDeployment adoption", "machineset", key, zap.Error(err)) +func (r *ReconcileMachineDeployment) MachineSetToDeployments() handler.MapFunc { + return func(ctx context.Context, o client.Object) []ctrlruntime.Request { + result := []reconcile.Request{} + + ms := &v1alpha1.MachineSet{} + key := client.ObjectKey{Namespace: o.GetNamespace(), Name: o.GetName()} + if err := r.Client.Get(ctx, key, ms); err != nil { + if !apierrors.IsNotFound(err) { + r.log.Errorw("Failed to retrieve MachineSet for possible MachineDeployment adoption", "machineset", key, zap.Error(err)) + } + return nil } - return nil - } - // Check if the controller reference is already set and - // return an empty result when one is found. - for _, ref := range ms.ObjectMeta.OwnerReferences { - if ref.Controller != nil && *ref.Controller { - return result + // Check if the controller reference is already set and + // return an empty result when one is found. + for _, ref := range ms.ObjectMeta.OwnerReferences { + if ref.Controller != nil && *ref.Controller { + return result + } } - } - mds := r.getMachineDeploymentsForMachineSet(ctx, r.log.With("machineset", key), ms) - if len(mds) == 0 { - r.log.Debugw("Found no MachineDeployments for MachineSet", "machineset", key) - return nil - } + mds := r.getMachineDeploymentsForMachineSet(ctx, r.log.With("machineset", key), ms) + if len(mds) == 0 { + r.log.Debugw("Found no MachineDeployments for MachineSet", "machineset", key) + return nil + } - for _, md := range mds { - name := client.ObjectKey{Namespace: md.Namespace, Name: md.Name} - result = append(result, reconcile.Request{NamespacedName: name}) - } + for _, md := range mds { + name := client.ObjectKey{Namespace: md.Namespace, Name: md.Name} + result = append(result, reconcile.Request{NamespacedName: name}) + } - return result + return result + } } func contains(list []string, strToSearch string) bool { diff --git a/pkg/controller/machineset/controller.go b/pkg/controller/machineset/controller.go index 995db4209..6301c3819 100644 --- a/pkg/controller/machineset/controller.go +++ b/pkg/controller/machineset/controller.go @@ -37,6 +37,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/record" + ctrlruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" @@ -64,7 +65,7 @@ var ( // The Manager will set fields on the Controller and Start it when the Manager is Started. func Add(mgr manager.Manager, log *zap.SugaredLogger) error { r := newReconciler(mgr, log) - return add(mgr, r, r.MachineToMachineSets) + return add(mgr, r, r.MachineToMachineSets()) } // newReconciler returns a new reconcile.Reconciler. @@ -93,7 +94,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) err // Watch for changes to MachineSet. err = c.Watch( - &source.Kind{Type: &clusterv1alpha1.MachineSet{}}, + source.Kind(mgr.GetCache(), &clusterv1alpha1.MachineSet{}), &handler.EnqueueRequestForObject{}, ) if err != nil { @@ -102,8 +103,8 @@ func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) err // Watch for changes to Machines and reconcile the owner MachineSet. err = c.Watch( - &source.Kind{Type: &clusterv1alpha1.Machine{}}, - &handler.EnqueueRequestForOwner{IsController: true, OwnerType: &clusterv1alpha1.MachineSet{}}, + source.Kind(mgr.GetCache(), &clusterv1alpha1.Machine{}), + handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &clusterv1alpha1.MachineSet{}, handler.OnlyControllerOwner()), ) if err != nil { return err @@ -113,7 +114,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) err // This watcher is required for use cases like adoption. In case a Machine doesn't have // a controller reference, it'll look for potential matching MachineSet to reconcile. return c.Watch( - &source.Kind{Type: &clusterv1alpha1.Machine{}}, + source.Kind(mgr.GetCache(), &clusterv1alpha1.Machine{}), handler.EnqueueRequestsFromMapFunc(mapFn), ) } @@ -378,7 +379,7 @@ func (r *ReconcileMachineSet) adoptOrphan(ctx context.Context, machineSet *clust func (r *ReconcileMachineSet) waitForMachineCreation(ctx context.Context, log *zap.SugaredLogger, machineList []*clusterv1alpha1.Machine) error { for _, machine := range machineList { - pollErr := wait.PollImmediate(stateConfirmationInterval, stateConfirmationTimeout, func() (bool, error) { + pollErr := wait.PollUntilContextTimeout(ctx, stateConfirmationInterval, stateConfirmationTimeout, false, func(ctx context.Context) (bool, error) { key := client.ObjectKey{Namespace: machine.Namespace, Name: machine.Name} if err := r.Client.Get(ctx, key, &clusterv1alpha1.Machine{}); err != nil { @@ -402,7 +403,7 @@ func (r *ReconcileMachineSet) waitForMachineCreation(ctx context.Context, log *z func (r *ReconcileMachineSet) waitForMachineDeletion(ctx context.Context, machineList []*clusterv1alpha1.Machine) error { for _, machine := range machineList { - pollErr := wait.PollImmediate(stateConfirmationInterval, stateConfirmationTimeout, func() (bool, error) { + pollErr := wait.PollUntilContextTimeout(ctx, stateConfirmationInterval, stateConfirmationTimeout, false, func(ctx context.Context) (bool, error) { m := &clusterv1alpha1.Machine{} key := client.ObjectKey{Namespace: machine.Namespace, Name: machine.Name} @@ -423,41 +424,42 @@ func (r *ReconcileMachineSet) waitForMachineDeletion(ctx context.Context, machin // MachineToMachineSets is a handler.ToRequestsFunc to be used to enqeue requests for reconciliation // for MachineSets that might adopt an orphaned Machine. -func (r *ReconcileMachineSet) MachineToMachineSets(o client.Object) []reconcile.Request { - result := []reconcile.Request{} - ctx := context.Background() +func (r *ReconcileMachineSet) MachineToMachineSets() handler.MapFunc { + return func(ctx context.Context, o client.Object) []ctrlruntime.Request { + result := []reconcile.Request{} - m := &clusterv1alpha1.Machine{} - key := client.ObjectKey{Namespace: o.GetNamespace(), Name: o.GetName()} - machineLog := r.log.With("machine", key) + m := &clusterv1alpha1.Machine{} + key := client.ObjectKey{Namespace: o.GetNamespace(), Name: o.GetName()} + machineLog := r.log.With("machine", key) - if err := r.Client.Get(ctx, key, m); err != nil { - if !apierrors.IsNotFound(err) { - machineLog.Errorw("Failed to retrieve Machine for possible MachineSet adoption", zap.Error(err)) + if err := r.Client.Get(ctx, key, m); err != nil { + if !apierrors.IsNotFound(err) { + machineLog.Errorw("Failed to retrieve Machine for possible MachineSet adoption", zap.Error(err)) + } + return nil } - return nil - } - // Check if the controller reference is already set and - // return an empty result when one is found. - for _, ref := range m.ObjectMeta.OwnerReferences { - if ref.Controller != nil && *ref.Controller { - return result + // Check if the controller reference is already set and + // return an empty result when one is found. + for _, ref := range m.ObjectMeta.OwnerReferences { + if ref.Controller != nil && *ref.Controller { + return result + } } - } - mss := r.getMachineSetsForMachine(ctx, machineLog, m) - if len(mss) == 0 { - machineLog.Debug("Found no MachineSet for Machine") - return nil - } + mss := r.getMachineSetsForMachine(ctx, machineLog, m) + if len(mss) == 0 { + machineLog.Debug("Found no MachineSet for Machine") + return nil + } - for _, ms := range mss { - name := client.ObjectKey{Namespace: ms.Namespace, Name: ms.Name} - result = append(result, reconcile.Request{NamespacedName: name}) - } + for _, ms := range mss { + name := client.ObjectKey{Namespace: ms.Namespace, Name: ms.Name} + result = append(result, reconcile.Request{NamespacedName: name}) + } - return result + return result + } } func contains(list []string, strToSearch string) bool { diff --git a/pkg/controller/nodecsrapprover/controller.go b/pkg/controller/nodecsrapprover/controller.go index 92f3ad688..e5ceda524 100644 --- a/pkg/controller/nodecsrapprover/controller.go +++ b/pkg/controller/nodecsrapprover/controller.go @@ -94,7 +94,7 @@ func Add(mgr manager.Manager, log *zap.SugaredLogger) error { return fmt.Errorf("failed to construct controller: %w", err) } - return cntrl.Watch(&source.Kind{Type: watchType}, &handler.EnqueueRequestForObject{}) + return cntrl.Watch(source.Kind(mgr.GetCache(), watchType), &handler.EnqueueRequestForObject{}) } func (r *reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { diff --git a/pkg/node/nodemanager/node_manager.go b/pkg/node/nodemanager/node_manager.go index 2bea84d6c..c4942a12e 100644 --- a/pkg/node/nodemanager/node_manager.go +++ b/pkg/node/nodemanager/node_manager.go @@ -64,7 +64,7 @@ func (nm *NodeManager) CordonNode(ctx context.Context, node *corev1.Node) error // that is not the case, there is a small chance the scheduler schedules // pods in between, those will then get deleted upon node deletion and // not evicted - return wait.Poll(1*time.Second, 10*time.Second, func() (bool, error) { + return wait.PollUntilContextTimeout(ctx, 1*time.Second, 10*time.Second, false, func(ctx context.Context) (bool, error) { node := &corev1.Node{} if err := nm.client.Get(ctx, types.NamespacedName{Name: nm.nodeName}, node); err != nil { return false, err diff --git a/pkg/providerconfig/types/types_test.go b/pkg/providerconfig/types/types_test.go index d1d7252bd..247b4e48a 100644 --- a/pkg/providerconfig/types/types_test.go +++ b/pkg/providerconfig/types/types_test.go @@ -22,7 +22,7 @@ import ( "testing" v1 "k8s.io/api/core/v1" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ) func TestConfigVarStringUnmarshalling(t *testing.T) { @@ -52,11 +52,11 @@ func TestConfigVarBoolUnmarshalling(t *testing.T) { }{ { jsonString: "true", - expected: ConfigVarBool{Value: pointer.Bool(true)}, + expected: ConfigVarBool{Value: ptr.To(true)}, }, { jsonString: `{"value":true}`, - expected: ConfigVarBool{Value: pointer.Bool(true)}, + expected: ConfigVarBool{Value: ptr.To(true)}, }, { jsonString: "null", @@ -76,11 +76,11 @@ func TestConfigVarBoolUnmarshalling(t *testing.T) { }, { jsonString: `{"value":false, "secretKeyRef":{"namespace":"ns","name":"name","key":"key"}}`, - expected: ConfigVarBool{Value: pointer.Bool(false), SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, + expected: ConfigVarBool{Value: ptr.To(false), SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, }, { jsonString: `{"value":true, "secretKeyRef":{"namespace":"ns","name":"name","key":"key"}}`, - expected: ConfigVarBool{Value: pointer.Bool(true), SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, + expected: ConfigVarBool{Value: ptr.To(true), SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, }, } @@ -129,7 +129,7 @@ func TestConfigVarBoolMarshalling(t *testing.T) { expected: `null`, }, { - cvb: ConfigVarBool{Value: pointer.Bool(true)}, + cvb: ConfigVarBool{Value: ptr.To(true)}, expected: `true`, }, { @@ -137,11 +137,11 @@ func TestConfigVarBoolMarshalling(t *testing.T) { expected: `{"secretKeyRef":{"namespace":"ns","name":"name","key":"key"}}`, }, { - cvb: ConfigVarBool{SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, Value: pointer.Bool(true)}, + cvb: ConfigVarBool{SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, Value: ptr.To(true)}, expected: `{"secretKeyRef":{"namespace":"ns","name":"name","key":"key"},"value":true}`, }, { - cvb: ConfigVarBool{SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, Value: pointer.Bool(false)}, + cvb: ConfigVarBool{SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, Value: ptr.To(false)}, expected: `{"secretKeyRef":{"namespace":"ns","name":"name","key":"key"},"value":false}`, }, } @@ -200,18 +200,18 @@ func TestConfigVarStringMarshallingAndUnmarshalling(t *testing.T) { func TestConfigVarBoolMarshallingAndUnmarshalling(t *testing.T) { testCases := []ConfigVarBool{ {}, - {Value: pointer.Bool(false)}, - {Value: pointer.Bool(true)}, + {Value: ptr.To(false)}, + {Value: ptr.To(true)}, {SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, - {Value: pointer.Bool(true), SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, + {Value: ptr.To(true), SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, {ConfigMapKeyRef: GlobalConfigMapKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, - {Value: pointer.Bool(true), ConfigMapKeyRef: GlobalConfigMapKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, + {Value: ptr.To(true), ConfigMapKeyRef: GlobalConfigMapKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, { ConfigMapKeyRef: GlobalConfigMapKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, }, { - Value: pointer.Bool(true), + Value: ptr.To(true), ConfigMapKeyRef: GlobalConfigMapKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, }, diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml index 119a07ae3..08d7651da 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml @@ -309,6 +309,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml index 8fb671824..e5110eda1 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml @@ -309,6 +309,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml index b29b20207..717e1177f 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml @@ -309,6 +309,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml index 6e525dc4e..2e982a519 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml @@ -309,6 +309,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index 8b5b3ba5f..ab541bf78 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -326,6 +326,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index 594cde2c8..a3eec0840 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -326,6 +326,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml index a0c323240..1e0a19220 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml @@ -317,6 +317,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml index d4c24db96..79908237b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml @@ -309,6 +309,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml index f95d97cd0..5948e4c23 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml @@ -315,6 +315,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml index f4cdb9b1d..bc0f71498 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml @@ -315,6 +315,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml index 8bc556202..7487319ef 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml @@ -315,6 +315,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml index 9cc2721da..fc0613d07 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml @@ -323,6 +323,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index 8ad4918c1..ced36dba2 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -332,6 +332,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index ca9a5ea4b..38cc9a7a5 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -332,6 +332,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml index d796580f4..97591d4bf 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml @@ -323,6 +323,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml index 4fa5f0fbc..9d3f95c33 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml @@ -315,6 +315,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml index c746bdfad..363a17d88 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml @@ -183,6 +183,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml index f2316d7b1..09ef75cc4 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml @@ -183,6 +183,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index c481c8d18..9bf3a029f 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -162,6 +162,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json index 0104db49f..2a9886a99 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.25.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.25.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.26.6.json b/pkg/userdata/flatcar/testdata/ignition_v1.26.6.json index 3d6d307d2..c548068fd 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.26.6.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.26.6.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.26.6%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.26.6%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go index 67d5405bd..26d4d49a5 100644 --- a/pkg/userdata/helper/kubelet.go +++ b/pkg/userdata/helper/kubelet.go @@ -31,7 +31,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletv1b1 "k8s.io/kubelet/config/v1beta1" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" kyaml "sigs.k8s.io/yaml" ) @@ -216,10 +216,10 @@ func kubeletConfiguration(log *zap.SugaredLogger, clusterDomain string, clusterD ClientCAFile: "/etc/kubernetes/pki/ca.crt", }, Webhook: kubeletv1b1.KubeletWebhookAuthentication{ - Enabled: pointer.Bool(true), + Enabled: ptr.To(true), }, Anonymous: kubeletv1b1.KubeletAnonymousAuthentication{ - Enabled: pointer.Bool(false), + Enabled: ptr.To(false), }, }, Authorization: kubeletv1b1.KubeletAuthorization{ @@ -291,12 +291,12 @@ func kubeletConfiguration(log *zap.SugaredLogger, clusterDomain string, clusterD // Instead of breaking the workflow, just print a warning and skip the configuration log.Infow("Skipping invalid ContainerLogMaxSize value for Kubelet configuration", "value", containerLogMaxFiles) } else { - cfg.ContainerLogMaxFiles = pointer.Int32(int32(maxFiles)) + cfg.ContainerLogMaxFiles = ptr.To(int32(maxFiles)) } } if enabled, ok := featureGates["SeccompDefault"]; ok && enabled { - cfg.SeccompDefault = pointer.Bool(true) + cfg.SeccompDefault = ptr.To(true) } buf, err := kyaml.Marshal(cfg) diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml index d9faed107..46ab37dd3 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml @@ -322,6 +322,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml index 8b961fb3b..c8341ea37 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml @@ -331,6 +331,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml index 74ca528fa..72b62b869 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml @@ -322,6 +322,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml index 9798827e0..9dc8b8f8b 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml @@ -322,6 +322,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml index 4b935d7bb..fb380ffbc 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml @@ -322,6 +322,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml index 4a396f4a2..f5feeec76 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml @@ -322,6 +322,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index 4f55669a3..462b3ad8b 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -340,6 +340,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index c94363c0b..13fb138d9 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -340,6 +340,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml index 688f7dd22..3eb848072 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml @@ -331,6 +331,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index 41562a0b3..5b7e2b2ea 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -328,6 +328,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml index 4827b5600..453aca786 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml @@ -322,6 +322,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml index dd50f43ee..d83e716ad 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml @@ -322,6 +322,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml index 451e55065..bdf75ea35 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml @@ -322,6 +322,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml index 6997f46cd..ceb90c26a 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml @@ -330,6 +330,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index f6af69a8b..65183b731 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -339,6 +339,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index 49829999d..1eb81af3b 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -339,6 +339,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml index 584c43377..e57136f96 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml @@ -330,6 +330,7 @@ write_files: cgroupDriver: systemd clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index bedf7108a..d7d3f5801 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -398,6 +398,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml index dd81f757d..ceff63465 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml @@ -399,6 +399,7 @@ write_files: - 10.10.10.12 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml index 52f389862..3d1cd0d67 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml @@ -399,6 +399,7 @@ write_files: - 10.10.10.12 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 1c361c0ea..638fa246f 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -388,6 +388,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml index a8e936d7e..2dc056290 100644 --- a/pkg/userdata/ubuntu/testdata/docker.yaml +++ b/pkg/userdata/ubuntu/testdata/docker.yaml @@ -393,6 +393,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index c795a5163..d639d4529 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -386,6 +386,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index e434071a0..44cc1d071 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -388,6 +388,7 @@ write_files: - 10.10.10.12 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index d5f636a34..aa3ffbd14 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -388,6 +388,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 14b64823a..1c17c1acf 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -397,6 +397,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml index 2e0b658e1..6f0f51723 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml @@ -399,6 +399,7 @@ write_files: - 10.10.10.12 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml index 6b1ae8c98..c023ec7d7 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml @@ -399,6 +399,7 @@ write_files: - 10.10.10.12 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index af796473b..e7097b703 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -394,6 +394,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 3fbef6c8f..bbc69ff3e 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -394,6 +394,7 @@ write_files: - 10.10.10.12 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/version-1.25.12.yaml b/pkg/userdata/ubuntu/testdata/version-1.25.12.yaml index f9527dd19..e1dcae9b1 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.25.12.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.25.12.yaml @@ -386,6 +386,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/version-1.26.7.yaml b/pkg/userdata/ubuntu/testdata/version-1.26.7.yaml index e604cff52..13ac2dac9 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.26.7.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.26.7.yaml @@ -386,6 +386,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/version-1.27.4.yaml b/pkg/userdata/ubuntu/testdata/version-1.27.4.yaml index 2d1ebffa1..e16c74d35 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.27.4.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.27.4.yaml @@ -386,6 +386,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/version-1.28.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.28.0.yaml index b9a4b3daa..d8f8906d1 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.28.0.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.28.0.yaml @@ -386,6 +386,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index 7376c55bc..766f4331a 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -405,6 +405,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index dc3272f9b..6aaee968c 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -412,6 +412,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 0504bcdf6..82f97564c 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -395,6 +395,7 @@ write_files: - 10.10.10.10 clusterDomain: cluster.local containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" cpuManagerReconcilePeriod: 0s evictionHard: imagefs.available: 15% diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index fe892dd90..488732f98 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -233,7 +233,7 @@ C8QmzsMaZhk+mVFr1sGy // wait for deployments to roll out for _, deployment := range deployments { - if err := wait.Poll(3*time.Second, 30*time.Second, func() (done bool, err error) { + if err := wait.PollUntilContextTimeout(ctx, 3*time.Second, 30*time.Second, false, func(ctx context.Context) (bool, error) { d := &appsv1.Deployment{} key := types.NamespacedName{Namespace: ns, Name: deployment} diff --git a/test/e2e/provisioning/deploymentscenario.go b/test/e2e/provisioning/deploymentscenario.go index 825f81f15..3ec6e182b 100644 --- a/test/e2e/provisioning/deploymentscenario.go +++ b/test/e2e/provisioning/deploymentscenario.go @@ -34,10 +34,12 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s if err != nil { return err } + ctx := context.Background() + // This test inherently relies on replicas being one so we enforce that machineDeployment.Spec.Replicas = getInt32Ptr(1) - machineDeployment, err = createAndAssure(machineDeployment, client, timeout) + machineDeployment, err = createAndAssure(ctx, machineDeployment, client, timeout) if err != nil { return fmt.Errorf("failed to verify creation of node for MachineDeployment: %w", err) } @@ -50,7 +52,7 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s klog.Infof("Waiting for second MachineSet to appear after updating MachineDeployment %s", machineDeployment.Name) var machineSets []clusterv1alpha1.MachineSet - if err := wait.Poll(5*time.Second, timeout, func() (bool, error) { + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, false, func(ctx context.Context) (bool, error) { machineSets, err = getMatchingMachineSets(machineDeployment, client) if err != nil { return false, err @@ -79,7 +81,7 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s oldMachineSet = machineSets[1] } var machines []clusterv1alpha1.Machine - if err := wait.Poll(5*time.Second, timeout, func() (bool, error) { + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, false, func(ctx context.Context) (bool, error) { machines, err = getMatchingMachinesForMachineset(&newestMachineSet, client) if err != nil { return false, err @@ -94,8 +96,8 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s klog.Infof("New MachineSet %s appeared with %v machines", newestMachineSet.Name, len(machines)) klog.Infof("Waiting for new MachineSet %s to get a ready node", newestMachineSet.Name) - if err := wait.Poll(5*time.Second, timeout, func() (bool, error) { - return hasMachineReadyNode(&machines[0], client) + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, false, func(ctx context.Context) (bool, error) { + return hasMachineReadyNode(ctx, &machines[0], client) }); err != nil { return err } @@ -103,9 +105,9 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s klog.Infof("Waiting for old MachineSet %s to be scaled down and have no associated machines", oldMachineSet.Name) - if err := wait.Poll(5*time.Second, timeout, func() (bool, error) { + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, false, func(ctx context.Context) (bool, error) { machineSet := &clusterv1alpha1.MachineSet{} - if err := client.Get(context.Background(), types.NamespacedName{Namespace: oldMachineSet.Namespace, Name: oldMachineSet.Name}, machineSet); err != nil { + if err := client.Get(ctx, types.NamespacedName{Namespace: oldMachineSet.Namespace, Name: oldMachineSet.Name}, machineSet); err != nil { return false, err } if *machineSet.Spec.Replicas != int32(0) { @@ -130,7 +132,7 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s klog.Infof("Successfully set replicas of MachineDeployment %s to 0", machineDeployment.Name) klog.Infof("Waiting for MachineDeployment %s to not have any associated machines", machineDeployment.Name) - if err := wait.Poll(5*time.Second, timeout, func() (bool, error) { + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, false, func(ctx context.Context) (bool, error) { machines, err := getMatchingMachines(machineDeployment, client) return len(machines) == 0, err }); err != nil { @@ -139,11 +141,11 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s klog.Infof("Successfully waited for MachineDeployment %s to not have any associated machines", machineDeployment.Name) klog.Infof("Deleting MachineDeployment %s and waiting for it to disappear", machineDeployment.Name) - if err := client.Delete(context.Background(), machineDeployment); err != nil { + if err := client.Delete(ctx, machineDeployment); err != nil { return fmt.Errorf("failed to delete MachineDeployment %s: %w", machineDeployment.Name, err) } - if err := wait.Poll(5*time.Second, timeout, func() (bool, error) { - err := client.Get(context.Background(), types.NamespacedName{Namespace: machineDeployment.Namespace, Name: machineDeployment.Name}, &clusterv1alpha1.MachineDeployment{}) + if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, false, func(ctx context.Context) (bool, error) { + err = client.Get(ctx, types.NamespacedName{Namespace: machineDeployment.Namespace, Name: machineDeployment.Name}, &clusterv1alpha1.MachineDeployment{}) if kerrors.IsNotFound(err) { return true, nil } diff --git a/test/e2e/provisioning/verify.go b/test/e2e/provisioning/verify.go index 5980a453d..5fdd7813c 100644 --- a/test/e2e/provisioning/verify.go +++ b/test/e2e/provisioning/verify.go @@ -61,7 +61,9 @@ func verifyCreateAndDelete(kubeConfig, manifestPath string, parameters []string, return err } - machineDeployment, err = createAndAssure(machineDeployment, client, timeout) + ctx := context.Background() + + machineDeployment, err = createAndAssure(ctx, machineDeployment, client, timeout) if err != nil { return fmt.Errorf("failed to verify creation of node for MachineDeployment: %w", err) } @@ -138,7 +140,7 @@ func prepare(kubeConfig, manifestPath string, parameters []string) (ctrlruntimec return client, manifest, nil } -func createAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Client, timeout time.Duration) (*clusterv1alpha1.MachineDeployment, error) { +func createAndAssure(ctx context.Context, machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Client, timeout time.Duration) (*clusterv1alpha1.MachineDeployment, error) { // we expect that no node for machine exists in the cluster err := assureNodeForMachineDeployment(machineDeployment, client, false) if err != nil { @@ -151,8 +153,8 @@ func createAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien // needs longer to validate a MachineDeployment than the kube-apiserver is willing to wait. // In real world scenarios this is not that critical, but for tests we need to pay closer // attention and retry the creation a few times. - err = wait.PollImmediate(3*time.Second, 180*time.Second, func() (bool, error) { - err := client.Create(context.Background(), machineDeployment) + err = wait.PollUntilContextTimeout(ctx, 3*time.Second, 180*time.Second, false, func(ctx context.Context) (bool, error) { + err := client.Create(ctx, machineDeployment) if err != nil { klog.Warningf("Creation of %q failed, retrying: %v", machineDeployment.Name, err) return false, nil @@ -167,7 +169,7 @@ func createAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien klog.Infof("MachineDeployment %q created", machineDeployment.Name) var pollErr error - err = wait.Poll(machineReadyCheckPeriod, timeout, func() (bool, error) { + err = wait.PollUntilContextTimeout(ctx, machineReadyCheckPeriod, timeout, false, func(ctx context.Context) (bool, error) { pollErr = assureNodeForMachineDeployment(machineDeployment, client, true) if pollErr == nil { return true, nil @@ -180,13 +182,13 @@ func createAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien klog.Infof("Found a node for MachineDeployment %s", machineDeployment.Name) klog.Infof("Waiting for node of MachineDeployment %s to become ready", machineDeployment.Name) - err = wait.Poll(machineReadyCheckPeriod, timeout, func() (bool, error) { + err = wait.PollUntilContextTimeout(ctx, machineReadyCheckPeriod, timeout, false, func(ctx context.Context) (bool, error) { machines, pollErr := getMatchingMachines(machineDeployment, client) if pollErr != nil || len(machines) < 1 { return false, nil } for _, machine := range machines { - hasReadyNode, pollErr := hasMachineReadyNode(&machine, client) + hasReadyNode, pollErr := hasMachineReadyNode(ctx, &machine, client) if err != nil { return false, pollErr } @@ -202,9 +204,9 @@ func createAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien return machineDeployment, nil } -func hasMachineReadyNode(machine *clusterv1alpha1.Machine, client ctrlruntimeclient.Client) (bool, error) { +func hasMachineReadyNode(ctx context.Context, machine *clusterv1alpha1.Machine, client ctrlruntimeclient.Client) (bool, error) { nodes := &corev1.NodeList{} - if err := client.List(context.Background(), nodes); err != nil { + if err := client.List(ctx, nodes); err != nil { return false, fmt.Errorf("failed to list nodes: %w", err) } for _, node := range nodes.Items { @@ -241,7 +243,7 @@ func deleteAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien } // Ensure machines are gone - if err := wait.Poll(machineReadyCheckPeriod, timeout, func() (bool, error) { + if err := wait.PollUntilContextTimeout(context.Background(), machineReadyCheckPeriod, timeout, false, func(ctx context.Context) (bool, error) { ownedMachines, err := getMatchingMachines(machineDeployment, client) if err != nil { return false, err @@ -258,7 +260,7 @@ func deleteAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien if err := client.Delete(context.Background(), machineDeployment); err != nil { return fmt.Errorf("failed to remove MachineDeployment %s, due to %w", machineDeployment.Name, err) } - return wait.Poll(machineReadyCheckPeriod, timeout, func() (bool, error) { + return wait.PollUntilContextTimeout(context.Background(), machineReadyCheckPeriod, timeout, false, func(ctx context.Context) (bool, error) { err := client.Get(context.Background(), types.NamespacedName{Namespace: machineDeployment.Namespace, Name: machineDeployment.Name}, &clusterv1alpha1.MachineDeployment{}) if kerrors.IsNotFound(err) { return true, nil From ca277282851f811794a05f9b5e4d67263c2a0fb8 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 22 Aug 2023 13:10:13 +0500 Subject: [PATCH 347/489] Install socat explicitly in KIND cluster (#1701) Signed-off-by: Waleed Malik --- hack/ci/setup-kind-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/ci/setup-kind-cluster.sh b/hack/ci/setup-kind-cluster.sh index b30c78820..303004853 100755 --- a/hack/ci/setup-kind-cluster.sh +++ b/hack/ci/setup-kind-cluster.sh @@ -131,7 +131,7 @@ EOF # unwrap the socket inside the kind cluster and make it available on a TCP port, # because containerd/Docker doesn't support sockets for mirrors. - docker exec $KIND_CLUSTER_NAME-control-plane bash -c 'socat TCP4-LISTEN:5001,fork,reuseaddr UNIX:/mirror/mirror.sock &' + docker exec $KIND_CLUSTER_NAME-control-plane bash -c 'apt update --quiet; apt install --quiet socat; socat TCP4-LISTEN:5001,fork,reuseaddr UNIX:/mirror/mirror.sock &' else kind create cluster --config kind-config.yaml fi From d69e4e9d19e34b41b22d9ac9ab25c12a0f8f786a Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 6 Sep 2023 00:35:39 +0500 Subject: [PATCH 348/489] Update project owners (#1703) Signed-off-by: Waleed Malik --- OWNERS_ALIASES | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index d5d600835..21494f318 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -5,14 +5,11 @@ aliases: machine-controller-maintainers: - ahmedwaleedmalik - embik - - hdurand0710 - kron4eg - - mfranczy - moadqassem - moelsayed - - sankalp-r - - themue - xmudrii - xrstf sig-virtualization: - - mfranczy + - cnvergence + - wozniakjan From 3a0177e2cf692d2096c511bfee1e405c0553250b Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 15 Sep 2023 23:20:16 +0500 Subject: [PATCH 349/489] Update Kubernetes patch versions for E2E tests (#1705) Signed-off-by: Waleed Malik --- pkg/userdata/helper/common_test.go | 8 ++++---- ..._v1.25.12.golden => download_binaries_v1.25.14.golden} | 2 +- ...es_v1.26.7.golden => download_binaries_v1.26.9.golden} | 2 +- ...es_v1.27.4.golden => download_binaries_v1.27.6.golden} | 2 +- ...es_v1.28.0.golden => download_binaries_v1.28.2.golden} | 2 +- ... kublet_systemd_unit_version-v1.25.14-external.golden} | 0 ...golden => kublet_systemd_unit_version-v1.25.14.golden} | 0 ...> kublet_systemd_unit_version-v1.26.9-external.golden} | 0 ....golden => kublet_systemd_unit_version-v1.26.9.golden} | 0 ...> kublet_systemd_unit_version-v1.27.6-external.golden} | 0 ....golden => kublet_systemd_unit_version-v1.27.6.golden} | 0 ...> kublet_systemd_unit_version-v1.28.2-external.golden} | 0 ....golden => kublet_systemd_unit_version-v1.28.2.golden} | 0 pkg/userdata/ubuntu/provider_test.go | 8 ++++---- .../{version-1.25.12.yaml => version-1.25.14.yaml} | 2 +- .../testdata/{version-1.26.7.yaml => version-1.26.9.yaml} | 2 +- .../testdata/{version-1.27.4.yaml => version-1.27.6.yaml} | 2 +- .../testdata/{version-1.28.0.yaml => version-1.28.2.yaml} | 2 +- test/e2e/provisioning/all_e2e_test.go | 8 ++++---- test/e2e/provisioning/helper.go | 8 ++++---- 20 files changed, 24 insertions(+), 24 deletions(-) rename pkg/userdata/helper/testdata/{download_binaries_v1.25.12.golden => download_binaries_v1.25.14.golden} (92%) rename pkg/userdata/helper/testdata/{download_binaries_v1.26.7.golden => download_binaries_v1.26.9.golden} (92%) rename pkg/userdata/helper/testdata/{download_binaries_v1.27.4.golden => download_binaries_v1.27.6.golden} (92%) rename pkg/userdata/helper/testdata/{download_binaries_v1.28.0.golden => download_binaries_v1.28.2.golden} (92%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.25.12-external.golden => kublet_systemd_unit_version-v1.25.14-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.25.12.golden => kublet_systemd_unit_version-v1.25.14.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.26.7-external.golden => kublet_systemd_unit_version-v1.26.9-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.26.7.golden => kublet_systemd_unit_version-v1.26.9.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.27.4-external.golden => kublet_systemd_unit_version-v1.27.6-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.27.4.golden => kublet_systemd_unit_version-v1.27.6.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.28.0-external.golden => kublet_systemd_unit_version-v1.28.2-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.28.0.golden => kublet_systemd_unit_version-v1.28.2.golden} (100%) rename pkg/userdata/ubuntu/testdata/{version-1.25.12.yaml => version-1.25.14.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.26.7.yaml => version-1.26.9.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.27.4.yaml => version-1.27.6.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.28.0.yaml => version-1.28.2.yaml} (99%) diff --git a/pkg/userdata/helper/common_test.go b/pkg/userdata/helper/common_test.go index 3b09f8e92..ebc4bc25b 100644 --- a/pkg/userdata/helper/common_test.go +++ b/pkg/userdata/helper/common_test.go @@ -26,9 +26,9 @@ var update = flag.Bool("update", false, "update testdata files") var ( versions = []*semver.Version{ - semver.MustParse("v1.25.12"), - semver.MustParse("v1.26.7"), - semver.MustParse("v1.27.4"), - semver.MustParse("v1.28.0"), + semver.MustParse("v1.25.14"), + semver.MustParse("v1.26.9"), + semver.MustParse("v1.27.6"), + semver.MustParse("v1.28.2"), } ) diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.25.12.golden b/pkg/userdata/helper/testdata/download_binaries_v1.25.14.golden similarity index 92% rename from pkg/userdata/helper/testdata/download_binaries_v1.25.12.golden rename to pkg/userdata/helper/testdata/download_binaries_v1.25.14.golden index 4786c596e..f50617b53 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.25.12.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.25.14.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.25.12/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.25.14/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.26.7.golden b/pkg/userdata/helper/testdata/download_binaries_v1.26.9.golden similarity index 92% rename from pkg/userdata/helper/testdata/download_binaries_v1.26.7.golden rename to pkg/userdata/helper/testdata/download_binaries_v1.26.9.golden index c72528aeb..2982cbae4 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.26.7.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.26.9.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.26.7/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.26.9/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.27.4.golden b/pkg/userdata/helper/testdata/download_binaries_v1.27.6.golden similarity index 92% rename from pkg/userdata/helper/testdata/download_binaries_v1.27.4.golden rename to pkg/userdata/helper/testdata/download_binaries_v1.27.6.golden index 602bf7e26..87010dae6 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.27.4.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.27.6.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.27.4/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.27.6/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.28.0.golden b/pkg/userdata/helper/testdata/download_binaries_v1.28.2.golden similarity index 92% rename from pkg/userdata/helper/testdata/download_binaries_v1.28.0.golden rename to pkg/userdata/helper/testdata/download_binaries_v1.28.2.golden index e438f0e67..4add0f3ee 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.28.0.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.28.2.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.28.0/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.28.2/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.12-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.14-external.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.12-external.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.14-external.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.12.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.14.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.12.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.14.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.7-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.9-external.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.7-external.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.9-external.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.7.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.9.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.7.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.9.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.4-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.6-external.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.4-external.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.6-external.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.4.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.6.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.4.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.6.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.2-external.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0-external.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.2-external.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.2.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.2.golden diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 37a4cd04f..de10a28a8 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -129,10 +129,10 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.25.12"), - semver.MustParse("v1.26.7"), - semver.MustParse("v1.27.4"), - semver.MustParse("v1.28.0"), + semver.MustParse("v1.25.14"), + semver.MustParse("v1.26.9"), + semver.MustParse("v1.27.6"), + semver.MustParse("v1.28.2"), } var tests []userDataTestCase diff --git a/pkg/userdata/ubuntu/testdata/version-1.25.12.yaml b/pkg/userdata/ubuntu/testdata/version-1.25.14.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.25.12.yaml rename to pkg/userdata/ubuntu/testdata/version-1.25.14.yaml index e1dcae9b1..c5a18ee0f 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.25.12.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.25.14.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.12}" + KUBE_VERSION="${KUBE_VERSION:-v1.25.14}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.26.7.yaml b/pkg/userdata/ubuntu/testdata/version-1.26.9.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.26.7.yaml rename to pkg/userdata/ubuntu/testdata/version-1.26.9.yaml index 13ac2dac9..c6bdf99ff 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.26.7.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.26.9.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.7}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.9}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.27.4.yaml b/pkg/userdata/ubuntu/testdata/version-1.27.6.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.27.4.yaml rename to pkg/userdata/ubuntu/testdata/version-1.27.6.yaml index e16c74d35..09bb8e03c 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.27.4.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.27.6.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.27.4}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.28.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.28.2.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.28.0.yaml rename to pkg/userdata/ubuntu/testdata/version-1.28.2.yaml index d8f8906d1..5680f5def 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.28.0.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.28.2.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 488732f98..3200731a5 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -343,7 +343,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.26. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.26.7", "1.27.4", "1.28.0"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.26.9", "1.27.6", "1.28.2"))) runScenarios(t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -423,7 +423,7 @@ func TestAWSProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := Not(VersionSelector("1.27.4", "1.28.0")) + selector := Not(VersionSelector("1.27.6", "1.28.2")) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -477,7 +477,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.4", "1.28.0"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.6", "1.28.2"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -499,7 +499,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.4", "1.28.0"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.6", "1.28.2"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index dc8c3412b..a8dace575 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,10 +33,10 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.25.12"), - semver.MustParse("v1.26.7"), - semver.MustParse("v1.27.4"), - semver.MustParse("v1.28.0"), + semver.MustParse("v1.25.14"), + semver.MustParse("v1.26.9"), + semver.MustParse("v1.27.6"), + semver.MustParse("v1.28.2"), } operatingSystems = []providerconfigtypes.OperatingSystem{ From ae2f55873d5c036507728dec86e037dbfbfe3031 Mon Sep 17 00:00:00 2001 From: Yakul Garg <2000yeshu@gmail.com> Date: Mon, 25 Sep 2023 11:40:26 +0530 Subject: [PATCH 350/489] - Fix isssue of creation of multiple vultr instance due to eventual consistency of tags (#1706) - Add support for vultr bare metal instances - Upgrade govulr to v3 - Add support for vultr VPC v1/v2, Handle API response codes Signed-off-by: Yakul Garg <2000yeshu@gmail.com> --- examples/vultr-machinedeployment.yaml | 13 + go.mod | 2 +- go.sum | 4 +- pkg/cloudprovider/provider/vultr/provider.go | 409 +++++++++++++++--- .../provider/vultr/types/types.go | 16 +- 5 files changed, 365 insertions(+), 79 deletions(-) diff --git a/examples/vultr-machinedeployment.yaml b/examples/vultr-machinedeployment.yaml index 3aa064daf..89ecbb0c9 100644 --- a/examples/vultr-machinedeployment.yaml +++ b/examples/vultr-machinedeployment.yaml @@ -43,9 +43,22 @@ spec: namespace: kube-system name: machine-controller-vultr key: apiKey + # Default is false meaning a virtual machine instance is created + # If true, a bare metal instance is created + physicalMachine: false region: blr plan: "vhf-8c-32gb" + # This takes precedence over enableVPC + vpcId: + - + # For more reference, see + # https://www.vultr.com/api/#tag/instances/operation/create-instance + enableVPC: false + enableVPC2: true + vpc2Id: + - # Required: app_id, image_id, os_id, snapshot_id, or iso_id must be provided. Currently only os_id is supported. + # This takes precedence over operatingSystem osId: 215 # Optional tags: diff --git a/go.mod b/go.mod index c15abc74d..12d583bfa 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/tinkerbell/tink v0.8.0 github.com/vmware/go-vcloud-director/v2 v2.21.0 github.com/vmware/govmomi v0.30.7 - github.com/vultr/govultr/v2 v2.17.2 + github.com/vultr/govultr/v3 v3.3.1 go.anx.io/go-anxcloud v0.5.3 go.uber.org/zap v1.25.0 golang.org/x/crypto v0.12.0 diff --git a/go.sum b/go.sum index 5b8cb0c49..1a7a8a691 100644 --- a/go.sum +++ b/go.sum @@ -720,8 +720,8 @@ github.com/vmware/govmomi v0.30.7 h1:YO8CcDpLJzmq6PK5/CBQbXyV21iCMh8SbdXt+xNkXp8 github.com/vmware/govmomi v0.30.7/go.mod h1:epgoslm97rLECMV4D+08ORzUBEU7boFSepKjt7AYVGg= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= -github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs= -github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI= +github.com/vultr/govultr/v3 v3.3.1 h1:gn46pSL0A3pxBKZklzwtYgSW2Iq7dW1euDRBlOsIzTo= +github.com/vultr/govultr/v3 v3.3.1/go.mod h1:7NjuHeQv5vgUWR2H1sPc9D+xffrT5ql+kNi6R3yuwzo= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= diff --git a/pkg/cloudprovider/provider/vultr/provider.go b/pkg/cloudprovider/provider/vultr/provider.go index 09e1f9eca..8e074e21e 100644 --- a/pkg/cloudprovider/provider/vultr/provider.go +++ b/pkg/cloudprovider/provider/vultr/provider.go @@ -18,11 +18,14 @@ package vultr import ( "context" + "encoding/base64" "errors" "fmt" + "net/http" "strconv" + "time" - "github.com/vultr/govultr/v2" + "github.com/vultr/govultr/v3" "go.uber.org/zap" "golang.org/x/oauth2" @@ -38,8 +41,20 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apimachinery/pkg/util/wait" ) +const ( + createCheckPeriod = 10 * time.Second + createCheckTimeout = 5 * time.Minute + createCheckFailedWaitPeriod = 10 * time.Second +) + +type ValidVPC struct { + IsAllValid bool + InvalidVpcs []string +} + type provider struct { configVarResolver *providerconfig.ConfigVarResolver } @@ -50,11 +65,17 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes } type Config struct { - APIKey string - Region string - Plan string - OsID string - Tags []string + PhysicalMachine bool + APIKey string + Region string + Plan string + OsID string + Tags []string + VpcID []string + EnableVPC bool + EnableIPv6 bool + EnableVPC2 bool + Vpc2ID []string } func getIDForOS(os providerconfigtypes.OperatingSystem) (int, error) { @@ -97,6 +118,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } c := Config{} + c.APIKey, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.APIKey, "VULTR_API_KEY") if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"apiKey\" field, error = %w", err) @@ -118,6 +140,12 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } c.Tags = rawConfig.Tags + c.PhysicalMachine = rawConfig.PhysicalMachine + c.EnableIPv6 = rawConfig.EnableIPv6 + c.VpcID = rawConfig.VpcID + c.EnableVPC = rawConfig.EnableVPC + c.EnableVPC2 = rawConfig.EnableVPC2 + c.Vpc2ID = rawConfig.Vpc2ID return &c, pconfig, err } @@ -126,6 +154,68 @@ func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.Machin return spec, nil } +func (p *provider) validateVpc(ctx context.Context, client *govultr.Client, c *Config, legacyVPC bool) (ValidVPC, error) { + validVpc := ValidVPC{IsAllValid: true} + accountvpcs := []string{} + var requestedvpcs []string + + if legacyVPC { + for { + vpcs, meta, err := func(ctx context.Context, client *govultr.Client) ([]govultr.VPC, *govultr.Meta, error) { + vpcs, meta, resp, err := client.VPC.List(ctx, &govultr.ListOptions{}) + if err != nil { + return nil, nil, vltErrorToTerminalError(resp.StatusCode, err) + } + defer resp.Body.Close() + + return vpcs, meta, nil + }(ctx, client) + if err != nil { + return validVpc, err + } + for _, v := range vpcs { + accountvpcs = append(accountvpcs, v.ID) + } + if meta.Links.Next == "" { + break + } + } + requestedvpcs = c.VpcID + } else { + for { + vpcs, meta, err := func(ctx context.Context, client *govultr.Client) ([]govultr.VPC2, *govultr.Meta, error) { + vpcs, meta, resp, err := client.VPC2.List(ctx, &govultr.ListOptions{}) + if err != nil { + return nil, nil, vltErrorToTerminalError(resp.StatusCode, err) + } + defer resp.Body.Close() + + return vpcs, meta, nil + }(ctx, client) + if err != nil { + return validVpc, err + } + for _, v := range vpcs { + accountvpcs = append(accountvpcs, v.ID) + } + if meta.Links.Next == "" { + break + } + } + requestedvpcs = c.Vpc2ID + } + accountvpcsset := sets.New[string](accountvpcs...) + // Iterator to provide user the exact mismatches + for _, v := range requestedvpcs { + if !accountvpcsset.Has(v) { + validVpc.IsAllValid = false + validVpc.InvalidVpcs = append(validVpc.InvalidVpcs, v) + } + } + + return validVpc, nil +} + func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { c, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { @@ -155,12 +245,13 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus client := getClient(ctx, c.APIKey) - plans, err := client.Region.Availability(ctx, c.Region, "") + plans, resp, err := client.Region.Availability(ctx, c.Region, "") // TODO: Validate region separately if err != nil { - return fmt.Errorf("invalid/not supported region specified %q: %w", c.Region, err) + return err } + resp.Body.Close() planFound := false @@ -168,37 +259,73 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus for _, plan := range plans.AvailablePlans { if plan == c.Plan { planFound = true + break } } if !planFound { - return fmt.Errorf("invalid/not supported plan specified %q: %w", c.Plan, err) + return fmt.Errorf("invalid/not supported plan specified %q, available plans are: %q, %w", c.Plan, plans.AvailablePlans, err) + } + + validvpc, err := p.validateVpc(ctx, client, c, false) + if err != nil { + return err + } + if !validvpc.IsAllValid { + return fmt.Errorf("invalid/not supported vpc id specified %v", validvpc.InvalidVpcs) + } + + if c.PhysicalMachine { + // Don't check for validity of legacy VPC as BareMetal doesn't support VPC v1 + return nil + } + + // Verify legacy VPCs + validvpc, err = p.validateVpc(ctx, client, c, true) + if err != nil { + return err } + + if !validvpc.IsAllValid { + return fmt.Errorf("invalid/not supported vpc id specified %v", validvpc.InvalidVpcs) + } + return nil } -func (p *provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (*vultrInstance, error) { - c, _, err := p.getConfig(machine.Spec.ProviderSpec) +func (p *provider) getPhysicalMachine(ctx context.Context, c *Config, machine *clusterv1alpha1.Machine) (*vultrPhysicalMachine, error) { + client := getClient(ctx, c.APIKey) + // Not looping on metadata assuming that tagged machines won;t cross + // pagination boundary + instances, _, resp, err := client.BareMetalServer.List(ctx, &govultr.ListOptions{ + Tag: string(machine.UID), + }) if err != nil { - return nil, cloudprovidererrors.TerminalError{ - Reason: common.InvalidConfigurationMachineError, - Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), + return nil, vltErrorToTerminalError(resp.StatusCode, err) + } + resp.Body.Close() + for _, instance := range instances { + if sets.NewString(instance.Tags...).Has(string(machine.UID)) { + return &vultrPhysicalMachine{instance: &instance}, nil } } + return nil, cloudprovidererrors.ErrInstanceNotFound +} +func (p *provider) getVirtualMachine(ctx context.Context, c *Config, machine *clusterv1alpha1.Machine) (*vultrVirtualMachine, error) { client := getClient(ctx, c.APIKey) - instances, _, err := client.Instance.List(ctx, &govultr.ListOptions{ + instances, _, resp, err := client.Instance.List(ctx, &govultr.ListOptions{ Tag: string(machine.UID), }) if err != nil { - return nil, vltErrorToTerminalError(err, "failed to list servers") + return nil, vltErrorToTerminalError(resp.StatusCode, err) } + resp.Body.Close() for _, instance := range instances { - for _, tag := range instance.Tags { - if tag == string(machine.UID) { - return &vultrInstance{instance: &instance}, nil - } + if sets.NewString(instance.Tags...).Has(string(machine.UID)) && + instance.Label == machine.Name { + return &vultrVirtualMachine{instance: &instance}, nil } } @@ -206,14 +333,101 @@ func (p *provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (* } func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - return p.get(ctx, machine) + c, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), + } + } + if !c.PhysicalMachine { + return p.getVirtualMachine(ctx, c, machine) + } + + return p.getPhysicalMachine(ctx, c, machine) } func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { return "", "", nil } -func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) waitForInstanceCreation(ctx context.Context, c *Config, instance instance.Instance, machine *clusterv1alpha1.Machine) error { + return wait.PollUntilContextTimeout(ctx, createCheckPeriod, createCheckTimeout, false, func(ctx context.Context) (bool, error) { + var err error + if !c.PhysicalMachine { + _, err = p.getVirtualMachine(ctx, c, machine) + } else { + _, err = p.getPhysicalMachine(ctx, c, machine) + } + + if err != nil { + if cloudprovidererrors.IsNotFound(err) { + // Continue the loop as the instances was successfully fetched + // just that our instance was not found + return false, nil + } + if isTerminalErr, _, _ := cloudprovidererrors.IsTerminalError(err); isTerminalErr { + return true, err + } + // Wait for some time as instance creation is successful + // just that we are not able to fetch it + time.Sleep(createCheckFailedWaitPeriod) + return false, fmt.Errorf("instance %q created but controller failed to fetch instance details", instance.Name()) + } + return true, nil + }) +} + +func (p *provider) createVirtualMachine(ctx context.Context, client *govultr.Client, c *Config, machine *clusterv1alpha1.Machine, osid int, userdata string) (*vultrVirtualMachine, error) { + tags := sets.List[string](sets.New(c.Tags...).Insert(string(machine.UID))) + + instanceCreateRequest := govultr.InstanceCreateReq{ + Region: c.Region, + Plan: c.Plan, + OsID: osid, + + Label: machine.Spec.Name, + UserData: base64.StdEncoding.EncodeToString([]byte(userdata)), + Tags: tags, + + EnableIPv6: &c.EnableIPv6, + EnableVPC: &c.EnableVPC, + AttachVPC: c.VpcID, + EnableVPC2: &c.EnableVPC2, + AttachVPC2: c.Vpc2ID, + } + instance, resp, err := client.Instance.Create(ctx, &instanceCreateRequest) + if err != nil { + return nil, vltErrorToTerminalError(resp.StatusCode, err) + } + resp.Body.Close() + + return &vultrVirtualMachine{instance: instance}, nil +} + +func (p *provider) createPhysicalMachine(ctx context.Context, client *govultr.Client, c *Config, machine *clusterv1alpha1.Machine, osid int, userdata string) (*vultrPhysicalMachine, error) { + tags := sets.NewString(c.Tags...).Insert(string(machine.UID)).List() + + bareMetalCreateRequest := govultr.BareMetalCreate{ + Region: c.Region, + Plan: c.Plan, + Label: machine.Spec.Name, + UserData: base64.StdEncoding.EncodeToString([]byte(userdata)), + EnableIPv6: &c.EnableIPv6, + Tags: tags, + OsID: osid, + AttachVPC2: c.Vpc2ID, + EnableVPC2: &c.EnableVPC2, + } + instance, resp, err := client.BareMetalServer.Create(ctx, &bareMetalCreateRequest) + if err != nil { + return nil, vltErrorToTerminalError(resp.StatusCode, err) + } + resp.Body.Close() + return &vultrPhysicalMachine{instance: instance}, nil +} + +func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -222,8 +436,6 @@ func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *cl } } - client := getClient(ctx, c.APIKey) - if c.OsID == "" { osID, err := getIDForOS(pc.OperatingSystem) if err != nil { @@ -234,33 +446,42 @@ func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *cl } c.OsID = strconv.Itoa(osID) } - - if c.Tags == nil { - c.Tags = []string{} - } - - c.Tags = append(c.Tags, string(machine.UID)) - strOsID, err := strconv.Atoi(c.OsID) if err != nil { - return nil, err + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Cannot parse operating system id %q, details = %v", pc.OperatingSystem, err), + } } + client := getClient(ctx, c.APIKey) - instanceCreateRequest := govultr.InstanceCreateReq{ - Region: c.Region, - Plan: c.Plan, - Label: machine.Spec.Name, - UserData: userdata, - Tags: c.Tags, - OsID: strOsID, + var instance instance.Instance + if !c.PhysicalMachine { + instance, err = p.createVirtualMachine(ctx, client, c, machine, strOsID, userdata) + if err != nil { + return nil, err + } + } else { + instance, err = p.createPhysicalMachine(ctx, client, c, machine, strOsID, userdata) + if err != nil { + return nil, err + } } - res, err := client.Instance.Create(ctx, &instanceCreateRequest) + err = p.waitForInstanceCreation(ctx, c, instance, machine) if err != nil { - return nil, vltErrorToTerminalError(err, "failed to create server") + if !c.PhysicalMachine { + if err := client.Instance.Delete(ctx, instance.ID()); err != nil { + log.Error("Failed to cleanup instance after failed creation: %v", err) + } + } else { + if err := client.BareMetalServer.Delete(ctx, instance.ID()); err != nil { + log.Error("Failed to cleanup bare metal instance after failed creation: %v", err) + } + } + return nil, err } - - return &vultrInstance{instance: res}, nil + return instance, nil } func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { @@ -281,8 +502,14 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine } client := getClient(ctx, c.APIKey) - if err = client.Instance.Delete(ctx, instance.ID()); err != nil { - return false, vltErrorToTerminalError(err, "failed to delete server") + if !c.PhysicalMachine { + if err := client.Instance.Delete(ctx, instance.ID()); err != nil { + return false, fmt.Errorf("failed to delete instance: %w", err) + } + } else { + if err := client.BareMetalServer.Delete(ctx, instance.ID()); err != nil { + return false, fmt.Errorf("failed to delete bare metal instance: %w", err) + } } return false, nil @@ -306,49 +533,76 @@ func (p *provider) MigrateUID(ctx context.Context, _ *zap.SugaredLogger, machine return fmt.Errorf("failed to decode providerconfig: %w", err) } client := getClient(ctx, c.APIKey) - instances, _, err := client.Instance.List(ctx, &govultr.ListOptions{PerPage: 1000}) - if err != nil { - return fmt.Errorf("failed to list instances: %w", err) - } - for _, instance := range instances { - if instance.Label == machine.Spec.Name && sets.NewString(instance.Tags...).Has(string(machine.UID)) { - _, err = client.Instance.Update(ctx, instance.ID, &govultr.InstanceUpdateReq{ - Tags: sets.NewString(instance.Tags...).Delete(string(machine.UID)).Insert(string(newUID)).List(), - }) - if err != nil { - return fmt.Errorf("failed to tag instance with new UID tag: %w", err) - } + if !c.PhysicalMachine { + instance, err := p.getVirtualMachine(ctx, c, machine) + if err != nil { + return err + } + _, resp, err := client.Instance.Update(ctx, instance.instance.ID, &govultr.InstanceUpdateReq{ + Tags: sets.NewString(instance.instance.Tags...).Delete(string(machine.UID)).Insert(string(newUID)).List(), + }) + if err != nil { + return vltErrorToTerminalError(resp.StatusCode, err) } + resp.Body.Close() + return nil } - + instance, err := p.getPhysicalMachine(ctx, c, machine) + if err != nil { + return fmt.Errorf("failed to get instance with UID tag: %w", err) + } + _, resp, err := client.BareMetalServer.Update(ctx, instance.instance.ID, &govultr.BareMetalUpdate{ + Tags: sets.NewString(instance.instance.Tags...).Delete(string(machine.UID)).Insert(string(newUID)).List(), + }) + if err != nil { + return vltErrorToTerminalError(resp.StatusCode, err) + } + resp.Body.Close() return nil } -type vultrInstance struct { +type vultrVirtualMachine struct { instance *govultr.Instance } +type vultrPhysicalMachine struct { + instance *govultr.BareMetalServer +} -func (v *vultrInstance) Name() string { +func (v *vultrVirtualMachine) Name() string { + return v.instance.Label +} +func (v *vultrPhysicalMachine) Name() string { return v.instance.Label } -func (v *vultrInstance) ID() string { +func (v *vultrVirtualMachine) ID() string { + return v.instance.ID +} +func (v *vultrPhysicalMachine) ID() string { return v.instance.ID } -func (v *vultrInstance) ProviderID() string { +func (v *vultrVirtualMachine) ProviderID() string { + return "vultr://" + v.instance.ID +} +func (v *vultrPhysicalMachine) ProviderID() string { return "vultr://" + v.instance.ID } -func (v *vultrInstance) Addresses() map[string]v1.NodeAddressType { +func (v *vultrVirtualMachine) Addresses() map[string]v1.NodeAddressType { addresses := map[string]v1.NodeAddressType{} addresses[v.instance.MainIP] = v1.NodeExternalIP addresses[v.instance.InternalIP] = v1.NodeInternalIP return addresses } +func (v *vultrPhysicalMachine) Addresses() map[string]v1.NodeAddressType { + addresses := map[string]v1.NodeAddressType{} + addresses[v.instance.MainIP] = v1.NodeExternalIP + return addresses +} -func (v *vultrInstance) Status() instance.Status { +func (v *vultrVirtualMachine) Status() instance.Status { switch v.instance.Status { case "active": return instance.StatusRunning @@ -359,15 +613,28 @@ func (v *vultrInstance) Status() instance.Status { return instance.StatusUnknown } } - -func vltErrorToTerminalError(err error, msg string) error { - prepareAndReturnError := func() error { - return fmt.Errorf("%s, due to %w", msg, err) +func (v *vultrPhysicalMachine) Status() instance.Status { + switch v.instance.Status { + case "active": + return instance.StatusRunning + case "pending": + return instance.StatusCreating + // "suspending" or "resizing" + default: + return instance.StatusUnknown } - if err != nil { - return prepareAndReturnError() +} + +func vltErrorToTerminalError(status int, err error) error { + switch status { + case http.StatusUnauthorized: + return cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: "A request has been rejected due to invalid credentials which were taken from the MachineSpec", + } + default: + return err } - return err } func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { diff --git a/pkg/cloudprovider/provider/vultr/types/types.go b/pkg/cloudprovider/provider/vultr/types/types.go index f4b61aee2..278ea6066 100644 --- a/pkg/cloudprovider/provider/vultr/types/types.go +++ b/pkg/cloudprovider/provider/vultr/types/types.go @@ -22,11 +22,17 @@ import ( ) type RawConfig struct { - APIKey providerconfigtypes.ConfigVarString `json:"apiKey,omitempty"` - Region providerconfigtypes.ConfigVarString `json:"region"` - Plan providerconfigtypes.ConfigVarString `json:"plan"` - OsID providerconfigtypes.ConfigVarString `json:"osId"` - Tags []string `json:"tags,omitempty"` + PhysicalMachine bool `json:"physicalMachine,omitempty"` + APIKey providerconfigtypes.ConfigVarString `json:"apiKey,omitempty"` + Region providerconfigtypes.ConfigVarString `json:"region"` + Plan providerconfigtypes.ConfigVarString `json:"plan"` + OsID providerconfigtypes.ConfigVarString `json:"osId"` + Tags []string `json:"tags,omitempty"` + VpcID []string `json:"vpcId,omitempty"` + Vpc2ID []string `json:"vpc2Id,omitempty"` + EnableVPC bool `json:"enableVPC,omitempty"` + EnableVPC2 bool `json:"enableVPC2,omitempty"` + EnableIPv6 bool `json:"enableIPv6,omitempty"` } func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { From da744cdc07a003b40a4aaf40a488095e5a8d99db Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 5 Oct 2023 18:05:49 +0500 Subject: [PATCH 351/489] Remove support for kubernetes v1.25 (#1709) * Remove support for kubernetes v1.25 Signed-off-by: Waleed Malik * Fix issue with port not being allocated in time Signed-off-by: Waleed Malik * Fix AWS EBS tests Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- .prow/e2e-features.yaml | 3 +- .prow/provider-openstack.yaml | 6 +- README.md | 2 +- hack/ci/setup-kind-cluster.sh | 7 +- pkg/userdata/amzn2/provider_test.go | 4 +- ...-v1.25-aws.yaml => kubelet-v1.28-aws.yaml} | 2 +- pkg/userdata/centos/provider_test.go | 4 +- ...-v1.25-aws.yaml => kubelet-v1.28-aws.yaml} | 2 +- pkg/userdata/flatcar/provider_test.go | 8 +- ...t_v1.25.0.yaml => cloud-init_v1.28.0.yaml} | 2 +- ...ion_v1.25.0.json => ignition_v1.28.0.json} | 2 +- pkg/userdata/helper/common_test.go | 1 - .../download_binaries_v1.25.14.golden | 17 - ...temd_unit_version-v1.25.14-external.golden | 36 -- ...ublet_systemd_unit_version-v1.25.14.golden | 35 -- pkg/userdata/rhel/provider_test.go | 10 +- ...-v1.25-aws.yaml => kubelet-v1.28-aws.yaml} | 2 +- ...utanix.yaml => kubelet-v1.28-nutanix.yaml} | 2 +- .../rhel/testdata/pod-cidr-azure-rhel.yaml | 2 +- pkg/userdata/rockylinux/provider_test.go | 4 +- ...-v1.25-aws.yaml => kubelet-v1.28-aws.yaml} | 2 +- pkg/userdata/ubuntu/provider_test.go | 15 +- pkg/userdata/ubuntu/testdata/containerd.yaml | 2 +- .../digitalocean-dualstack-IPv6+IPv4.yaml | 2 +- .../testdata/digitalocean-dualstack.yaml | 2 +- .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 2 +- pkg/userdata/ubuntu/testdata/docker.yaml | 2 +- .../kubelet-version-without-v-prefix.yaml | 2 +- .../ubuntu/testdata/multiple-dns-servers.yaml | 2 +- .../ubuntu/testdata/multiple-ssh-keys.yaml | 2 +- .../openstack-dualstack-IPv6+IPv4.yaml | 2 +- .../ubuntu/testdata/openstack-dualstack.yaml | 2 +- .../openstack-overwrite-cloud-config.yaml | 2 +- pkg/userdata/ubuntu/testdata/openstack.yaml | 2 +- .../ubuntu/testdata/version-1.25.14.yaml | 458 ------------------ .../ubuntu/testdata/vsphere-mirrors.yaml | 2 +- .../ubuntu/testdata/vsphere-proxy.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere.yaml | 2 +- test/e2e/provisioning/all_e2e_test.go | 7 +- test/e2e/provisioning/helper.go | 1 - 40 files changed, 61 insertions(+), 603 deletions(-) rename pkg/userdata/amzn2/testdata/{kubelet-v1.25-aws.yaml => kubelet-v1.28-aws.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.25-aws.yaml => kubelet-v1.28-aws.yaml} (99%) rename pkg/userdata/flatcar/testdata/{cloud-init_v1.25.0.yaml => cloud-init_v1.28.0.yaml} (99%) rename pkg/userdata/flatcar/testdata/{ignition_v1.25.0.json => ignition_v1.28.0.json} (99%) delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.25.14.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.14-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.14.golden rename pkg/userdata/rhel/testdata/{kubelet-v1.25-aws.yaml => kubelet-v1.28-aws.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.25-nutanix.yaml => kubelet-v1.28-nutanix.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.25-aws.yaml => kubelet-v1.28-aws.yaml} (99%) delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.25.14.yaml diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index c4ea60b65..1974d28be 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -78,7 +78,8 @@ presubmits: memory: 7Gi - name: pull-machine-controller-e2e-ubuntu-upgrade - always_run: true + # In-tree CCM is not supported for openstack starting from k8s 1.26. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. + # run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index f89a151ba..395304a3b 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -14,7 +14,8 @@ presubmits: - name: pull-machine-controller-e2e-openstack - run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" + # In-tree CCM is not supported for openstack starting from k8s 1.26. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. + # run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -46,7 +47,8 @@ presubmits: memory: 7Gi - name: pull-machine-controller-e2e-openstack-project-auth - run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" + # In-tree CCM is not supported for openstack starting from k8s 1.26. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. + # run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: diff --git a/README.md b/README.md index 0dc5483dd..d3adc571e 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,9 @@ machine-controller tries to follow the Kubernetes version Currently supported K8S versions are: +- 1.28 - 1.27 - 1.26 -- 1.25 ### Community Providers diff --git a/hack/ci/setup-kind-cluster.sh b/hack/ci/setup-kind-cluster.sh index 303004853..c68467511 100755 --- a/hack/ci/setup-kind-cluster.sh +++ b/hack/ci/setup-kind-cluster.sh @@ -199,12 +199,15 @@ if [ -z "${DISABLE_CLUSTER_EXPOSER:-}" ]; then echodate "Successfully set up iptables rules for nodeports" + # Wait for 10 seconds before checking if the apiserver is reachable. + sleep 10 + # Compute external kube-apiserver address # If svc is not found then we need to check cluster-exposer logs PORT=$(kubectl --kubeconfig /etc/kubeconfig/kubeconfig get svc -l prow.k8s.io/id=$PROW_JOB_ID -o jsonpath="{.items..spec.ports[0].nodePort}") - if [ -z "$PORT" ] || [ -z "$NODE_NAME" ] || [ -z "$NODE_IP" ]; then - echodate "This script was unable to determine the external IP for kube-apiserver." + if [ -z "$PORT" ]; then + echodate "This script was unable to determine the nodeport for kube-apiserver." exit 1 fi diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index eb2af6bce..b74b09282 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -168,11 +168,11 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "kubelet-v1.25-aws", + name: "kubelet-v1.28-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.25.0", + Kubelet: "1.28.0", }, }, }, diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml index 08d7651da..2c1bcbefc 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml @@ -143,7 +143,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index 88e1260a3..57a94030c 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -169,11 +169,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.25-aws", + name: "kubelet-v1.28-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.25.0", + Kubelet: "1.28.0", }, }, }, diff --git a/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml index 5948e4c23..99f15e731 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index 1ac154916..eb91a40ef 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -124,7 +124,7 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "ignition_v1.25.0", + name: "ignition_v1.28.0", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -140,7 +140,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.25.0", + Kubelet: "1.28.0", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -248,7 +248,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.25.0", + name: "cloud-init_v1.28.0", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -264,7 +264,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.25.0", + Kubelet: "1.28.0", }, }, ccProvider: &fakeCloudConfigProvider{ diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml similarity index 99% rename from pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml rename to pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml index 363a17d88..7f944534d 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.25.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml @@ -431,7 +431,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.28.0.json similarity index 99% rename from pkg/userdata/flatcar/testdata/ignition_v1.25.0.json rename to pkg/userdata/flatcar/testdata/ignition_v1.28.0.json index 2a9886a99..141859dae 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.25.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.28.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.25.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.28.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/common_test.go b/pkg/userdata/helper/common_test.go index ebc4bc25b..4f4bb710c 100644 --- a/pkg/userdata/helper/common_test.go +++ b/pkg/userdata/helper/common_test.go @@ -26,7 +26,6 @@ var update = flag.Bool("update", false, "update testdata files") var ( versions = []*semver.Version{ - semver.MustParse("v1.25.14"), semver.MustParse("v1.26.9"), semver.MustParse("v1.27.6"), semver.MustParse("v1.28.2"), diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.25.14.golden b/pkg/userdata/helper/testdata/download_binaries_v1.25.14.golden deleted file mode 100644 index f50617b53..000000000 --- a/pkg/userdata/helper/testdata/download_binaries_v1.25.14.golden +++ /dev/null @@ -1,17 +0,0 @@ -mkdir -p /opt/bin/ -mkdir -p /var/lib/calico -mkdir -p /etc/kubernetes/manifests -mkdir -p /etc/cni/net.d -mkdir -p /opt/cni/bin -if [ ! -f /opt/cni/bin/loopback ]; then - curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - -fi -if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.25.14/bin/linux/amd64/kubelet - chmod +x /opt/bin/kubelet -fi - -if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh -fi diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.14-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.14-external.golden deleted file mode 100644 index 50f4f5138..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.14-external.golden +++ /dev/null @@ -1,36 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.14.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.14.golden deleted file mode 100644 index e70567560..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.25.14.golden +++ /dev/null @@ -1,35 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index 6f101211f..f4e8b338c 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -102,11 +102,11 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.25-aws", + name: "kubelet-v1.28-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.25.0", + Kubelet: "1.28.0", }, }, }, @@ -187,11 +187,11 @@ func TestUserDataGeneration(t *testing.T) { externalCloudProvider: true, }, { - name: "kubelet-v1.25-nutanix", + name: "kubelet-v1.28-nutanix", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.25.0", + Kubelet: "1.28.0", }, }, cloudProviderName: stringPtr("nutanix"), @@ -201,7 +201,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.25.0", + Kubelet: "1.28.0", }, }, cloudProviderName: stringPtr("azure"), diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.28-aws.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.28-aws.yaml index 46ab37dd3..07b8bddcf 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.28-aws.yaml @@ -143,7 +143,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml index c8341ea37..1ab290bec 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index 5b7e2b2ea..414032b72 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -148,7 +148,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/provider_test.go b/pkg/userdata/rockylinux/provider_test.go index 2ec2339c6..b306379b4 100644 --- a/pkg/userdata/rockylinux/provider_test.go +++ b/pkg/userdata/rockylinux/provider_test.go @@ -102,11 +102,11 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.25-aws", + name: "kubelet-v1.28-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.25.0", + Kubelet: "1.28.0", }, }, }, diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml index 453aca786..694e86538 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index de10a28a8..e0a976144 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -94,7 +94,7 @@ kPe6XoSbiLm/kxk32T0= ) const ( - defaultVersion = "1.25.5" + defaultVersion = "1.27.6" ) type fakeCloudConfigProvider struct { @@ -129,7 +129,6 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.25.14"), semver.MustParse("v1.26.9"), semver.MustParse("v1.27.6"), semver.MustParse("v1.28.2"), @@ -235,7 +234,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.7", + Kubelet: "1.27.0", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -260,7 +259,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.7", + Kubelet: "1.27.0", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -423,7 +422,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.7", + Kubelet: "1.27.0", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -449,7 +448,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.7", + Kubelet: "1.27.0", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -475,7 +474,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.7", + Kubelet: "1.27.0", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -505,7 +504,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.7", + Kubelet: "1.27.0", }, }, ccProvider: &fakeCloudConfigProvider{ diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index d7d3f5801..b3e80ca68 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -152,7 +152,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml index ceff63465..5d7a20718 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml index 3d1cd0d67..ba3784b47 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 638fa246f..62ccbd59b 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -152,7 +152,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml index 2dc056290..d713090a7 100644 --- a/pkg/userdata/ubuntu/testdata/docker.yaml +++ b/pkg/userdata/ubuntu/testdata/docker.yaml @@ -152,7 +152,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index d639d4529..a38eb1ab6 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 44cc1d071..76150fed8 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index aa3ffbd14..7de5f9f0e 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -152,7 +152,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml index 6f0f51723..2f6334f43 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml index c023ec7d7..97304a73e 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index e7097b703..a93d52ee9 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index bbc69ff3e..486ff34da 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.25.14.yaml b/pkg/userdata/ubuntu/testdata/version-1.25.14.yaml deleted file mode 100644 index c5a18ee0f..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.25.14.yaml +++ /dev/null @@ -1,458 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index 766f4331a..7318b6140 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -160,7 +160,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 6aaee968c..4d5feb72a 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -160,7 +160,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 82f97564c..6dd690532 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 3200731a5..fc962fcf3 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -84,8 +84,9 @@ const ( ) const ( - defaultKubernetesVersion = "1.25.9" - defaultContainerRuntime = "containerd" + defaultKubernetesVersion = "1.27.6" + awsDefaultKubernetesVersion = "1.26.9" + defaultContainerRuntime = "containerd" ) var testRunIdentifier = flag.String("identifier", "local", "The unique identifier for this test run") @@ -574,7 +575,7 @@ func TestAWSEbsEncryptionEnabledProvisioningE2E(t *testing.T) { name: "AWS with ebs encryption enabled", osName: "ubuntu", containerRuntime: defaultContainerRuntime, - kubernetesVersion: defaultKubernetesVersion, + kubernetesVersion: awsDefaultKubernetesVersion, executor: verifyCreateAndDelete, } testScenario(t, scenario, fmt.Sprintf("aws-%s", *testRunIdentifier), params, AWSEBSEncryptedManifest, false) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index a8dace575..5ba7817e4 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,7 +33,6 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.25.14"), semver.MustParse("v1.26.9"), semver.MustParse("v1.27.6"), semver.MustParse("v1.28.2"), From e6de2fe75fb141c6d8a937e0756f1376e0e792f9 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 10 Oct 2023 15:15:56 +0500 Subject: [PATCH 352/489] Pin flatcar to 3510.2.8 (#1711) Signed-off-by: Waleed Malik --- pkg/cloudprovider/provider/aws/provider.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 442d95585..2b19d243f 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -168,7 +168,9 @@ var ( providerconfigtypes.OperatingSystemFlatcar: { awstypes.CPUArchitectureX86_64: { // Be as precise as possible - otherwise we might get a nightly dev build - description: "Flatcar Container Linux stable *", + // Pin flatcar to 3510.2.8 since the latest version 3602.2.0 is broken. Reference: https://github.com/kubermatic/kubermatic/issues/12690 + // TODO: Remove this pinning once the issue is fixed. + description: "Flatcar Container Linux stable 3510.2.8 *", // The AWS marketplace ID from AWS owner: "075585003325", }, From 367dbf4c1950d1fad5f9229f571aff19581391f2 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 11 Oct 2023 16:34:01 +0500 Subject: [PATCH 353/489] Revert "Pin flatcar to 3510.2.8 (#1711)" (#1715) This reverts commit e6de2fe75fb141c6d8a937e0756f1376e0e792f9. --- pkg/cloudprovider/provider/aws/provider.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 2b19d243f..442d95585 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -168,9 +168,7 @@ var ( providerconfigtypes.OperatingSystemFlatcar: { awstypes.CPUArchitectureX86_64: { // Be as precise as possible - otherwise we might get a nightly dev build - // Pin flatcar to 3510.2.8 since the latest version 3602.2.0 is broken. Reference: https://github.com/kubermatic/kubermatic/issues/12690 - // TODO: Remove this pinning once the issue is fixed. - description: "Flatcar Container Linux stable 3510.2.8 *", + description: "Flatcar Container Linux stable *", // The AWS marketplace ID from AWS owner: "075585003325", }, From c2fa0c33694c7a220f01d103eeccdd8ec1b1b190 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Oct 2023 10:57:58 +0200 Subject: [PATCH 354/489] Bump golang.org/x/net from 0.14.0 to 0.17.0 (#1719) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.14.0 to 0.17.0. - [Commits](https://github.com/golang/net/compare/v0.14.0...v0.17.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 12d583bfa..395a89c41 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/vultr/govultr/v3 v3.3.1 go.anx.io/go-anxcloud v0.5.3 go.uber.org/zap v1.25.0 - golang.org/x/crypto v0.12.0 + golang.org/x/crypto v0.14.0 golang.org/x/oauth2 v0.11.0 gomodules.xyz/jsonpatch/v2 v2.4.0 google.golang.org/api v0.137.0 @@ -167,11 +167,11 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/net v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/term v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.3 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 1a7a8a691..ee14e56a6 100644 --- a/go.sum +++ b/go.sum @@ -806,8 +806,8 @@ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -901,8 +901,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1006,15 +1006,15 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1027,8 +1027,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 921c8e114f72d0e753b3231f8f3265cc2dcfd430 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 23 Oct 2023 11:41:06 +0500 Subject: [PATCH 355/489] Remove Zenhub integration (#1724) Signed-off-by: Waleed Malik --- README.md | 2 -- Zenhub.md | 15 --------------- 2 files changed, 17 deletions(-) delete mode 100644 Zenhub.md diff --git a/README.md b/README.md index d3adc571e..4407629df 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,6 @@ Thanks for taking the time to join our community and start contributing! - Please familiarize yourself with the [Code of Conduct][4] before contributing. - See [CONTRIBUTING.md][5] for instructions on the developer certificate of origin that we require. -- Read how [we're using ZenHub][6] for project and roadmap planning ### Pull Requests @@ -166,6 +165,5 @@ See [the list of releases][7] to find out about feature changes. [3]: http://slack.kubermatic.io/ [4]: code-of-conduct.md [5]: CONTRIBUTING.md -[6]: Zenhub.md [7]: https://github.com/kubermatic/machine-controller/releases [8]: https://docs.kubermatic.com/operatingsystemmanager diff --git a/Zenhub.md b/Zenhub.md deleted file mode 100644 index a041cb39e..000000000 --- a/Zenhub.md +++ /dev/null @@ -1,15 +0,0 @@ -# ZenHub - -As an Open Source community, it is necessary for our work, communication, and collaboration to be done in the open. -GitHub provides a central repository for code, pull requests, issues, and documentation. When applicable, we will use Google Docs for design reviews, proposals, and other working documents. - -While GitHub issues, milestones, and labels generally work pretty well, the Kubermatic team has found that product planning requires some additional tooling that GitHub projects do not offer. - -In our effort to minimize tooling while enabling product management insights, we have decided to use [ZenHub Open-Source](https://www.zenhub.com/blog/open-source/) to overlay product and project tracking on top of GitHub. -ZenHub is a GitHub application that provides Kanban visualization, Epic tracking, fine-grained prioritization, and more. It's primary backing storage system is existing GitHub issues along with additional metadata stored in ZenHub's database. - -If you are an user or Developer, you do not _need_ to use ZenHub for your regular workflow (e.g to see open bug reports or feature requests, work on pull requests). However, if you'd like to be able to visualize the high-level project goals and roadmap, you will need to use the free version of ZenHub. - -## Using ZenHub - -ZenHub can be integrated within the GitHub interface using their [Chrome or FireFox extensions](https://www.zenhub.com/extension). In addition, you can use their dedicated web application. From f2458f0ea2d803dbf7051e8ac95345817cfce66e Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 23 Oct 2023 15:29:07 +0500 Subject: [PATCH 356/489] Drop centos support for Hetzner (#1725) Signed-off-by: Waleed Malik --- docs/operating-system.md | 2 +- test/e2e/provisioning/all_e2e_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/operating-system.md b/docs/operating-system.md index 8d096a529..b223f42b1 100644 --- a/docs/operating-system.md +++ b/docs/operating-system.md @@ -11,7 +11,7 @@ | Digitalocean | ✓ | ✓ | x | x | x | ✓ | | Equinix Metal | ✓ | ✓ | ✓ | x | x | ✓ | | Google Cloud Platform | ✓ | x | x | x | x | x | -| Hetzner | ✓ | ✓ | x | x | x | ✓ | +| Hetzner | ✓ | x | x | x | x | ✓ | | KubeVirt | ✓ | ✓ | ✓ | ✓ | x | ✓ | | Nutanix | ✓ | ✓ | x | x | x | x | | Openstack | ✓ | ✓ | ✓ | ✓ | x | ✓ | diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index fc962fcf3..995b9fe89 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -704,7 +704,7 @@ func TestHetznerProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, HZ_E2E_TOKEN environment variable cannot be empty") } - selector := OsSelector("ubuntu", "centos", "rockylinux") + selector := OsSelector("ubuntu", "rockylinux") // act params := []string{fmt.Sprintf("<< HETZNER_TOKEN >>=%s", hzToken)} From 34c7d12cae1b6ab595fd3a7c55109f6c4f9eb421 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 23 Oct 2023 21:35:07 +0500 Subject: [PATCH 357/489] Configure provider-id for the machines/nodes (#1723) * Configure provider-id for the machines/nodes MC will inject provider-id for the machines that are created against cloud providers that don't have in-tree or external CCM support Signed-off-by: Waleed Malik * Add exception in golangci-lint Signed-off-by: Waleed Malik * Refactored code Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- .golangci.yml | 1 + pkg/admission/machines.go | 10 +++++ pkg/cloudprovider/provider/anexia/instance.go | 3 ++ .../provider/digitalocean/provider.go | 3 ++ .../provider/equinixmetal/provider.go | 3 ++ pkg/cloudprovider/provider/gce/instance.go | 3 ++ .../provider/hetzner/provider.go | 3 ++ .../provider/kubevirt/provider.go | 3 ++ pkg/cloudprovider/provider/linode/provider.go | 3 ++ .../provider/nutanix/provider.go | 3 ++ .../provider/opennebula/provider.go | 3 ++ .../provider/openstack/provider.go | 3 ++ .../provider/vmwareclouddirector/provider.go | 3 ++ .../provider/vsphere/provider.go | 3 ++ pkg/cloudprovider/provider/vultr/provider.go | 6 +++ pkg/controller/machine/controller.go | 40 ++++++++++++++++++- pkg/providerconfig/types/types.go | 23 +++++++++++ 17 files changed, 114 insertions(+), 2 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 222c27acb..17629a711 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -67,3 +67,4 @@ issues: - 'cyclomatic complexity 34 of func `\(\*provider\)\.getConfig` is high' - 'cyclomatic complexity 31 of func `\(\*provider\)\.Validate` is high' - 'cyclomatic complexity 33 of func `\(\*provider\)\.Create` is high' + - 'cyclomatic complexity 32 of func `\(\*Reconciler\)\.ensureInstanceExistsForMachine` is high' diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index d96b9e977..2ee451bcb 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -64,6 +64,16 @@ func (ad *admissionData) mutateMachines(ctx context.Context, ar admissionv1.Admi if oldMachine.Spec.Name != machine.Spec.Name && machine.Spec.Name == machine.Name { oldMachine.Spec.Name = machine.Spec.Name } + + if oldMachine.Spec.ProviderID != nil && machine.Spec.ProviderID != nil && *oldMachine.Spec.ProviderID != *machine.Spec.ProviderID { + return nil, fmt.Errorf("providerID is immutable") + } + + // Allow mutation of the ProviderID field, as it can only be computed after the machine is created. + if oldMachine.Spec.ProviderID == nil && machine.Spec.ProviderID != nil { + oldMachine.Spec.ProviderID = machine.Spec.ProviderID + } + // Allow mutation when: // * machine has the `MigrationBypassSpecNoModificationRequirementAnnotation` annotation (used for type migration) bypassValidationForMigration := machine.Annotations[BypassSpecNoModificationRequirementAnnotation] == "true" diff --git a/pkg/cloudprovider/provider/anexia/instance.go b/pkg/cloudprovider/provider/anexia/instance.go index d84d90f7c..0c8343b1f 100644 --- a/pkg/cloudprovider/provider/anexia/instance.go +++ b/pkg/cloudprovider/provider/anexia/instance.go @@ -51,6 +51,9 @@ func (ai *anexiaInstance) ID() string { } func (ai *anexiaInstance) ProviderID() string { + if ai == nil || ai.ID() == "" { + return "" + } return ai.ID() } diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index 928d3bc7e..649da5acc 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -506,6 +506,9 @@ func (d *doInstance) ID() string { } func (d *doInstance) ProviderID() string { + if d.droplet == nil || d.droplet.Name == "" { + return "" + } return fmt.Sprintf("digitalocean://%d", d.droplet.ID) } diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index 2832be59a..bcaf075f2 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -400,6 +400,9 @@ func (s *metalDevice) ID() string { } func (s *metalDevice) ProviderID() string { + if s.device == nil || s.device.ID == "" { + return "" + } return "equinixmetal://" + s.device.ID } diff --git a/pkg/cloudprovider/provider/gce/instance.go b/pkg/cloudprovider/provider/gce/instance.go index 1d61d4bae..2b6476195 100644 --- a/pkg/cloudprovider/provider/gce/instance.go +++ b/pkg/cloudprovider/provider/gce/instance.go @@ -61,6 +61,9 @@ func (gi *googleInstance) ID() string { } func (gi *googleInstance) ProviderID() string { + if gi.ci == nil || gi.ci.Name == "" { + return "" + } return fmt.Sprintf("gce://%s/%s/%s", gi.projectID, gi.zone, gi.ci.Name) } diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 697cdf6d4..a35d899fe 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -553,6 +553,9 @@ func (s *hetznerServer) ID() string { } func (s *hetznerServer) ProviderID() string { + if s.server == nil || s.server.ID == 0 { + return "" + } return fmt.Sprintf("hcloud://%d", s.server.ID) } diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 1f2a161c4..8dffccf6b 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -162,6 +162,9 @@ func (k *kubeVirtServer) ID() string { } func (k *kubeVirtServer) ProviderID() string { + if k.vmi.Name == "" { + return "" + } return "kubevirt://" + k.vmi.Name } diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index 5ac0a7c3b..eac81b491 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -405,6 +405,9 @@ func (d *linodeInstance) ID() string { } func (d *linodeInstance) ProviderID() string { + if d == nil || d.ID() == "" { + return "" + } return fmt.Sprintf("linode://%s", d.ID()) } diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 699c778d9..cdf112739 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -89,6 +89,9 @@ func (nutanixServer Server) ID() string { } func (nutanixServer Server) ProviderID() string { + if nutanixServer.ID() == "" { + return "" + } return fmt.Sprintf("nutanix://%s", nutanixServer.ID()) } diff --git a/pkg/cloudprovider/provider/opennebula/provider.go b/pkg/cloudprovider/provider/opennebula/provider.go index 728ca9e63..1a3f442fd 100644 --- a/pkg/cloudprovider/provider/opennebula/provider.go +++ b/pkg/cloudprovider/provider/opennebula/provider.go @@ -426,6 +426,9 @@ func (i *openNebulaInstance) ID() string { } func (i *openNebulaInstance) ProviderID() string { + if i.vm == nil || i.vm.ID == 0 { + return "" + } return "opennebula://" + strconv.Itoa(i.vm.ID) } diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 8ff4e9dc7..a5b24ef8f 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -936,6 +936,9 @@ func (d *osInstance) ID() string { } func (d *osInstance) ProviderID() string { + if d.server == nil || d.server.ID == "" { + return "" + } return "openstack:///" + d.server.ID } diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go index c1136b79e..325fda982 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -123,6 +123,9 @@ func (s Server) ID() string { } func (s Server) ProviderID() string { + if s.ID() == "" { + return "" + } return fmt.Sprintf("vmware-cloud-director://%s", s.ID()) } diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 30e751f03..7a13cd43f 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -101,6 +101,9 @@ func (vsphereServer Server) ID() string { } func (vsphereServer Server) ProviderID() string { + if vsphereServer.uuid == "" { + return "" + } return "vsphere://" + vsphereServer.uuid } diff --git a/pkg/cloudprovider/provider/vultr/provider.go b/pkg/cloudprovider/provider/vultr/provider.go index 8e074e21e..fcad5b23b 100644 --- a/pkg/cloudprovider/provider/vultr/provider.go +++ b/pkg/cloudprovider/provider/vultr/provider.go @@ -584,9 +584,15 @@ func (v *vultrPhysicalMachine) ID() string { } func (v *vultrVirtualMachine) ProviderID() string { + if v.instance == nil || v.instance.ID == "" { + return "" + } return "vultr://" + v.instance.ID } func (v *vultrPhysicalMachine) ProviderID() string { + if v.instance == nil || v.instance.ID == "" { + return "" + } return "vultr://" + v.instance.ID } diff --git a/pkg/controller/machine/controller.go b/pkg/controller/machine/controller.go index abc092a0b..0a0b1d301 100644 --- a/pkg/controller/machine/controller.go +++ b/pkg/controller/machine/controller.go @@ -98,6 +98,9 @@ const ( // AnnotationAutoscalerIdentifier is used by the cluster-autoscaler // cluster-api provider to match Nodes to Machines. AnnotationAutoscalerIdentifier = "cluster.k8s.io/machine" + + // ProviderID pattern. + ProviderIDPattern = "kubermatic://%s/%s" ) // Reconciler is the controller implementation for machine resources. @@ -481,7 +484,24 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, mach return r.ensureInstanceExistsForMachine(ctx, log, prov, machine, userdataPlugin, providerConfig) } - // case 3.3: if the node exists make sure if it has labels and taints attached to it. + // case 3.3: if the node exists and both external and internal CCM are not available. Then set the provider-id for the node. + inTree, err := providerconfigtypes.IntreeCloudProviderImplementationSupported(providerConfig.CloudProvider, machine.Spec.Versions.Kubelet) + if err != nil { + return nil, fmt.Errorf("failed to check if cloud provider %q has in-tree implementation: %w", providerConfig.CloudProvider, err) + } + + if !inTree && !r.nodeSettings.ExternalCloudProvider && node.Spec.ProviderID == "" { + providerID := fmt.Sprintf(ProviderIDPattern, providerConfig.CloudProvider, machine.UID) + if err := r.updateNode(ctx, node, func(n *corev1.Node) { + n.Spec.ProviderID = providerID + }); err != nil { + return nil, fmt.Errorf("failed to update node %s with the ProviderID: %w", node.Name, err) + } + + r.recorder.Event(machine, corev1.EventTypeNormal, "ProviderIDUpdated", "Successfully updated providerID on node") + nodeLog.Info("Added ProviderID to the node") + } + // case 3.4: if the node exists make sure if it has labels and taints attached to it. return nil, r.ensureNodeLabelsAnnotationsAndTaints(ctx, nodeLog, node, machine) } @@ -981,10 +1001,26 @@ func (r *Reconciler) ensureInstanceExistsForMachine( return a.Type < b.Type }) + var providerID string + if machine.Spec.ProviderID == nil { + inTree, err := providerconfigtypes.IntreeCloudProviderImplementationSupported(providerConfig.CloudProvider, machine.Spec.Versions.Kubelet) + if err != nil { + return nil, fmt.Errorf("failed to check if cloud provider %q has in-tree implementation: %w", providerConfig.CloudProvider, err) + } + + // If both external and internal CCM are not available. We set provider-id for the machine explicitly. + if !inTree && !r.nodeSettings.ExternalCloudProvider { + providerID = fmt.Sprintf(ProviderIDPattern, providerConfig.CloudProvider, machine.UID) + } + } + if err := r.updateMachine(machine, func(m *clusterv1alpha1.Machine) { m.Status.Addresses = machineAddresses + if providerID != "" { + m.Spec.ProviderID = &providerID + } }); err != nil { - return nil, fmt.Errorf("failed to update machine after setting .status.addresses: %w", err) + return nil, fmt.Errorf("failed to update machine after setting .status.addresses and providerID: %w", err) } return r.ensureNodeOwnerRef(ctx, log, providerInstance, machine, providerConfig) diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 18bd2010d..046f4cd9f 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -23,6 +23,8 @@ import ( "fmt" "strconv" + "github.com/Masterminds/semver/v3" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/jsonutil" @@ -107,6 +109,27 @@ var ( } ) +func IntreeCloudProviderImplementationSupported(cloudProvider CloudProvider, version string) (inTree bool, err error) { + kubeletVer, err := semver.NewVersion(version) + if err != nil { + return false, fmt.Errorf("failed to parse kubelet version: %w", err) + } + + switch cloudProvider { + case CloudProviderAzure, CloudProviderVsphere, CloudProviderGoogle: + return true, nil + case CloudProviderAWS: + // In-tree AWS support was removed in Kubernetes 1.27. + ltKube127Condition, _ := semver.NewConstraint("< 1.27") + if ltKube127Condition.Check(kubeletVer) { + return true, nil + } + return false, nil + default: + return false, nil + } +} + // DNSConfig contains a machine's DNS configuration. type DNSConfig struct { Servers []string `json:"servers"` From e2c9f78673181a1b27f6be50c51d457b31de3c19 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Oct 2023 08:59:09 +0200 Subject: [PATCH 358/489] Bump google.golang.org/grpc from 1.57.0 to 1.57.1 (#1727) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.57.0 to 1.57.1. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.57.0...v1.57.1) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 395a89c41..d97cd4e89 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( golang.org/x/oauth2 v0.11.0 gomodules.xyz/jsonpatch/v2 v2.4.0 google.golang.org/api v0.137.0 - google.golang.org/grpc v1.57.0 + google.golang.org/grpc v1.57.1 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.28.0 diff --git a/go.sum b/go.sum index ee14e56a6..b7b263da8 100644 --- a/go.sum +++ b/go.sum @@ -1209,8 +1209,8 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= +google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc/examples v0.0.0-20210728214646-ad0a2a847cdf/go.mod h1:bF8wuZSAZTcbF7ZPKrDI/qY52toTP/yxLpRRY4Eu9Js= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 1ab32474920b6d995094a3775de6a1aa003be452 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 30 Oct 2023 15:32:14 +0500 Subject: [PATCH 359/489] Upgrade to Go 1.21.3 (#1728) * Upgrade to go 1.21.3 Signed-off-by: Waleed Malik * Remoev deprecated linters Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- .golangci.yml | 3 --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 16 ++++++++-------- .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 8 ++++---- .prow/verify.yaml | 8 ++++---- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- 21 files changed, 40 insertions(+), 43 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 17629a711..ea70b8697 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -11,7 +11,6 @@ linters: - asciicheck - bidichk - bodyclose - - deadcode - depguard - durationcheck - errcheck @@ -35,11 +34,9 @@ linters: - promlinter - revive - staticcheck - - structcheck - tenv - unconvert - unused - - varcheck - wastedassign - whitespace disable-all: true diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 1974d28be..202605728 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -119,7 +119,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 8e7df95ec..dfd7fc12c 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index ec5a62fe7..a6942daf6 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 926dd2e5c..b57d81f39 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 511fa1636..ee89f3037 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -156,7 +156,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -187,7 +187,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -218,7 +218,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -249,7 +249,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index d93235398..dcbb400ae 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index a40f42393..be4145ff2 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index dca573aa3..fae3c5e7d 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 03565e3c7..f09cb7cca 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 3ca308140..23e7079f1 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 3f8ced0c8..e77547e49 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index f8bccf074..b9a2817a1 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 63f8059c6..d4412e694 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 395304a3b..59a72b046 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 789eeee80..58f98244f 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 7ef994489..16a89ecb0 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 748d199a2..f3be1bd70 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index b29c74dda..da46a158e 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.21.0 + - image: golang:1.21.3 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.21.0 + - image: golang:1.21.3 command: - make args: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.21.0 + - image: golang:1.21.3 command: - make args: diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 2147af2b0..eb150e4b3 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.21.0 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.21.3 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index cc8787f18..6a43c150e 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-1 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 containerize ./hack/verify-licenses.sh go mod vendor From 1a01cb23895a91a41a6e428c399c901a6c925a35 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Nov 2023 12:23:37 +0100 Subject: [PATCH 360/489] Bump go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc (#1731) Bumps [go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc](https://github.com/open-telemetry/opentelemetry-go-contrib) from 0.42.0 to 0.46.0. - [Release notes](https://github.com/open-telemetry/opentelemetry-go-contrib/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go-contrib/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-go-contrib/compare/zpages/v0.42.0...zpages/v0.46.0) --- updated-dependencies: - dependency-name: go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 22 +++++++++++----------- go.sum | 43 +++++++++++++++++++++++-------------------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index d97cd4e89..5419a8e34 100644 --- a/go.mod +++ b/go.mod @@ -22,10 +22,10 @@ require ( github.com/davecgh/go-spew v1.1.1 github.com/digitalocean/godo v1.102.0 github.com/flatcar/container-linux-config-transpiler v0.9.4 - github.com/go-logr/logr v1.2.4 + github.com/go-logr/logr v1.3.0 github.com/go-logr/zapr v1.2.4 github.com/go-test/deep v1.0.8 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.3.1 github.com/gophercloud/gophercloud v1.5.0 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb github.com/hetznercloud/hcloud-go v1.39.0 @@ -50,7 +50,7 @@ require ( golang.org/x/oauth2 v0.11.0 gomodules.xyz/jsonpatch/v2 v2.4.0 google.golang.org/api v0.137.0 - google.golang.org/grpc v1.57.1 + google.golang.org/grpc v1.59.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.28.0 @@ -116,7 +116,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect @@ -160,10 +160,10 @@ require ( github.com/spf13/cobra v1.7.0 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 // indirect - go.opentelemetry.io/otel v1.16.0 // indirect - go.opentelemetry.io/otel/metric v1.16.0 // indirect - go.opentelemetry.io/otel/trace v1.16.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect + go.opentelemetry.io/otel v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.20.0 // indirect + go.opentelemetry.io/otel/trace v1.20.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect @@ -175,9 +175,9 @@ require ( golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.3 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230815205213-6bfd019c3878 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 // indirect + google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index b7b263da8..5277aa503 100644 --- a/go.sum +++ b/go.sum @@ -228,7 +228,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v0.10.1 h1:c0g45+xCJhdgFGw7a5QAfdS4byAbud7miNWJ1WwEVf8= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -270,8 +270,9 @@ github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= @@ -375,8 +376,9 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -406,8 +408,9 @@ github.com/google/s2a-go v0.1.5/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkj github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -750,16 +753,16 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0 h1:ZOLJc06r4CB42laIXg/7udr0pbZyuAihN10A/XuiQRY= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.0/go.mod h1:5z+/ZWJQKXa9YT34fQNx5K8Hd1EoIhvtUygUQPqEOgQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 h1:PzIubN4/sjByhDRHLviCjJuweBXWFZWhghjg7cS28+M= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0/go.mod h1:Ct6zzQEuGK3WpJs2n4dn+wfJYzd/+hNnxMRTWjGn30M= go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= -go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= -go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= -go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= -go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= +go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= +go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= -go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= -go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= +go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -1180,12 +1183,12 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230815205213-6bfd019c3878 h1:Iveh6tGCJkHAjJgEqUQYGDGgbwmhjoAOz8kO/ajxefY= -google.golang.org/genproto v0.0.0-20230815205213-6bfd019c3878/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= -google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878 h1:WGq4lvB/mlicysM/dUT3SBvijH4D3sm/Ny1A4wmt2CI= -google.golang.org/genproto/googleapis/api v0.0.0-20230815205213-6bfd019c3878/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 h1:lv6/DhyiFFGsmzxbsUUTOkN29II+zeWHxvT8Lpdxsv0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= +google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= +google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1209,8 +1212,8 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.57.1 h1:upNTNqv0ES+2ZOOqACwVtS3Il8M12/+Hz41RCPzAjQg= -google.golang.org/grpc v1.57.1/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/grpc/examples v0.0.0-20210728214646-ad0a2a847cdf/go.mod h1:bF8wuZSAZTcbF7ZPKrDI/qY52toTP/yxLpRRY4Eu9Js= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 953e653fdb9b3c33890b1990b884491b57968983 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Thu, 23 Nov 2023 16:49:40 +0100 Subject: [PATCH 361/489] aws: fix rocky linux description filter (#1733) Signed-off-by: Jan Wozniak --- pkg/cloudprovider/provider/aws/provider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 442d95585..1320f6147 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -115,12 +115,12 @@ var ( }, providerconfigtypes.OperatingSystemRockyLinux: { awstypes.CPUArchitectureX86_64: { - description: "Rocky-8-ec2-8*.x86_64", + description: "*Rocky-8-ec2-8*.x86_64", // The AWS marketplace ID from Rocky Linux Community Platform Engineering (CPE) owner: "792107900819", }, awstypes.CPUArchitectureARM64: { - description: "Rocky-8-ec2-8*.aarch64", + description: "*Rocky-8-ec2-8*.aarch64", // The AWS marketplace ID from Rocky Linux Community Platform Engineering (CPE) owner: "792107900819", }, From 22c0e36d6fbd6d4367dc8f2d538661f3495683c2 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 27 Nov 2023 16:54:45 +0500 Subject: [PATCH 362/489] Migrate to dl.k8s.io from kubernetes release bucket (#1737) Signed-off-by: Waleed Malik --- image-builder/download_kubernetes.sh | 4 +- pkg/cloudprovider/provider/vsphere/helper.go | 3 +- pkg/node/eviction/eviction.go | 4 +- .../amzn2/testdata/kubelet-v1.26-aws.yaml | 2 +- .../kubelet-v1.26.6-aws-external.yaml | 2 +- .../amzn2/testdata/kubelet-v1.26.6-aws.yaml | 2 +- .../kubelet-v1.26.6-vsphere-mirrors.yaml | 2 +- .../kubelet-v1.26.6-vsphere-proxy.yaml | 2 +- .../testdata/kubelet-v1.26.6-vsphere.yaml | 2 +- .../amzn2/testdata/kubelet-v1.27-aws.yaml | 2 +- .../amzn2/testdata/kubelet-v1.28-aws.yaml | 2 +- .../centos/testdata/kubelet-v1.26-aws.yaml | 892 +++++++++--------- .../kubelet-v1.26.6-aws-external.yaml | 2 +- .../centos/testdata/kubelet-v1.26.6-aws.yaml | 2 +- .../testdata/kubelet-v1.26.6-nutanix.yaml | 2 +- .../kubelet-v1.26.6-vsphere-mirrors.yaml | 2 +- .../kubelet-v1.26.6-vsphere-proxy.yaml | 2 +- .../testdata/kubelet-v1.26.6-vsphere.yaml | 2 +- .../centos/testdata/kubelet-v1.27-aws.yaml | 2 +- .../centos/testdata/kubelet-v1.28-aws.yaml | 2 +- .../flatcar/testdata/cloud-init_v1.26.6.yaml | 2 +- .../flatcar/testdata/cloud-init_v1.28.0.yaml | 2 +- pkg/userdata/flatcar/testdata/containerd.yaml | 2 +- .../flatcar/testdata/ignition_v1.26.6.json | 2 +- .../flatcar/testdata/ignition_v1.28.0.json | 2 +- .../helper/download_binaries_script.go | 4 +- .../helper/download_binaries_script_test.go | 6 +- .../testdata/download_binaries_v1.26.9.golden | 2 +- .../testdata/download_binaries_v1.27.6.golden | 2 +- .../testdata/download_binaries_v1.28.2.golden | 2 +- .../safe_download_binaries_v1.26.6.golden | 2 +- .../testdata/kubelet-v1.26-aws-external.yaml | 2 +- .../rhel/testdata/kubelet-v1.26-aws.yaml | 2 +- .../kubelet-v1.26.6-aws-external.yaml | 2 +- .../rhel/testdata/kubelet-v1.26.6-aws.yaml | 2 +- .../kubelet-v1.26.6-vsphere-mirrors.yaml | 2 +- .../kubelet-v1.26.6-vsphere-proxy.yaml | 2 +- .../testdata/kubelet-v1.26.6-vsphere.yaml | 2 +- .../rhel/testdata/kubelet-v1.28-aws.yaml | 2 +- .../rhel/testdata/kubelet-v1.28-nutanix.yaml | 2 +- .../rhel/testdata/pod-cidr-azure-rhel.yaml | 2 +- .../kubelet-v1.26.6-aws-external.yaml | 2 +- .../testdata/kubelet-v1.26.6-aws.yaml | 2 +- .../testdata/kubelet-v1.26.6-nutanix.yaml | 2 +- .../kubelet-v1.26.6-vsphere-mirrors.yaml | 2 +- .../kubelet-v1.26.6-vsphere-proxy.yaml | 2 +- .../testdata/kubelet-v1.26.6-vsphere.yaml | 2 +- .../testdata/kubelet-v1.28-aws.yaml | 2 +- pkg/userdata/ubuntu/testdata/containerd.yaml | 2 +- .../digitalocean-dualstack-IPv6+IPv4.yaml | 2 +- .../testdata/digitalocean-dualstack.yaml | 2 +- .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 2 +- pkg/userdata/ubuntu/testdata/docker.yaml | 2 +- .../kubelet-version-without-v-prefix.yaml | 2 +- .../ubuntu/testdata/multiple-dns-servers.yaml | 2 +- .../ubuntu/testdata/multiple-ssh-keys.yaml | 2 +- pkg/userdata/ubuntu/testdata/nutanix.yaml | 2 +- .../openstack-dualstack-IPv6+IPv4.yaml | 2 +- .../ubuntu/testdata/openstack-dualstack.yaml | 2 +- .../openstack-overwrite-cloud-config.yaml | 2 +- pkg/userdata/ubuntu/testdata/openstack.yaml | 2 +- .../ubuntu/testdata/version-1.26.9.yaml | 2 +- .../ubuntu/testdata/version-1.27.6.yaml | 2 +- .../ubuntu/testdata/version-1.28.2.yaml | 2 +- .../ubuntu/testdata/vsphere-mirrors.yaml | 2 +- .../ubuntu/testdata/vsphere-proxy.yaml | 2 +- pkg/userdata/ubuntu/testdata/vsphere.yaml | 2 +- 67 files changed, 512 insertions(+), 523 deletions(-) diff --git a/image-builder/download_kubernetes.sh b/image-builder/download_kubernetes.sh index 14fc74e7a..0b5cdd042 100755 --- a/image-builder/download_kubernetes.sh +++ b/image-builder/download_kubernetes.sh @@ -46,7 +46,7 @@ else echo " * Using version $K8S_RELEASE" fi -wget --quiet https://storage.googleapis.com/kubernetes-release/release/$K8S_RELEASE/bin/linux/amd64/{kubeadm,kubelet,kubectl}.sha1 -P "$TEMPDIR" +wget --quiet https://dl.k8s.io/$K8S_RELEASE/bin/linux/amd64/{kubeadm,kubelet,kubectl}.sha1 -P "$TEMPDIR" for util in kubeadm kubelet kubectl; do echo " * $util" @@ -59,7 +59,7 @@ for util in kubeadm kubelet kubectl; do exit 1 fi else - wget "https://storage.googleapis.com/kubernetes-release/release/$K8S_RELEASE/bin/linux/amd64/$util" -P "$TEMPDIR" + wget "https://dl.k8s.io/$K8S_RELEASE/bin/linux/amd64/$util" -P "$TEMPDIR" CALCULATED_SHA1="$(sha1sum "$TEMPDIR/$util" | cut -f1 -d ' ')" EXPECTED_SHA1="$(<"$TEMPDIR/$util.sha1")" diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go index 9680c4d80..522e77ff9 100644 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ b/pkg/cloudprovider/provider/vsphere/helper.go @@ -282,9 +282,8 @@ func resolveDatastoreRef(ctx context.Context, log *zap.SugaredLogger, config *Co return nil, fmt.Errorf("failed to get datastore: %w", err) } return types.NewReference(datastore.Reference()), nil - } else { - return nil, fmt.Errorf("please provide either a datastore or a datastore cluster") } + return nil, fmt.Errorf("please provide either a datastore or a datastore cluster") } func uploadAndAttachISO(ctx context.Context, log *zap.SugaredLogger, session *Session, vmRef *object.VirtualMachine, localIsoFilePath string) error { diff --git a/pkg/node/eviction/eviction.go b/pkg/node/eviction/eviction.go index e51bd1c7d..9c2b6f1ec 100644 --- a/pkg/node/eviction/eviction.go +++ b/pkg/node/eviction/eviction.go @@ -140,10 +140,8 @@ func (ne *NodeEviction) evictPods(ctx context.Context, log *zap.SugaredLogger, p } else if kerrors.IsTooManyRequests(err) { // PDB prevents eviction, return and make the controller retry later return - } else { - errCh <- fmt.Errorf("error evicting pod %s/%s on node %s: %w", p.Namespace, p.Name, ne.nodeName, err) - return } + errCh <- fmt.Errorf("error evicting pod %s/%s on node %s: %w", p.Namespace, p.Name, ne.nodeName, err) } }(pod) } diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml index e5110eda1..8cc1660ba 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml @@ -145,7 +145,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml index 717e1177f..aeb983125 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml @@ -145,7 +145,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml index 2e982a519..d32468fcb 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml @@ -145,7 +145,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index ab541bf78..96b3a92a6 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -158,7 +158,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index a3eec0840..ccee1c339 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -158,7 +158,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml index 1e0a19220..c974e8043 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml @@ -150,7 +150,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml index 79908237b..587763ace 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml @@ -145,7 +145,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml index 2c1bcbefc..bbcad8969 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml @@ -145,7 +145,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml index 32442cf78..0ca4f01cc 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml @@ -1,458 +1,448 @@ #cloud-config - ssh_pwauth: false write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - + - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + - path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + - path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + - path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + + - path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + + - path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + + - path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + + - path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + - path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + + - path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + - path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + - path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + - path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + - path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + + - path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + + - path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + - path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target runcmd: -- systemctl enable --now setup.service + - systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml index bc0f71498..1ad244638 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml @@ -151,7 +151,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml index 7487319ef..e12e0a730 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml @@ -151,7 +151,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml index fc0613d07..752862e7a 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml @@ -158,7 +158,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index ced36dba2..5dbae2241 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -164,7 +164,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index 38cc9a7a5..0032322de 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -164,7 +164,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml index 97591d4bf..cdac24c72 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml @@ -156,7 +156,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml index 9d3f95c33..17cc1663b 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml @@ -151,7 +151,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml index 99f15e731..984a3ab78 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml @@ -151,7 +151,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml index 09ef75cc4..8d8c941e1 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml @@ -433,7 +433,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml index 7f944534d..fdc2dc0c1 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml @@ -433,7 +433,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index 9bf3a029f..3ca71c792 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -412,7 +412,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.26.6.json b/pkg/userdata/flatcar/testdata/ignition_v1.26.6.json index c548068fd..ba1218586 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.26.6.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.26.6.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.26.6%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.26.6%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.28.0.json b/pkg/userdata/flatcar/testdata/ignition_v1.28.0.json index 141859dae..21b63882e 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.28.0.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.28.0.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.28.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fstorage.googleapis.com%2Fkubernetes-release%2Frelease%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.28.0%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20%5C%24%7BTORCX_BINDIR%7D%2Fcontainerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/download_binaries_script.go b/pkg/userdata/helper/download_binaries_script.go index 8495c450b..6a2ec7a86 100644 --- a/pkg/userdata/helper/download_binaries_script.go +++ b/pkg/userdata/helper/download_binaries_script.go @@ -98,7 +98,7 @@ cd - {{- /* kubelet */}} KUBE_VERSION="${KUBE_VERSION:-{{ .KubeVersion }}}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" -kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" +kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" {{- /* create versioned kube dir */}} @@ -146,7 +146,7 @@ fi {{- if .DownloadKubelet }} {{- /* kubelet */}} if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v{{ .KubeletVersion }}/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v{{ .KubeletVersion }}/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi {{- end }} diff --git a/pkg/userdata/helper/download_binaries_script_test.go b/pkg/userdata/helper/download_binaries_script_test.go index ec71332b1..786e153ac 100644 --- a/pkg/userdata/helper/download_binaries_script_test.go +++ b/pkg/userdata/helper/download_binaries_script_test.go @@ -25,6 +25,8 @@ import ( "github.com/kubermatic/machine-controller/pkg/test" ) +const goldenExtension = ".golden" + func TestDownloadBinariesScript(t *testing.T) { for _, version := range versions { name := fmt.Sprintf("download_binaries_%s", version.Original()) @@ -33,7 +35,7 @@ func TestDownloadBinariesScript(t *testing.T) { if err != nil { t.Error(err) } - goldenName := name + ".golden" + goldenName := name + goldenExtension test.CompareOutput(t, goldenName, script, *update) }) } @@ -46,7 +48,7 @@ func TestSafeDownloadBinariesScript(t *testing.T) { if err != nil { t.Error(err) } - goldenName := name + ".golden" + goldenName := name + goldenExtension test.CompareOutput(t, goldenName, script, *update) }) } diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.26.9.golden b/pkg/userdata/helper/testdata/download_binaries_v1.26.9.golden index 2982cbae4..9a1ba6122 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.26.9.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.26.9.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.26.9/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.26.9/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.27.6.golden b/pkg/userdata/helper/testdata/download_binaries_v1.27.6.golden index 87010dae6..710598b6f 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.27.6.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.27.6.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.27.6/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.27.6/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.28.2.golden b/pkg/userdata/helper/testdata/download_binaries_v1.28.2.golden index 4add0f3ee..0a7cbba30 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.28.2.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.28.2.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://storage.googleapis.com/kubernetes-release/release/v1.28.2/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.28.2/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/safe_download_binaries_v1.26.6.golden b/pkg/userdata/helper/testdata/safe_download_binaries_v1.26.6.golden index fe3d167a5..7a61bdb36 100644 --- a/pkg/userdata/helper/testdata/safe_download_binaries_v1.26.6.golden +++ b/pkg/userdata/helper/testdata/safe_download_binaries_v1.26.6.golden @@ -42,7 +42,7 @@ ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipp cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" -kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" +kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml index 72b62b869..69a878243 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml @@ -145,7 +145,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml index 9dc8b8f8b..c28ceeee6 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml @@ -145,7 +145,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml index fb380ffbc..08a794776 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml @@ -145,7 +145,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml index f5feeec76..bdceecdc2 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml @@ -145,7 +145,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index 462b3ad8b..714a578f9 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -159,7 +159,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index 13fb138d9..9aec6e03f 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -159,7 +159,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml index 3eb848072..9960e050d 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml @@ -151,7 +151,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.28-aws.yaml index 07b8bddcf..6f2841e37 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.28-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.28-aws.yaml @@ -145,7 +145,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml index 1ab290bec..4bb017d54 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml @@ -153,7 +153,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index 414032b72..cde79dbce 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -150,7 +150,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml index d83e716ad..f86374982 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml @@ -146,7 +146,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml index bdf75ea35..1b28cdae9 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml @@ -146,7 +146,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml index ceb90c26a..cc3360212 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml @@ -153,7 +153,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index 65183b731..9c5477924 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -159,7 +159,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index 1eb81af3b..11a34ded7 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -159,7 +159,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml index e57136f96..e306a464c 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml @@ -151,7 +151,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml index 694e86538..8d243a220 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml @@ -146,7 +146,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index b3e80ca68..fb5eaab31 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -154,7 +154,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml index 5d7a20718..8013cb5c2 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml index ba3784b47..0e2b57630 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index 62ccbd59b..e3e4c794f 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -154,7 +154,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml index d713090a7..1a9628695 100644 --- a/pkg/userdata/ubuntu/testdata/docker.yaml +++ b/pkg/userdata/ubuntu/testdata/docker.yaml @@ -154,7 +154,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index a38eb1ab6..0dff8169e 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 76150fed8..9330fa523 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 7de5f9f0e..0a01a0378 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -154,7 +154,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 1c17c1acf..6a148f382 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -155,7 +155,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml index 2f6334f43..57b85c21e 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml index 97304a73e..a38bfa3b8 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index a93d52ee9..faa42aaaa 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 486ff34da..934dbbf6c 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/version-1.26.9.yaml b/pkg/userdata/ubuntu/testdata/version-1.26.9.yaml index c6bdf99ff..f35f5be94 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.26.9.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.26.9.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.26.9}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/version-1.27.6.yaml b/pkg/userdata/ubuntu/testdata/version-1.27.6.yaml index 09bb8e03c..b89ceb594 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.27.6.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.27.6.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/version-1.28.2.yaml b/pkg/userdata/ubuntu/testdata/version-1.28.2.yaml index 5680f5def..3e379099f 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.28.2.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.28.2.yaml @@ -152,7 +152,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.28.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index 7318b6140..13fdbb121 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -162,7 +162,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 4d5feb72a..6f660c644 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -162,7 +162,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 6dd690532..8f34e2980 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -153,7 +153,7 @@ write_files: cd - KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" mkdir -p "$kube_dir" : >"$kube_sum_file" From c162be2e68129994223ef9dd6467390be68c6516 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 29 Nov 2023 15:30:48 +0500 Subject: [PATCH 363/489] Flatcar Support for GCE (#1739) * Flatcar Support for GCE Signed-off-by: Waleed Malik * Update compatibility matrix Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- docs/operating-system.md | 2 +- pkg/cloudprovider/provider/gce/config.go | 6 ++++-- test/e2e/provisioning/all_e2e_test.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/operating-system.md b/docs/operating-system.md index b223f42b1..fd6994ece 100644 --- a/docs/operating-system.md +++ b/docs/operating-system.md @@ -10,7 +10,7 @@ | Azure | ✓ | ✓ | ✓ | ✓ | x | ✓ | | Digitalocean | ✓ | ✓ | x | x | x | ✓ | | Equinix Metal | ✓ | ✓ | ✓ | x | x | ✓ | -| Google Cloud Platform | ✓ | x | x | x | x | x | +| Google Cloud Platform | ✓ | x | ✓ | x | x | x | | Hetzner | ✓ | x | x | x | x | ✓ | | KubeVirt | ✓ | ✓ | ✓ | ✓ | x | ✓ | | Nutanix | ✓ | ✓ | x | x | x | x | diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index d47d16635..0ba7252db 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -45,12 +45,14 @@ const ( // imageProjects maps the OS to the Google Cloud image projects. var imageProjects = map[providerconfigtypes.OperatingSystem]string{ - providerconfigtypes.OperatingSystemUbuntu: "ubuntu-os-cloud", + providerconfigtypes.OperatingSystemUbuntu: "ubuntu-os-cloud", + providerconfigtypes.OperatingSystemFlatcar: "kinvolk-public", } // imageFamilies maps the OS to the Google Cloud image projects. var imageFamilies = map[providerconfigtypes.OperatingSystem]string{ - providerconfigtypes.OperatingSystemUbuntu: "ubuntu-2204-lts", + providerconfigtypes.OperatingSystemUbuntu: "ubuntu-2204-lts", + providerconfigtypes.OperatingSystemFlatcar: "flatcar-stable", } // diskTypes are the disk types of the Google Cloud. Map is used for diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 995b9fe89..0fad8c668 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -685,7 +685,7 @@ func TestGCEProvisioningE2E(t *testing.T) { } // Act. GCE does not support CentOS. - selector := OsSelector("ubuntu") + selector := OsSelector("ubuntu", "flatcar") params := []string{ fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>=%s", safeBase64Encoding(googleServiceAccount)), } From 4eb097a4635227836cdf9e40cc157ff526afd610 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Wed, 6 Dec 2023 13:15:00 +0100 Subject: [PATCH 364/489] Use JoinPath to construct vSphere url (#1738) Signed-off-by: Marvin Beckers --- pkg/cloudprovider/provider/vsphere/client.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/vsphere/client.go b/pkg/cloudprovider/provider/vsphere/client.go index f89f9c0e0..4a2e688eb 100644 --- a/pkg/cloudprovider/provider/vsphere/client.go +++ b/pkg/cloudprovider/provider/vsphere/client.go @@ -109,11 +109,13 @@ func (s *RESTSession) Logout(ctx context.Context) { } func createVim25Client(ctx context.Context, config *Config) (*vim25.Client, error) { - clientURL, err := url.Parse(fmt.Sprintf("%s/sdk", config.VSphereURL)) + endpointURL, err := url.Parse(config.VSphereURL) if err != nil { return nil, err } + clientURL := endpointURL.JoinPath("/sdk") + // creating the govmoni Client in roundabout way because we need to set the proper CA bundle: reference https://github.com/vmware/govmomi/issues/1200 soapClient := soap.NewClient(clientURL, config.AllowInsecure) // set our CA bundle From 5f5c471434368967a0e5e3cfa2a28a73a9f1064e Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Tue, 12 Dec 2023 12:40:06 +0100 Subject: [PATCH 365/489] Update Prow jobs to build image with Go 1.21.5 (#1746) * Update Prow jobs to build image with Go 1.21.5 Signed-off-by: Marvin Beckers * Also update golang images Signed-off-by: Marvin Beckers * Also update Dockerfile and Makefile Signed-off-by: Marvin Beckers * Update OSM flags Signed-off-by: Marvin Beckers * Use fixed build image Signed-off-by: Marvin Beckers * ??? Signed-off-by: Marvin Beckers --------- Signed-off-by: Marvin Beckers --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 16 ++++++++-------- .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 8 ++++---- .prow/verify.yaml | 8 ++++---- Dockerfile | 2 +- Makefile | 2 +- examples/operating-system-manager.yaml | 4 ++-- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- 23 files changed, 44 insertions(+), 44 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 202605728..fe59ab740 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -119,7 +119,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index dfd7fc12c..68d162289 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index a6942daf6..d3462ad51 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index b57d81f39..2c6896ceb 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index ee89f3037..9adfb1063 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -156,7 +156,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -187,7 +187,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -218,7 +218,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -249,7 +249,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index dcbb400ae..a055a5b3f 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index be4145ff2..393ef95fd 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index fae3c5e7d..0e4888a7a 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index f09cb7cca..592af7d08 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 23e7079f1..204cfba35 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index e77547e49..a6f1204ce 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index b9a2817a1..dc900ef89 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index d4412e694..bf40ac5a2 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 59a72b046..c32083ad7 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 58f98244f..42eb04cd7 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 16a89ecb0..e0abf8433 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index f3be1bd70..8af222d88 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index da46a158e..261ba1c32 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.21.3 + - image: golang:1.21.5 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.21.3 + - image: golang:1.21.5 command: - make args: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 + - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.21.3 + - image: golang:1.21.5 command: - make args: diff --git a/Dockerfile b/Dockerfile index 04ad940d3..eb4168adb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.21.0 +ARG GO_VERSION=1.21.5 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index 1697eb852..82350088c 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.21.0 +GO_VERSION ?= 1.21.5 GOOS ?= $(shell go env GOOS) diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index 7fc7a471e..ee052ae5c 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -995,8 +995,8 @@ spec: name: webhook command: - /usr/local/bin/webhook - - -logtostderr - - -v=4 + - -log-debug=false + - -log-format=json - -namespace=kube-system volumeMounts: - name: operating-system-manager-admission-cert diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index eb150e4b3..0afcaa361 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.21.3 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.21.5 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 6a43c150e..cb13ab003 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-6 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 containerize ./hack/verify-licenses.sh go mod vendor From ed3adcca3951ab30ed51c3a9ce7e3847a29a8303 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Wed, 13 Dec 2023 13:10:08 +0100 Subject: [PATCH 366/489] Deprecate sig-virtualization (#1749) with merge of sig-virtualization into sig-cluster-management, there is no need for standalone OWNERS hierarchy for kubevirt Signed-off-by: Jan Wozniak --- OWNERS_ALIASES | 3 --- pkg/cloudprovider/provider/kubevirt/OWNERS | 13 ------------- 2 files changed, 16 deletions(-) delete mode 100644 pkg/cloudprovider/provider/kubevirt/OWNERS diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 21494f318..dc87eeb2a 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -10,6 +10,3 @@ aliases: - moelsayed - xmudrii - xrstf - sig-virtualization: - - cnvergence - - wozniakjan diff --git a/pkg/cloudprovider/provider/kubevirt/OWNERS b/pkg/cloudprovider/provider/kubevirt/OWNERS deleted file mode 100644 index ac28b34e2..000000000 --- a/pkg/cloudprovider/provider/kubevirt/OWNERS +++ /dev/null @@ -1,13 +0,0 @@ -# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md - -approvers: - - sig-virtualization - -reviewers: - - sig-virtualization - -labels: - - sig/virtualization - -options: - no_parent_owners: true From 8f003c39c6c9b7f31a8f79aacc2a999e5487d961 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Thu, 14 Dec 2023 14:16:08 +0100 Subject: [PATCH 367/489] kubevirt: allow setting storage volume access types (#1740) this is important for live migration because only RWX volumes can be live migrated Signed-off-by: Jan Wozniak --- .../provider/kubevirt/provider.go | 28 +++++++++++++------ .../provider/kubevirt/types/types.go | 5 ++-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 8dffccf6b..e16f7d27d 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -99,6 +99,7 @@ type Config struct { Namespace string OSImageSource *cdiv1beta1.DataVolumeSource StorageClassName string + StorageAccessType corev1.PersistentVolumeAccessMode PVCSize resource.Quantity Instancetype *kubevirtv1.InstancetypeMatcher Preference *kubevirtv1.PreferenceMatcher @@ -144,9 +145,10 @@ type NodeAffinityPreset struct { } type SecondaryDisks struct { - Name string - Size resource.Quantity - StorageClassName string + Name string + Size resource.Quantity + StorageClassName string + StorageAccessType corev1.PersistentVolumeAccessMode } type kubeVirtServer struct { @@ -302,11 +304,13 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.storageClass" field: %w`, err) } config.SecondaryDisks = append(config.SecondaryDisks, SecondaryDisks{ - Name: fmt.Sprintf("secondarydisk%d", i), - Size: pvc, - StorageClassName: scString, + Name: fmt.Sprintf("secondarydisk%d", i), + Size: pvc, + StorageClassName: scString, + StorageAccessType: p.getStorageAccessType(sd.StorageAccessType), }) } + config.StorageAccessType = p.getStorageAccessType(rawConfig.VirtualMachine.Template.PrimaryDisk.StorageAccessType) config.NodeAffinityPreset, err = p.parseNodeAffinityPreset(rawConfig.Affinity.NodeAffinityPreset) if err != nil { @@ -320,6 +324,14 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return &config, pconfig, nil } +func (p *provider) getStorageAccessType(accessType providerconfigtypes.ConfigVarString) corev1.PersistentVolumeAccessMode { + at, _ := p.configVarResolver.GetConfigVarStringValue(accessType) + if at == "" { + return corev1.ReadWriteOnce + } + return corev1.PersistentVolumeAccessMode(at) +} + func (p *provider) parseNodeAffinityPreset(nodeAffinityPreset kubevirttypes.NodeAffinityPreset) (NodeAffinityPreset, error) { nodeAffinity := NodeAffinityPreset{} var err error @@ -844,7 +856,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. PVC: &corev1.PersistentVolumeClaimSpec{ StorageClassName: ptr.To(config.StorageClassName), AccessModes: []corev1.PersistentVolumeAccessMode{ - "ReadWriteOnce", + config.StorageAccessType, }, Resources: corev1.ResourceRequirements{ Requests: pvcRequest, @@ -863,7 +875,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. PVC: &corev1.PersistentVolumeClaimSpec{ StorageClassName: ptr.To(sd.StorageClassName), AccessModes: []corev1.PersistentVolumeAccessMode{ - "ReadWriteOnce", + config.StorageAccessType, }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceStorage: sd.Size}, diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index e32a232fe..0c854d4d9 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -90,8 +90,9 @@ type SecondaryDisks struct { // Disk. type Disk struct { - Size providerconfigtypes.ConfigVarString `json:"size,omitempty"` - StorageClassName providerconfigtypes.ConfigVarString `json:"storageClassName,omitempty"` + Size providerconfigtypes.ConfigVarString `json:"size,omitempty"` + StorageClassName providerconfigtypes.ConfigVarString `json:"storageClassName,omitempty"` + StorageAccessType providerconfigtypes.ConfigVarString `json:"storageAccessType,omitempty"` } // Affinity. From 803ee10b9d671c59a26b382fd0558198fab29fff Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 5 Jan 2024 14:28:38 +0500 Subject: [PATCH 368/489] AWS: increase spot instance price (#1754) Signed-off-by: Waleed Malik --- test/e2e/provisioning/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 5ba7817e4..593fd56ab 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -238,7 +238,7 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar scenarioParams = append(scenarioParams, fmt.Sprintf("<< DATA_DISK_SIZE >>=%v", 30)) scenarioParams = append(scenarioParams, fmt.Sprintf("<< DISK_SIZE >>=%v", 25)) scenarioParams = append(scenarioParams, fmt.Sprintf("<< CUSTOM-IMAGE >>=%v", "")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.02")) + scenarioParams = append(scenarioParams, fmt.Sprintf("<< MAX_PRICE >>=%s", "0.023")) } if strings.Contains(cloudProvider, string(providerconfigtypes.CloudProviderEquinixMetal)) { From 3461bb5dc13384b22776569e4a95074b34557a2f Mon Sep 17 00:00:00 2001 From: Andromeda Date: Fri, 5 Jan 2024 11:44:38 +0000 Subject: [PATCH 369/489] Adds field to check if a IP reservation is still valid, (#1753) if the IP is no longer valid a new one is requested. This fixes a bug where we use an IP that expired already for a VM, if the VM provisioning failed, which then leads to a race condition. In which we wait for the VM provisioning to finish, but it can't due the IP being no longer useable, but we also never request a new IP in that case. Signed-off-by: Zofia hagenguth --- pkg/cloudprovider/provider/anexia/provider.go | 3 ++- .../provider/anexia/provider_test.go | 1 + .../provider/anexia/types/types.go | 18 ++++++++++-------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index e381a388b..c861e1eef 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -233,7 +233,7 @@ func getIPAddress(ctx context.Context, log *zap.SugaredLogger, client anxclient. status := reconcileContext.Status // only use IP if it is still unbound - if status.ReservedIP != "" && status.IPState == anxtypes.IPStateUnbound { + if status.ReservedIP != "" && status.IPState == anxtypes.IPStateUnbound && (!status.IPProvisioningExpires.IsZero() && status.IPProvisioningExpires.After(time.Now())) { log.Infow("Re-using already provisioned IP", "ip", status.ReservedIP) return status.ReservedIP, nil } @@ -259,6 +259,7 @@ func getIPAddress(ctx context.Context, log *zap.SugaredLogger, client anxclient. ip := res.Data[0].Address status.ReservedIP = ip status.IPState = anxtypes.IPStateUnbound + status.IPProvisioningExpires = time.Now().Add(anxtypes.IPProvisioningExpires) return ip, nil } diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index 7754ebb6e..376610dff 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -301,6 +301,7 @@ func TestAnexiaProvider(t *testing.T) { expectedIP := "8.8.8.8" providerStatus.ReservedIP = expectedIP providerStatus.IPState = anxtypes.IPStateUnbound + providerStatus.IPProvisioningExpires = time.Now().Add(anxtypes.IPProvisioningExpires) reservedIP, err := getIPAddress(ctx, log, client) testhelper.AssertNoErr(t, err) testhelper.AssertEquals(t, expectedIP, reservedIP) diff --git a/pkg/cloudprovider/provider/anexia/types/types.go b/pkg/cloudprovider/provider/anexia/types/types.go index 3686defe1..f97a5389a 100644 --- a/pkg/cloudprovider/provider/anexia/types/types.go +++ b/pkg/cloudprovider/provider/anexia/types/types.go @@ -34,8 +34,9 @@ const ( GetRequestTimeout = 1 * time.Minute DeleteRequestTimeout = 1 * time.Minute - IPStateBound = "Bound" - IPStateUnbound = "Unbound" + IPStateBound = "Bound" + IPStateUnbound = "Unbound" + IPProvisioningExpires = 1800 * time.Second VmxNet3NIC = "vmxnet3" MachinePoweredOn = "poweredOn" @@ -72,12 +73,13 @@ type RawConfig struct { } type ProviderStatus struct { - InstanceID string `json:"instanceID"` - ProvisioningID string `json:"provisioningID"` - DeprovisioningID string `json:"deprovisioningID"` - ReservedIP string `json:"reservedIP"` - IPState string `json:"ipState"` - Conditions []v1.Condition `json:"conditions,omitempty"` + InstanceID string `json:"instanceID"` + ProvisioningID string `json:"provisioningID"` + DeprovisioningID string `json:"deprovisioningID"` + ReservedIP string `json:"reservedIP"` + IPState string `json:"ipState"` + IPProvisioningExpires time.Time `json:"ipProvisioningExpires"` + Conditions []v1.Condition `json:"conditions,omitempty"` } func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { From 5291aac61a9fbb4b099ade056cb0aa155ca4c019 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Jan 2024 13:01:38 +0100 Subject: [PATCH 370/489] Bump golang.org/x/crypto from 0.14.0 to 0.17.0 (#1750) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.14.0 to 0.17.0. - [Commits](https://github.com/golang/crypto/compare/v0.14.0...v0.17.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 5419a8e34..76c3b1ed9 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/vultr/govultr/v3 v3.3.1 go.anx.io/go-anxcloud v0.5.3 go.uber.org/zap v1.25.0 - golang.org/x/crypto v0.14.0 + golang.org/x/crypto v0.17.0 golang.org/x/oauth2 v0.11.0 gomodules.xyz/jsonpatch/v2 v2.4.0 google.golang.org/api v0.137.0 @@ -169,9 +169,9 @@ require ( go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.3 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 5277aa503..f4dac3bd9 100644 --- a/go.sum +++ b/go.sum @@ -809,8 +809,8 @@ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1009,15 +1009,15 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1030,8 +1030,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From ea213412adf9e5efcd54e8b672e34d5395d002b7 Mon Sep 17 00:00:00 2001 From: Mara Sophie Grosch Date: Mon, 8 Jan 2024 09:17:45 +0100 Subject: [PATCH 371/489] webhook: refactor to use controller-runtime server (#1751) Refactor the webhook to use the controller-runtime provided webhook.Server instead of a plain HTTP server. The biggest benefit this brings, is it automatically loading changed certificate files instead of forever using the ones present when the server was started. This could have been implemented by using the controller-runtime CertWatcher manually, but that would've been more code and aligning machine-controller with the ecosystem-provided standards seems worth it. Signed-off-by: Mara Sophie Grosch --- cmd/webhook/main.go | 16 +++++++---- pkg/admission/admission.go | 59 +++++++++++++++++++++++++++++--------- 2 files changed, 55 insertions(+), 20 deletions(-) diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index d501bafaf..606e2664c 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -33,6 +33,7 @@ import ( "k8s.io/client-go/tools/clientcmd" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ctrlruntimelog "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager/signals" ) type options struct { @@ -143,6 +144,13 @@ func main() { NodeFlags: nodeFlags, Namespace: opt.namespace, VersionConstraints: constraint, + + // we could change this to get the CertDir from the configured CertName + // and KeyName, but doing so does not bring us any benefits but would + // technically break compatibility. + CertDir: "/", + CertName: opt.admissionTLSCertPath, + KeyName: opt.admissionTLSKeyPath, }.Build() if err != nil { log.Fatalw("Failed to create admission hook", zap.Error(err)) @@ -150,12 +158,8 @@ func main() { log.Infow("Listening", "address", opt.admissionListenAddress) - if err := srv.ListenAndServeTLS(opt.admissionTLSCertPath, opt.admissionTLSKeyPath); err != nil { + serverContext := signals.SetupSignalHandler() + if err := srv.Start(serverContext); err != nil { log.Fatalw("Failed to start server", zap.Error(err)) } - defer func() { - if err := srv.Close(); err != nil { - log.Fatalw("Failed to shutdown server", zap.Error(err)) - } - }() } diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go index bf563bb6e..b10d76871 100644 --- a/pkg/admission/admission.go +++ b/pkg/admission/admission.go @@ -22,9 +22,10 @@ import ( "errors" "fmt" "io" + "net" "net/http" "reflect" - "time" + "strconv" "github.com/Masterminds/semver/v3" "go.uber.org/zap" @@ -39,6 +40,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/webhook" ) type admissionData struct { @@ -64,10 +67,13 @@ type Builder struct { NodeFlags *node.Flags Namespace string VersionConstraints *semver.Constraints + + CertDir string + CertName string + KeyName string } -func (build Builder) Build() (*http.Server, error) { - mux := http.NewServeMux() +func (build Builder) Build() (webhook.Server, error) { ad := &admissionData{ log: build.Log, client: build.Client, @@ -82,18 +88,43 @@ func (build Builder) Build() (*http.Server, error) { return nil, fmt.Errorf("error updating nodeSettings, %w", err) } - mux.HandleFunc("/machinedeployments", handleFuncFactory(build.Log, ad.mutateMachineDeployments)) - mux.HandleFunc("/machines", handleFuncFactory(build.Log, ad.mutateMachines)) - mux.HandleFunc("/healthz", healthZHandler) + options := webhook.Options{ + CertDir: build.CertDir, + CertName: build.CertName, + KeyName: build.KeyName, + } - return &http.Server{ - Addr: build.ListenAddress, - Handler: http.TimeoutHandler(mux, 25*time.Second, "timeout"), - }, nil -} + if build.ListenAddress != "" { + host, port, err := net.SplitHostPort(build.ListenAddress) + if err != nil { + return nil, fmt.Errorf("error parsing ListenAddress: %w", err) + } + + options.Host = host + + if port != "" { + port, err := strconv.ParseInt(port, 10, 16) + if err != nil { + return nil, fmt.Errorf("error parsing port from ListenAddress: %w", err) + } + + options.Port = int(port) + } + } + + server := webhook.NewServer(options) + + server.Register("/machinedeployments", handleFuncFactory(build.Log, ad.mutateMachineDeployments)) + server.Register("/machines", handleFuncFactory(build.Log, ad.mutateMachines)) + + checkers := healthz.Handler{ + Checks: map[string]healthz.Checker{ + "ping": healthz.Ping, + }, + } + server.Register("/healthz/", http.StripPrefix("/healthz/", &checkers)) -func healthZHandler(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) + return server, nil } func newJSONPatch(original, current runtime.Object) ([]jsonpatch.JsonPatchOperation, error) { @@ -136,7 +167,7 @@ func createAdmissionResponse(log *zap.SugaredLogger, original, mutated runtime.O type mutator func(context.Context, admissionv1.AdmissionRequest) (*admissionv1.AdmissionResponse, error) -func handleFuncFactory(log *zap.SugaredLogger, mutate mutator) func(http.ResponseWriter, *http.Request) { +func handleFuncFactory(log *zap.SugaredLogger, mutate mutator) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { review, err := readReview(r) if err != nil { From 3f02ef62f2828eb5b6400275083c442480685ef9 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 8 Jan 2024 19:01:46 +0500 Subject: [PATCH 372/489] Support for Kubernetes 1.29 (#1755) * Add support for kubernetes v1.29 Signed-off-by: Waleed Malik * Update dependencies and k8s API to v1.29 Signed-off-by: Waleed Malik * Enable in-tree providers Signed-off-by: Waleed Malik * Update fixtures Signed-off-by: Waleed Malik * Refactored code Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- README.md | 1 + cmd/machine-controller/main.go | 36 +- examples/alibaba-machinedeployment.yaml | 2 +- examples/anexia-machinedeployment.yaml | 2 +- examples/aws-machinedeployment.yaml | 2 +- examples/azure-machinedeployment.yaml | 2 +- examples/digitalocean-machinedeployment.yaml | 2 +- examples/equinixmetal-machinedeployment.yaml | 2 +- examples/gce-machinedeployment.yaml | 2 +- examples/hetzner-machinedeployment.yaml | 2 +- examples/kubevirt-machinedeployment.yaml | 2 +- examples/linode-machinedeployment.yaml | 2 +- examples/nutanix-machinedeployment.yaml | 2 +- examples/opennebula-machinedeployment.yaml | 2 +- examples/openstack-machinedeployment.yaml | 2 +- examples/operating-system-manager.yaml | 1688 +++++++++-------- examples/scaleway-machinedeployment.yaml | 2 +- ...ware-cloud-director-machinedeployment.yaml | 2 +- ...e-datastore-cluster-machinedeployment.yaml | 2 +- examples/vsphere-machinedeployment.yaml | 2 +- examples/vultr-machinedeployment.yaml | 6 +- go.mod | 187 +- go.sum | 417 ++-- .../provider/kubevirt/provider.go | 4 +- pkg/node/poddeletion/pod_deletion.go | 4 +- pkg/rhsm/subscription_manager_test.go | 9 +- pkg/userdata/amzn2/provider_test.go | 11 +- .../amzn2/testdata/kubelet-v1.26-aws.yaml | 1 + .../kubelet-v1.26.6-aws-external.yaml | 1 + .../amzn2/testdata/kubelet-v1.26.6-aws.yaml | 1 + .../kubelet-v1.26.6-vsphere-mirrors.yaml | 1 + .../kubelet-v1.26.6-vsphere-proxy.yaml | 1 + .../testdata/kubelet-v1.26.6-vsphere.yaml | 1 + .../amzn2/testdata/kubelet-v1.27-aws.yaml | 1 + .../amzn2/testdata/kubelet-v1.28-aws.yaml | 3 +- .../amzn2/testdata/kubelet-v1.29-aws.yaml | 454 +++++ pkg/userdata/centos/provider_test.go | 11 +- .../kubelet-v1.26.6-aws-external.yaml | 1 + .../centos/testdata/kubelet-v1.26.6-aws.yaml | 1 + .../testdata/kubelet-v1.26.6-nutanix.yaml | 1 + .../kubelet-v1.26.6-vsphere-mirrors.yaml | 1 + .../kubelet-v1.26.6-vsphere-proxy.yaml | 1 + .../testdata/kubelet-v1.26.6-vsphere.yaml | 1 + .../centos/testdata/kubelet-v1.27-aws.yaml | 1 + .../centos/testdata/kubelet-v1.28-aws.yaml | 3 +- .../centos/testdata/kubelet-v1.29-aws.yaml | 460 +++++ pkg/userdata/flatcar/provider_test.go | 70 +- .../flatcar/testdata/cloud-init_v1.26.6.yaml | 1 + .../flatcar/testdata/cloud-init_v1.28.0.yaml | 979 +++++----- .../flatcar/testdata/cloud-init_v1.28.5.yaml | 521 +++++ .../flatcar/testdata/cloud-init_v1.29.0.yaml | 521 +++++ pkg/userdata/flatcar/testdata/containerd.yaml | 1 + .../flatcar/testdata/ignition_v1.26.6.json | 2 +- .../flatcar/testdata/ignition_v1.28.0.json | 232 ++- .../flatcar/testdata/ignition_v1.28.5.json | 1 + .../flatcar/testdata/ignition_v1.29.0.json | 1 + pkg/userdata/helper/common_test.go | 7 +- .../download_binaries_v1.26.12.golden | 17 + ...olden => download_binaries_v1.27.9.golden} | 2 +- ...olden => download_binaries_v1.28.5.golden} | 2 +- ...olden => download_binaries_v1.29.0.golden} | 2 +- ...emd_unit_version-v1.26.12-external.golden} | 0 ...blet_systemd_unit_version-v1.26.12.golden} | 0 ...temd_unit_version-v1.27.9-external.golden} | 0 ...ublet_systemd_unit_version-v1.27.9.golden} | 0 ...temd_unit_version-v1.28.5-external.golden} | 0 ...ublet_systemd_unit_version-v1.28.5.golden} | 0 ...stemd_unit_version-v1.29.0-external.golden | 36 + ...kublet_systemd_unit_version-v1.29.0.golden | 35 + pkg/userdata/rhel/provider_test.go | 25 +- .../testdata/kubelet-v1.26-aws-external.yaml | 1 + .../rhel/testdata/kubelet-v1.26-aws.yaml | 1 + .../kubelet-v1.26.6-aws-external.yaml | 1 + .../rhel/testdata/kubelet-v1.26.6-aws.yaml | 1 + .../kubelet-v1.26.6-vsphere-mirrors.yaml | 1 + .../kubelet-v1.26.6-vsphere-proxy.yaml | 1 + .../testdata/kubelet-v1.26.6-vsphere.yaml | 1 + .../rhel/testdata/kubelet-v1.28-aws.yaml | 3 +- .../rhel/testdata/kubelet-v1.28-nutanix.yaml | 3 +- .../rhel/testdata/kubelet-v1.29-aws.yaml | 501 +++++ .../rhel/testdata/kubelet-v1.29-nutanix.yaml | 510 +++++ .../rhel/testdata/pod-cidr-azure-rhel.yaml | 3 +- pkg/userdata/rockylinux/provider_test.go | 11 +- .../kubelet-v1.26.6-aws-external.yaml | 1 + .../testdata/kubelet-v1.26.6-aws.yaml | 1 + .../testdata/kubelet-v1.26.6-nutanix.yaml | 1 + .../kubelet-v1.26.6-vsphere-mirrors.yaml | 1 + .../kubelet-v1.26.6-vsphere-proxy.yaml | 1 + .../testdata/kubelet-v1.26.6-vsphere.yaml | 1 + .../testdata/kubelet-v1.28-aws.yaml | 3 +- .../testdata/kubelet-v1.29-aws.yaml | 467 +++++ pkg/userdata/ubuntu/provider_test.go | 9 +- pkg/userdata/ubuntu/testdata/containerd.yaml | 1 + .../digitalocean-dualstack-IPv6+IPv4.yaml | 1 + .../testdata/digitalocean-dualstack.yaml | 1 + .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 1 + pkg/userdata/ubuntu/testdata/docker.yaml | 1 + .../kubelet-version-without-v-prefix.yaml | 1 + .../ubuntu/testdata/multiple-dns-servers.yaml | 1 + .../ubuntu/testdata/multiple-ssh-keys.yaml | 1 + pkg/userdata/ubuntu/testdata/nutanix.yaml | 3 +- .../openstack-dualstack-IPv6+IPv4.yaml | 1 + .../ubuntu/testdata/openstack-dualstack.yaml | 1 + .../openstack-overwrite-cloud-config.yaml | 1 + pkg/userdata/ubuntu/testdata/openstack.yaml | 1 + .../ubuntu/testdata/version-1.26.12.yaml | 459 +++++ ...ersion-1.27.6.yaml => version-1.27.9.yaml} | 3 +- ...ersion-1.28.2.yaml => version-1.28.5.yaml} | 3 +- ...ersion-1.26.9.yaml => version-1.29.0.yaml} | 3 +- .../ubuntu/testdata/vsphere-mirrors.yaml | 1 + .../ubuntu/testdata/vsphere-proxy.yaml | 1 + pkg/userdata/ubuntu/testdata/vsphere.yaml | 1 + test/e2e/provisioning/all_e2e_test.go | 12 +- test/e2e/provisioning/helper.go | 7 +- ...hinedeployment-azure-redhat-satellite.yaml | 2 +- 115 files changed, 6177 insertions(+), 1650 deletions(-) create mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.29-aws.yaml create mode 100644 pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml create mode 100644 pkg/userdata/flatcar/testdata/cloud-init_v1.28.5.yaml create mode 100644 pkg/userdata/flatcar/testdata/cloud-init_v1.29.0.yaml create mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.28.5.json create mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.29.0.json create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.26.12.golden rename pkg/userdata/helper/testdata/{download_binaries_v1.26.9.golden => download_binaries_v1.27.9.golden} (91%) rename pkg/userdata/helper/testdata/{download_binaries_v1.27.6.golden => download_binaries_v1.28.5.golden} (91%) rename pkg/userdata/helper/testdata/{download_binaries_v1.28.2.golden => download_binaries_v1.29.0.golden} (91%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.26.9-external.golden => kublet_systemd_unit_version-v1.26.12-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.26.9.golden => kublet_systemd_unit_version-v1.26.12.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.27.6-external.golden => kublet_systemd_unit_version-v1.27.9-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.27.6.golden => kublet_systemd_unit_version-v1.27.9.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.28.2-external.golden => kublet_systemd_unit_version-v1.28.5-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.28.2.golden => kublet_systemd_unit_version-v1.28.5.golden} (100%) create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.0-external.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.0.golden create mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.29-aws.yaml create mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.29-nutanix.yaml create mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.29-aws.yaml create mode 100644 pkg/userdata/ubuntu/testdata/version-1.26.12.yaml rename pkg/userdata/ubuntu/testdata/{version-1.27.6.yaml => version-1.27.9.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.28.2.yaml => version-1.28.5.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.26.9.yaml => version-1.29.0.yaml} (99%) diff --git a/README.md b/README.md index 4407629df..470eb79aa 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ machine-controller tries to follow the Kubernetes version Currently supported K8S versions are: +- 1.29 - 1.28 - 1.27 - 1.26 diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index a3c1761f4..0931c26f8 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -52,11 +52,13 @@ import ( "k8s.io/client-go/kubernetes/scheme" restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/healthz" ctrlruntimelog "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" "sigs.k8s.io/controller-runtime/pkg/metrics" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" ) var ( @@ -337,13 +339,29 @@ func createManager(syncPeriod time.Duration, options controllerRunOptions) (mana namespace = defaultLeaderElectionNamespace } + metricsOptions := metricsserver.Options{BindAddress: metricsAddress} + if profiling { + m := http.NewServeMux() + m.HandleFunc("/", pprof.Index) + m.HandleFunc("/cmdline", pprof.Cmdline) + m.HandleFunc("/profile", pprof.Profile) + m.HandleFunc("/symbol", pprof.Symbol) + m.HandleFunc("/trace", pprof.Trace) + metricsOptions.ExtraHandlers = map[string]http.Handler{ + "/debug/pprof/": m, + } + } + mgr, err := manager.New(options.cfg, manager.Options{ - SyncPeriod: &syncPeriod, + Cache: cache.Options{ + DefaultNamespaces: map[string]cache.Config{}, + SyncPeriod: &syncPeriod, + }, LeaderElection: enableLeaderElection, LeaderElectionID: defaultLeaderElectionID, LeaderElectionNamespace: namespace, HealthProbeBindAddress: healthProbeAddress, - MetricsBindAddress: metricsAddress, + Metrics: metricsOptions, }) if err != nil { return nil, fmt.Errorf("failed to build ctrlruntime manager: %w", err) @@ -360,20 +378,6 @@ func createManager(syncPeriod time.Duration, options controllerRunOptions) (mana if err := mgr.AddHealthzCheck("apiserver-connection", health.ApiserverReachable(options.kubeClient)); err != nil { return nil, fmt.Errorf("failed to add health check: %w", err) } - - if profiling { - m := http.NewServeMux() - m.HandleFunc("/", pprof.Index) - m.HandleFunc("/cmdline", pprof.Cmdline) - m.HandleFunc("/profile", pprof.Profile) - m.HandleFunc("/symbol", pprof.Symbol) - m.HandleFunc("/trace", pprof.Trace) - - if err := mgr.AddMetricsExtraHandler("/debug/pprof/", m); err != nil { - return nil, fmt.Errorf("failed to add pprof http handlers: %w", err) - } - } - if err := mgr.Add(&controllerBootstrap{ mgr: mgr, opt: options, diff --git a/examples/alibaba-machinedeployment.yaml b/examples/alibaba-machinedeployment.yaml index e25a10b2b..154b6196c 100644 --- a/examples/alibaba-machinedeployment.yaml +++ b/examples/alibaba-machinedeployment.yaml @@ -62,4 +62,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index acafdaf75..7e9745903 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -54,4 +54,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/aws-machinedeployment.yaml b/examples/aws-machinedeployment.yaml index 8f56c6892..b82c2b505 100644 --- a/examples/aws-machinedeployment.yaml +++ b/examples/aws-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/azure-machinedeployment.yaml b/examples/azure-machinedeployment.yaml index 6612809f7..755a1e5a7 100644 --- a/examples/azure-machinedeployment.yaml +++ b/examples/azure-machinedeployment.yaml @@ -91,4 +91,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/digitalocean-machinedeployment.yaml b/examples/digitalocean-machinedeployment.yaml index bdf1db222..804c8c3b4 100644 --- a/examples/digitalocean-machinedeployment.yaml +++ b/examples/digitalocean-machinedeployment.yaml @@ -57,4 +57,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/equinixmetal-machinedeployment.yaml b/examples/equinixmetal-machinedeployment.yaml index 20e4733b4..dbce45cce 100644 --- a/examples/equinixmetal-machinedeployment.yaml +++ b/examples/equinixmetal-machinedeployment.yaml @@ -51,4 +51,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index dca6466f8..7c598af34 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -90,4 +90,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/hetzner-machinedeployment.yaml b/examples/hetzner-machinedeployment.yaml index 07a7324fe..1d3adb228 100644 --- a/examples/hetzner-machinedeployment.yaml +++ b/examples/hetzner-machinedeployment.yaml @@ -67,4 +67,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index d510c9490..c96d47350 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -69,4 +69,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/linode-machinedeployment.yaml b/examples/linode-machinedeployment.yaml index a08d60acf..f00928ac4 100644 --- a/examples/linode-machinedeployment.yaml +++ b/examples/linode-machinedeployment.yaml @@ -53,4 +53,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/nutanix-machinedeployment.yaml b/examples/nutanix-machinedeployment.yaml index 862bb15bb..bb7e863a6 100644 --- a/examples/nutanix-machinedeployment.yaml +++ b/examples/nutanix-machinedeployment.yaml @@ -82,4 +82,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/opennebula-machinedeployment.yaml b/examples/opennebula-machinedeployment.yaml index 768135a99..ab4309da7 100644 --- a/examples/opennebula-machinedeployment.yaml +++ b/examples/opennebula-machinedeployment.yaml @@ -67,4 +67,4 @@ spec: # use cloud-init for flatcar as ignition doesn't know anything about OpenNebula yet provisioningUtility: "cloud-init" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index ccffe0096..ddfa3c030 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -166,4 +166,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index ee052ae5c..7f78d2c59 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -14,360 +14,402 @@ spec: listKind: OperatingSystemConfigList plural: operatingsystemconfigs shortNames: - - osc + - osc singular: operatingsystemconfig scope: Namespaced versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: OperatingSystemConfig is the object that represents the OperatingSystemConfig - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: OperatingSystemConfigSpec represents the operating system - configuration spec. - properties: - bootstrapConfig: - description: BootstrapConfig is used for initial configuration of - machine and to fetch the kubernetes secret that contains the provisioning - config. - properties: - files: - description: Files is a list of files that should exist in the - instance - items: - description: File is a file that should get written to the host's - file system. The content can either be inlined or referenced - from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. - properties: - inline: - description: Inline is a struct that contains information - about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: Encoding is the file's encoding (e.g. - base64). - type: string - required: - - data - type: object - type: object - path: - description: Path is the path of the file system where the - file should get written to. - type: string - permissions: - default: 644 - description: Permissions describes with which permissions - the file should get written to the file system. Should - be in decimal base and without any leading zeroes. - format: int32 - type: integer - required: - - content - - path - type: object - type: array - modules: - description: CloudInitModules contains the supported cloud-init - modules - properties: - bootcmd: - description: BootCMD module runs arbitrary commands very early - in the boot process, only slightly after a boothook would - run. - items: - type: string - type: array - rh_subscription: - additionalProperties: - type: string - description: RHSubscription registers a Red Hat system either - by username and password or activation and org + - name: v1alpha1 + schema: + openAPIV3Schema: + description: OperatingSystemConfig is the object that represents the OperatingSystemConfig + properties: + apiVersion: + description: + "APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + kind: + description: + "Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + metadata: + type: object + spec: + description: + OperatingSystemConfigSpec represents the operating system + configuration spec. + properties: + bootstrapConfig: + description: + BootstrapConfig is used for initial configuration of + machine and to fetch the kubernetes secret that contains the provisioning + config. + properties: + files: + description: + Files is a list of files that should exist in the + instance + items: + description: + File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: + Inline is a struct that contains information + about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding (e.g. + base64). + type: string + required: + - data + type: object + type: object + path: + description: + Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path type: object - runcmd: - description: RunCMD Run arbitrary commands at a rc.local like - level with output to the console. - items: - type: string - type: array - yum_repo_dir: - description: 'YumRepoDir the repo parts directory where individual - yum repo config files will be written. Default: /etc/yum.repos.d' - type: string - yum_repos: - additionalProperties: + type: array + modules: + description: + CloudInitModules contains the supported cloud-init + modules + properties: + bootcmd: + description: + BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string + type: array + rh_subscription: additionalProperties: type: string + description: + RHSubscription registers a Red Hat system either + by username and password or activation and org type: object - description: YumRepos adds yum repository configuration to - the system. - type: object - type: object - units: - description: Units a list of the systemd unit files which will - run on the instance - items: - description: Unit is a systemd unit used for the operating system - config. - properties: - content: - description: Content is the unit's content. - type: string - dropIns: - description: DropIns is a list of drop-ins for this unit. + runcmd: + description: + RunCMD Run arbitrary commands at a rc.local like + level with output to the console. items: - description: DropIn is a drop-in configuration for a systemd - unit. - properties: - content: - description: Content is the content of the drop-in. - type: string - name: - description: Name is the name of the drop-in. - type: string - required: - - content - - name - type: object + type: string type: array - enable: - description: Enable describes whether the unit is enabled - or not. - type: boolean - mask: - description: Mask describes whether the unit is masked or - not. - type: boolean - name: - description: Name is the name of a unit. + yum_repo_dir: + description: + "YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d" type: string - required: - - name + yum_repos: + additionalProperties: + additionalProperties: + type: string + type: object + description: + YumRepos adds yum repository configuration to + the system. + type: object type: object - type: array - userSSHKeys: - description: UserSSHKeys is a list of attached user ssh keys - items: - type: string - type: array - type: object - cloudProvider: - description: CloudProvider represent the cloud provider that support - the given operating system version - properties: - name: - description: Name represents the name of the supported cloud provider - enum: - - aws - - azure - - digitalocean - - gce - - hetzner - - kubevirt - - linode - - nutanix - - openstack - - equinixmetal - - vsphere - - fake - - alibaba - - anexia - - scaleway - - baremetal - - external - - vmware-cloud-director - - opennebula - type: string - spec: - description: Spec represents the os/image reference in the supported - cloud provider - type: object - x-kubernetes-preserve-unknown-fields: true - required: - - name - type: object - osName: - description: 'OSType represent the operating system name e.g: ubuntu' - enum: - - flatcar - - rhel - - centos - - ubuntu - - amzn2 - - rockylinux - type: string - osVersion: - description: OSVersion the version of the operating system - type: string - provisioningConfig: - description: ProvisioningConfig is used for provisioning the worker - node. - properties: - files: - description: Files is a list of files that should exist in the - instance - items: - description: File is a file that should get written to the host's - file system. The content can either be inlined or referenced - from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. - properties: - inline: - description: Inline is a struct that contains information - about the inlined data. + units: + description: + Units a list of the systemd unit files which will + run on the instance + items: + description: + Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. + items: + description: + DropIn is a drop-in configuration for a systemd + unit. properties: - data: - description: Data is the file's data. + content: + description: Content is the content of the drop-in. type: string - encoding: - description: Encoding is the file's encoding (e.g. - base64). + name: + description: Name is the name of the drop-in. type: string required: - - data + - content + - name type: object - type: object - path: - description: Path is the path of the file system where the - file should get written to. - type: string - permissions: - default: 644 - description: Permissions describes with which permissions - the file should get written to the file system. Should - be in decimal base and without any leading zeroes. - format: int32 - type: integer - required: - - content - - path - type: object - type: array - modules: - description: CloudInitModules contains the supported cloud-init - modules - properties: - bootcmd: - description: BootCMD module runs arbitrary commands very early - in the boot process, only slightly after a boothook would - run. - items: - type: string - type: array - rh_subscription: - additionalProperties: - type: string - description: RHSubscription registers a Red Hat system either - by username and password or activation and org + type: array + enable: + description: + Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: + Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. + type: string + required: + - name type: object - runcmd: - description: RunCMD Run arbitrary commands at a rc.local like - level with output to the console. - items: - type: string - type: array - yum_repo_dir: - description: 'YumRepoDir the repo parts directory where individual - yum repo config files will be written. Default: /etc/yum.repos.d' + type: array + userSSHKeys: + description: UserSSHKeys is a list of attached user ssh keys + items: type: string - yum_repos: - additionalProperties: - additionalProperties: + type: array + type: object + cloudProvider: + description: + CloudProvider represent the cloud provider that support + the given operating system version + properties: + name: + description: Name represents the name of the supported cloud provider + enum: + - aws + - azure + - digitalocean + - gce + - hetzner + - kubevirt + - linode + - nutanix + - openstack + - equinixmetal + - vsphere + - fake + - alibaba + - anexia + - scaleway + - baremetal + - external + - vmware-cloud-director + - opennebula + type: string + spec: + description: + Spec represents the os/image reference in the supported + cloud provider + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - name + type: object + osName: + description: "OSType represent the operating system name e.g: ubuntu" + enum: + - flatcar + - rhel + - centos + - ubuntu + - amzn2 + - rockylinux + type: string + osVersion: + description: OSVersion the version of the operating system + type: string + provisioningConfig: + description: + ProvisioningConfig is used for provisioning the worker + node. + properties: + files: + description: + Files is a list of files that should exist in the + instance + items: + description: + File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: + Inline is a struct that contains information + about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding (e.g. + base64). + type: string + required: + - data + type: object + type: object + path: + description: + Path is the path of the file system where the + file should get written to. type: string - type: object - description: YumRepos adds yum repository configuration to - the system. + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path type: object - type: object - units: - description: Units a list of the systemd unit files which will - run on the instance - items: - description: Unit is a systemd unit used for the operating system - config. + type: array + modules: + description: + CloudInitModules contains the supported cloud-init + modules properties: - content: - description: Content is the unit's content. - type: string - dropIns: - description: DropIns is a list of drop-ins for this unit. + bootcmd: + description: + BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. items: - description: DropIn is a drop-in configuration for a systemd - unit. - properties: - content: - description: Content is the content of the drop-in. - type: string - name: - description: Name is the name of the drop-in. - type: string - required: - - content - - name - type: object + type: string type: array - enable: - description: Enable describes whether the unit is enabled - or not. - type: boolean - mask: - description: Mask describes whether the unit is masked or - not. - type: boolean - name: - description: Name is the name of a unit. + rh_subscription: + additionalProperties: + type: string + description: + RHSubscription registers a Red Hat system either + by username and password or activation and org + type: object + runcmd: + description: + RunCMD Run arbitrary commands at a rc.local like + level with output to the console. + items: + type: string + type: array + yum_repo_dir: + description: + "YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d" type: string - required: - - name + yum_repos: + additionalProperties: + additionalProperties: + type: string + type: object + description: + YumRepos adds yum repository configuration to + the system. + type: object type: object - type: array - userSSHKeys: - description: UserSSHKeys is a list of attached user ssh keys - items: - type: string - type: array - type: object - provisioningUtility: - default: cloud-init - description: ProvisioningUtility used for configuring the worker node. - Defaults to cloud-init. - enum: - - cloud-init - - ignition - type: string - required: - - bootstrapConfig - - cloudProvider - - osName - - osVersion - - provisioningConfig - type: object - required: - - spec - type: object - served: true - storage: true + units: + description: + Units a list of the systemd unit files which will + run on the instance + items: + description: + Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. + items: + description: + DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object + type: array + enable: + description: + Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: + Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. + type: string + required: + - name + type: object + type: array + userSSHKeys: + description: UserSSHKeys is a list of attached user ssh keys + items: + type: string + type: array + type: object + provisioningUtility: + default: cloud-init + description: + ProvisioningUtility used for configuring the worker node. + Defaults to cloud-init. + enum: + - cloud-init + - ignition + type: string + required: + - bootstrapConfig + - cloudProvider + - osName + - osVersion + - provisioningConfig + type: object + required: + - spec + type: object + served: true + storage: true --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition @@ -383,504 +425,566 @@ spec: listKind: OperatingSystemProfileList plural: operatingsystemprofiles shortNames: - - osp + - osp singular: operatingsystemprofile scope: Namespaced versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: OperatingSystemProfile is the object that represents the OperatingSystemProfile - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: OperatingSystemProfileSpec represents the operating system - configuration spec. - properties: - bootstrapConfig: - description: BootstrapConfig is used for initial configuration of - machine and to fetch the kubernetes secret that contains the provisioning - config. - properties: - files: - description: Files is a list of files that should exist in the - instance - items: - description: File is a file that should get written to the host's - file system. The content can either be inlined or referenced - from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. - properties: - inline: - description: Inline is a struct that contains information - about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: Encoding is the file's encoding (e.g. - base64). - type: string - required: - - data - type: object - type: object - path: - description: Path is the path of the file system where the - file should get written to. - type: string - permissions: - default: 644 - description: Permissions describes with which permissions - the file should get written to the file system. Should - be in decimal base and without any leading zeroes. - format: int32 - type: integer - required: - - content - - path - type: object - type: array - modules: - description: CloudInitModules field contains the optional cloud-init - modules which are supported by OSM - properties: - bootcmd: - description: BootCMD module runs arbitrary commands very early - in the boot process, only slightly after a boothook would - run. - items: - type: string - type: array - rh_subscription: - additionalProperties: - type: string - description: RHSubscription registers a Red Hat system either - by username and password or activation and org - type: object - runcmd: - description: RunCMD Run arbitrary commands at a rc.local like - level with output to the console. - items: - type: string - type: array - yum_repo_dir: - description: 'YumRepoDir the repo parts directory where individual - yum repo config files will be written. Default: /etc/yum.repos.d' - type: string - yum_repos: - additionalProperties: - additionalProperties: - type: string - type: object - description: YumRepos adds yum repository configuration to - the system. - type: object - type: object - supportedContainerRuntimes: - description: SupportedContainerRuntimes represents the container - runtimes supported by the given OS - items: - description: ContainerRuntimeSpec aggregates information about - a specific container runtime - properties: - files: - description: Files to add to the main files list when the - containerRuntime is selected - items: - description: File is a file that should get written to - the host's file system. The content can either be inlined - or referenced from a secret in the same namespace. + - name: v1alpha1 + schema: + openAPIV3Schema: + description: OperatingSystemProfile is the object that represents the OperatingSystemProfile + properties: + apiVersion: + description: + "APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + kind: + description: + "Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + metadata: + type: object + spec: + description: + OperatingSystemProfileSpec represents the operating system + configuration spec. + properties: + bootstrapConfig: + description: + BootstrapConfig is used for initial configuration of + machine and to fetch the kubernetes secret that contains the provisioning + config. + properties: + files: + description: + Files is a list of files that should exist in the + instance + items: + description: + File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. properties: - content: - description: Content describe the file's content. + inline: + description: + Inline is a struct that contains information + about the inlined data. properties: - inline: - description: Inline is a struct that contains - information about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: Encoding is the file's encoding - (e.g. base64). - type: string - required: - - data - type: object + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding (e.g. + base64). + type: string + required: + - data type: object - path: - description: Path is the path of the file system where - the file should get written to. - type: string - permissions: - default: 644 - description: Permissions describes with which permissions - the file should get written to the file system. - Should be in decimal base and without any leading - zeroes. - format: int32 - type: integer - required: - - content - - path type: object + path: + description: + Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + modules: + description: + CloudInitModules field contains the optional cloud-init + modules which are supported by OSM + properties: + bootcmd: + description: + BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string type: array - name: - description: Name of the Container runtime - enum: - - docker - - containerd - type: string - templates: + rh_subscription: additionalProperties: type: string - description: Templates to add to the available templates - when the containerRuntime is selected + description: + RHSubscription registers a Red Hat system either + by username and password or activation and org type: object - required: - - files - - name - type: object - type: array - templates: - additionalProperties: - type: string - description: Templates to be included in units and files - type: object - units: - description: Units a list of the systemd unit files which will - run on the instance - items: - description: Unit is a systemd unit used for the operating system - config. - properties: - content: - description: Content is the unit's content. - type: string - dropIns: - description: DropIns is a list of drop-ins for this unit. + runcmd: + description: + RunCMD Run arbitrary commands at a rc.local like + level with output to the console. items: - description: DropIn is a drop-in configuration for a systemd - unit. - properties: - content: - description: Content is the content of the drop-in. - type: string - name: - description: Name is the name of the drop-in. - type: string - required: - - content - - name - type: object + type: string type: array - enable: - description: Enable describes whether the unit is enabled - or not. - type: boolean - mask: - description: Mask describes whether the unit is masked or - not. - type: boolean - name: - description: Name is the name of a unit. + yum_repo_dir: + description: + "YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d" type: string - required: - - name + yum_repos: + additionalProperties: + additionalProperties: + type: string + type: object + description: + YumRepos adds yum repository configuration to + the system. + type: object type: object - type: array - type: object - osName: - description: 'OSType represent the operating system name e.g: ubuntu' - enum: - - flatcar - - rhel - - centos - - ubuntu - - amzn2 - - rockylinux - type: string - osVersion: - description: OSVersion the version of the operating system - type: string - provisioningConfig: - description: ProvisioningConfig is used for provisioning the worker - node. - properties: - files: - description: Files is a list of files that should exist in the - instance - items: - description: File is a file that should get written to the host's - file system. The content can either be inlined or referenced - from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. - properties: - inline: - description: Inline is a struct that contains information - about the inlined data. + supportedContainerRuntimes: + description: + SupportedContainerRuntimes represents the container + runtimes supported by the given OS + items: + description: + ContainerRuntimeSpec aggregates information about + a specific container runtime + properties: + files: + description: + Files to add to the main files list when the + containerRuntime is selected + items: + description: + File is a file that should get written to + the host's file system. The content can either be inlined + or referenced from a secret in the same namespace. properties: - data: - description: Data is the file's data. - type: string - encoding: - description: Encoding is the file's encoding (e.g. - base64). + content: + description: Content describe the file's content. + properties: + inline: + description: + Inline is a struct that contains + information about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding + (e.g. base64). + type: string + required: + - data + type: object + type: object + path: + description: + Path is the path of the file system where + the file should get written to. type: string + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. + Should be in decimal base and without any leading + zeroes. + format: int32 + type: integer required: - - data + - content + - path type: object - type: object - path: - description: Path is the path of the file system where the - file should get written to. - type: string - permissions: - default: 644 - description: Permissions describes with which permissions - the file should get written to the file system. Should - be in decimal base and without any leading zeroes. - format: int32 - type: integer - required: - - content - - path - type: object - type: array - modules: - description: CloudInitModules field contains the optional cloud-init - modules which are supported by OSM - properties: - bootcmd: - description: BootCMD module runs arbitrary commands very early - in the boot process, only slightly after a boothook would - run. - items: - type: string - type: array - rh_subscription: - additionalProperties: - type: string - description: RHSubscription registers a Red Hat system either - by username and password or activation and org + type: array + name: + description: Name of the Container runtime + enum: + - docker + - containerd + type: string + templates: + additionalProperties: + type: string + description: + Templates to add to the available templates + when the containerRuntime is selected + type: object + required: + - files + - name type: object - runcmd: - description: RunCMD Run arbitrary commands at a rc.local like - level with output to the console. - items: - type: string - type: array - yum_repo_dir: - description: 'YumRepoDir the repo parts directory where individual - yum repo config files will be written. Default: /etc/yum.repos.d' + type: array + templates: + additionalProperties: type: string - yum_repos: - additionalProperties: - additionalProperties: + description: Templates to be included in units and files + type: object + units: + description: + Units a list of the systemd unit files which will + run on the instance + items: + description: + Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. type: string - type: object - description: YumRepos adds yum repository configuration to - the system. + dropIns: + description: DropIns is a list of drop-ins for this unit. + items: + description: + DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object + type: array + enable: + description: + Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: + Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. + type: string + required: + - name type: object - type: object - supportedContainerRuntimes: - description: SupportedContainerRuntimes represents the container - runtimes supported by the given OS - items: - description: ContainerRuntimeSpec aggregates information about - a specific container runtime - properties: - files: - description: Files to add to the main files list when the - containerRuntime is selected - items: - description: File is a file that should get written to - the host's file system. The content can either be inlined - or referenced from a secret in the same namespace. + type: array + type: object + osName: + description: "OSType represent the operating system name e.g: ubuntu" + enum: + - flatcar + - rhel + - centos + - ubuntu + - amzn2 + - rockylinux + type: string + osVersion: + description: OSVersion the version of the operating system + type: string + provisioningConfig: + description: + ProvisioningConfig is used for provisioning the worker + node. + properties: + files: + description: + Files is a list of files that should exist in the + instance + items: + description: + File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. properties: - content: - description: Content describe the file's content. + inline: + description: + Inline is a struct that contains information + about the inlined data. properties: - inline: - description: Inline is a struct that contains - information about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: Encoding is the file's encoding - (e.g. base64). - type: string - required: - - data - type: object + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding (e.g. + base64). + type: string + required: + - data type: object - path: - description: Path is the path of the file system where - the file should get written to. - type: string - permissions: - default: 644 - description: Permissions describes with which permissions - the file should get written to the file system. - Should be in decimal base and without any leading - zeroes. - format: int32 - type: integer - required: - - content - - path type: object + path: + description: + Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + modules: + description: + CloudInitModules field contains the optional cloud-init + modules which are supported by OSM + properties: + bootcmd: + description: + BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string type: array - name: - description: Name of the Container runtime - enum: - - docker - - containerd - type: string - templates: + rh_subscription: additionalProperties: type: string - description: Templates to add to the available templates - when the containerRuntime is selected + description: + RHSubscription registers a Red Hat system either + by username and password or activation and org type: object - required: - - files - - name - type: object - type: array - templates: - additionalProperties: - type: string - description: Templates to be included in units and files - type: object - units: - description: Units a list of the systemd unit files which will - run on the instance - items: - description: Unit is a systemd unit used for the operating system - config. - properties: - content: - description: Content is the unit's content. - type: string - dropIns: - description: DropIns is a list of drop-ins for this unit. + runcmd: + description: + RunCMD Run arbitrary commands at a rc.local like + level with output to the console. items: - description: DropIn is a drop-in configuration for a systemd - unit. - properties: - content: - description: Content is the content of the drop-in. - type: string - name: - description: Name is the name of the drop-in. - type: string - required: - - content - - name - type: object + type: string type: array - enable: - description: Enable describes whether the unit is enabled - or not. - type: boolean - mask: - description: Mask describes whether the unit is masked or - not. - type: boolean - name: - description: Name is the name of a unit. + yum_repo_dir: + description: + "YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d" type: string - required: - - name + yum_repos: + additionalProperties: + additionalProperties: + type: string + type: object + description: + YumRepos adds yum repository configuration to + the system. + type: object type: object - type: array - type: object - provisioningUtility: - default: cloud-init - description: ProvisioningUtility used for configuring the worker node. - Defaults to cloud-init. - enum: - - cloud-init - - ignition - type: string - supportedCloudProviders: - description: SupportedCloudProviders represent the cloud providers - that support the given operating system version - items: - description: CloudProviderSpec contains the os/image reference for - a specific supported cloud provider - properties: - name: - description: Name represents the name of the supported cloud - provider - enum: - - aws - - azure - - digitalocean - - gce - - hetzner - - kubevirt - - linode - - nutanix - - openstack - - equinixmetal - - vsphere - - fake - - alibaba - - anexia - - scaleway - - baremetal - - external - - vmware-cloud-director - - opennebula - type: string - spec: - description: Spec represents the os/image reference in the supported - cloud provider + supportedContainerRuntimes: + description: + SupportedContainerRuntimes represents the container + runtimes supported by the given OS + items: + description: + ContainerRuntimeSpec aggregates information about + a specific container runtime + properties: + files: + description: + Files to add to the main files list when the + containerRuntime is selected + items: + description: + File is a file that should get written to + the host's file system. The content can either be inlined + or referenced from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: + Inline is a struct that contains + information about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: + Encoding is the file's encoding + (e.g. base64). + type: string + required: + - data + type: object + type: object + path: + description: + Path is the path of the file system where + the file should get written to. + type: string + permissions: + default: 644 + description: + Permissions describes with which permissions + the file should get written to the file system. + Should be in decimal base and without any leading + zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + name: + description: Name of the Container runtime + enum: + - docker + - containerd + type: string + templates: + additionalProperties: + type: string + description: + Templates to add to the available templates + when the containerRuntime is selected + type: object + required: + - files + - name + type: object + type: array + templates: + additionalProperties: + type: string + description: Templates to be included in units and files type: object - x-kubernetes-preserve-unknown-fields: true - required: - - name + units: + description: + Units a list of the systemd unit files which will + run on the instance + items: + description: + Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. + items: + description: + DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object + type: array + enable: + description: + Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: + Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. + type: string + required: + - name + type: object + type: array type: object - type: array - version: - description: Version is the version of the operating System Profile - pattern: v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$ - type: string - required: - - bootstrapConfig - - osName - - osVersion - - provisioningConfig - - supportedCloudProviders - - version - type: object - required: - - spec - type: object - served: true - storage: true + provisioningUtility: + default: cloud-init + description: + ProvisioningUtility used for configuring the worker node. + Defaults to cloud-init. + enum: + - cloud-init + - ignition + type: string + supportedCloudProviders: + description: + SupportedCloudProviders represent the cloud providers + that support the given operating system version + items: + description: + CloudProviderSpec contains the os/image reference for + a specific supported cloud provider + properties: + name: + description: + Name represents the name of the supported cloud + provider + enum: + - aws + - azure + - digitalocean + - gce + - hetzner + - kubevirt + - linode + - nutanix + - openstack + - equinixmetal + - vsphere + - fake + - alibaba + - anexia + - scaleway + - baremetal + - external + - vmware-cloud-director + - opennebula + type: string + spec: + description: + Spec represents the os/image reference in the supported + cloud provider + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - name + type: object + type: array + version: + description: Version is the version of the operating System Profile + pattern: v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$ + type: string + required: + - bootstrapConfig + - osName + - osVersion + - provisioningConfig + - supportedCloudProviders + - version + type: object + required: + - spec + type: object + served: true + storage: true --- apiVersion: cert-manager.io/v1 kind: Issuer @@ -1323,8 +1427,8 @@ spec: name: operating-system-manager command: - /usr/local/bin/osm-controller - - -logtostderr - - -v=4 + - -log-debug=false + - -log-format=json - -worker-count=5 - -cluster-dns=10.10.10.10 - -metrics-address=0.0.0.0:8080 diff --git a/examples/scaleway-machinedeployment.yaml b/examples/scaleway-machinedeployment.yaml index 9a14eb3cc..b996f3f0c 100644 --- a/examples/scaleway-machinedeployment.yaml +++ b/examples/scaleway-machinedeployment.yaml @@ -61,4 +61,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/vmware-cloud-director-machinedeployment.yaml b/examples/vmware-cloud-director-machinedeployment.yaml index eb342dceb..40d076a8e 100644 --- a/examples/vmware-cloud-director-machinedeployment.yaml +++ b/examples/vmware-cloud-director-machinedeployment.yaml @@ -89,4 +89,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml index 58d93c5ef..1bae0b162 100644 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ b/examples/vsphere-datastore-cluster-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index 368f1f16a..4e1bb6cef 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/examples/vultr-machinedeployment.yaml b/examples/vultr-machinedeployment.yaml index 89ecbb0c9..03e2b7f07 100644 --- a/examples/vultr-machinedeployment.yaml +++ b/examples/vultr-machinedeployment.yaml @@ -50,13 +50,13 @@ spec: plan: "vhf-8c-32gb" # This takes precedence over enableVPC vpcId: - - + - # For more reference, see # https://www.vultr.com/api/#tag/instances/operation/create-instance enableVPC: false enableVPC2: true vpc2Id: - - + - # Required: app_id, image_id, os_id, snapshot_id, or iso_id must be provided. Currently only os_id is supported. # This takes precedence over operatingSystem osId: 215 @@ -70,4 +70,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.26.6 + kubelet: 1.28.5 diff --git a/go.mod b/go.mod index 76c3b1ed9..22c6d6539 100644 --- a/go.mod +++ b/go.mod @@ -1,77 +1,79 @@ module github.com/kubermatic/machine-controller -go 1.20 +go 1.21 + +toolchain go1.21.5 require ( - cloud.google.com/go/logging v1.8.1 - cloud.google.com/go/monitoring v1.15.1 + cloud.google.com/go/logging v1.9.0 + cloud.google.com/go/monitoring v1.17.0 github.com/Azure/azure-sdk-for-go v65.0.0+incompatible github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/BurntSushi/toml v1.3.2 github.com/Masterminds/semver/v3 v3.2.1 github.com/Masterminds/sprig/v3 v3.2.3 - github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20230725113508-e18d1b6d4ff8 - github.com/aliyun/alibaba-cloud-sdk-go v1.62.512 - github.com/aws/aws-sdk-go-v2 v1.20.1 - github.com/aws/aws-sdk-go-v2/config v1.18.33 - github.com/aws/aws-sdk-go-v2/credentials v1.13.32 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.112.0 - github.com/aws/aws-sdk-go-v2/service/sts v1.21.2 - github.com/aws/smithy-go v1.14.1 + github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240104224209-592a2a64f555 + github.com/aliyun/alibaba-cloud-sdk-go v1.62.654 + github.com/aws/aws-sdk-go-v2 v1.24.1 + github.com/aws/aws-sdk-go-v2/config v1.26.3 + github.com/aws/aws-sdk-go-v2/credentials v1.16.14 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.142.1 + github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 + github.com/aws/smithy-go v1.19.0 github.com/davecgh/go-spew v1.1.1 - github.com/digitalocean/godo v1.102.0 + github.com/digitalocean/godo v1.107.0 github.com/flatcar/container-linux-config-transpiler v0.9.4 - github.com/go-logr/logr v1.3.0 - github.com/go-logr/zapr v1.2.4 + github.com/go-logr/logr v1.4.1 + github.com/go-logr/zapr v1.3.0 github.com/go-test/deep v1.0.8 - github.com/google/uuid v1.3.1 - github.com/gophercloud/gophercloud v1.5.0 + github.com/google/uuid v1.5.0 + github.com/gophercloud/gophercloud v1.8.0 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb github.com/hetznercloud/hcloud-go v1.39.0 - github.com/linode/linodego v1.20.1 + github.com/linode/linodego v1.26.0 github.com/nutanix-cloud-native/prism-go-client v0.3.4 - github.com/packethost/packngo v0.30.0 + github.com/packethost/packngo v0.31.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 - github.com/prometheus/client_golang v1.16.0 - github.com/scaleway/scaleway-sdk-go v1.0.0-beta.20 + github.com/prometheus/client_golang v1.18.0 + github.com/scaleway/scaleway-sdk-go v1.0.0-beta.22 github.com/sethvargo/go-password v0.2.0 github.com/spf13/pflag v1.0.5 github.com/tinkerbell/tink v0.8.0 github.com/vmware/go-vcloud-director/v2 v2.21.0 - github.com/vmware/govmomi v0.30.7 - github.com/vultr/govultr/v3 v3.3.1 - go.anx.io/go-anxcloud v0.5.3 - go.uber.org/zap v1.25.0 + github.com/vmware/govmomi v0.34.2 + github.com/vultr/govultr/v3 v3.6.0 + go.anx.io/go-anxcloud v0.6.2 + go.uber.org/zap v1.26.0 golang.org/x/crypto v0.17.0 - golang.org/x/oauth2 v0.11.0 + golang.org/x/oauth2 v0.15.0 gomodules.xyz/jsonpatch/v2 v2.4.0 - google.golang.org/api v0.137.0 - google.golang.org/grpc v1.59.0 + google.golang.org/api v0.155.0 + google.golang.org/grpc v1.60.1 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.28.0 - k8s.io/apiextensions-apiserver v0.28.0 - k8s.io/apimachinery v0.28.0 - k8s.io/client-go v0.28.0 - k8s.io/cloud-provider v0.28.0 + k8s.io/api v0.29.0 + k8s.io/apiextensions-apiserver v0.29.0 + k8s.io/apimachinery v0.29.0 + k8s.io/client-go v0.29.0 + k8s.io/cloud-provider v0.29.0 k8s.io/klog v1.0.0 - k8s.io/kubelet v0.28.0 - k8s.io/utils v0.0.0-20230726121419-3b25d923346b - kubevirt.io/api v1.0.0 - kubevirt.io/containerized-data-importer-api v1.57.0 - sigs.k8s.io/controller-runtime v0.15.1 - sigs.k8s.io/yaml v1.3.0 + k8s.io/kubelet v0.29.0 + k8s.io/utils v0.0.0-20240102154912-e7106e64919e + kubevirt.io/api v1.1.1 + kubevirt.io/containerized-data-importer-api v1.58.0 + sigs.k8s.io/controller-runtime v0.16.3 + sigs.k8s.io/yaml v1.4.0 ) require ( - cloud.google.com/go v0.110.7 // indirect - cloud.google.com/go/compute v1.23.0 // indirect + cloud.google.com/go v0.111.0 // indirect + cloud.google.com/go/compute v1.23.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/longrunning v0.5.1 // indirect + cloud.google.com/go/longrunning v0.5.4 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.29 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect @@ -84,32 +86,35 @@ require ( github.com/PaesslerAG/gval v1.2.2 // indirect github.com/PaesslerAG/jsonpath v0.1.1 // indirect github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect - github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect + github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.13.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/docker/distribution v2.8.2+incompatible // indirect - github.com/emicklei/go-restful/v3 v3.10.2 // indirect - github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/distribution/reference v0.5.0 // indirect + github.com/docker/distribution v2.8.3+incompatible // indirect + github.com/emicklei/go-restful/v3 v3.11.1 // indirect + github.com/evanphx/json-patch v5.7.0+incompatible // indirect + github.com/evanphx/json-patch/v5 v5.7.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/flatcar/ignition v0.36.2 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.20.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.4 // indirect - github.com/go-resty/resty/v2 v2.7.0 // indirect + github.com/go-openapi/jsonpointer v0.20.2 // indirect + github.com/go-openapi/jsonreference v0.20.4 // indirect + github.com/go-openapi/swag v0.22.7 // indirect + github.com/go-resty/resty/v2 v2.11.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect @@ -120,11 +125,11 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/s2a-go v0.1.5 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-retryablehttp v0.7.4 // indirect + github.com/hashicorp/go-retryablehttp v0.7.5 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.16 // indirect @@ -136,58 +141,60 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/ginkgo/v2 v2.10.0 // indirect - github.com/onsi/gomega v1.27.8 // indirect + github.com/onsi/ginkgo/v2 v2.13.2 // indirect + github.com/onsi/gomega v1.30.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/openshift/api v0.0.0-20230815201604-a2362cf53230 // indirect + github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046 // indirect github.com/peterhellberg/link v1.2.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect - github.com/spf13/cast v1.5.1 // indirect - github.com/spf13/cobra v1.7.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/cobra v1.8.0 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 // indirect - go.opentelemetry.io/otel v1.20.0 // indirect - go.opentelemetry.io/otel/metric v1.20.0 // indirect - go.opentelemetry.io/otel/trace v1.20.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect + go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.21.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.9.3 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/time v0.5.0 // indirect + golang.org/x/tools v0.16.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/component-base v0.28.0 // indirect - k8s.io/klog/v2 v2.100.1 // indirect - k8s.io/kube-openapi v0.0.0-20230811205723-7ac0aad8c58d // indirect + k8s.io/component-base v0.29.0 // indirect + k8s.io/klog/v2 v2.110.1 // indirect + k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/go.sum b/go.sum index f4dac3bd9..2956f758d 100644 --- a/go.sum +++ b/go.sum @@ -18,27 +18,28 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.110.7 h1:rJyC7nWRg2jWGZ4wSJ5nY65GTdYJkg0cd/uXb+ACI6o= -cloud.google.com/go v0.110.7/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.111.0 h1:YHLKNupSD1KqjDbQ3+LVdQ81h/UJbJyZG203cEfnQgM= +cloud.google.com/go v0.111.0/go.mod h1:0mibmpKP1TyOOFYQY5izo0LnT+ecvOQ0Sg3OdmMiNRU= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= -cloud.google.com/go/logging v1.8.1 h1:26skQWPeYhvIasWKm48+Eq7oUqdcdbwsCVwz5Ys0FvU= -cloud.google.com/go/logging v1.8.1/go.mod h1:TJjR+SimHwuC8MZ9cjByQulAMgni+RkXeI3wwctHJEI= -cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= -cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= -cloud.google.com/go/monitoring v1.15.1 h1:65JhLMd+JiYnXr6j5Z63dUYCuOg770p8a/VC+gil/58= -cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= +cloud.google.com/go/logging v1.9.0 h1:iEIOXFO9EmSiTjDmfpbRjOxECO7R8C7b8IXUGOj7xZw= +cloud.google.com/go/logging v1.9.0/go.mod h1:1Io0vnZv4onoUnsVUQY3HZ3Igb1nBchky0A0y7BBBhE= +cloud.google.com/go/longrunning v0.5.4 h1:w8xEcbZodnA2BbW6sVirkkoC+1gP8wS57EUUgGS0GVg= +cloud.google.com/go/longrunning v0.5.4/go.mod h1:zqNVncI0BOP8ST6XQD1+VcvuShMmq7+xFSzOL++V0dI= +cloud.google.com/go/monitoring v1.17.0 h1:blrdvF0MkPPivSO041ihul7rFMhXdVp8Uq7F59DKXTU= +cloud.google.com/go/monitoring v1.17.0/go.mod h1:KwSsX5+8PnXv5NJnICZzW2R8pWTis8ypC4zmdRD63Tw= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -86,6 +87,7 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= @@ -95,8 +97,8 @@ github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20230725113508-e18d1b6d4ff8 h1:hzczEtHROO5bxA9QWva112sguv3REVxKWq5oTY5biOE= -github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20230725113508-e18d1b6d4ff8/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= +github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240104224209-592a2a64f555 h1:kWb9OISprBC94fTeagHWzz+TQOx5IrwQOY88JyEVNjc= +github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240104224209-592a2a64f555/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= github.com/PaesslerAG/gval v1.2.2 h1:Y7iBzhgE09IGTt5QgGQ2IdaYYYOU134YGHBThD+wm9E= github.com/PaesslerAG/gval v1.2.2/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac= @@ -111,16 +113,17 @@ github.com/agnivade/levenshtein v1.1.0/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVb github.com/ajeddeloh/go-json v0.0.0-20160803184958-73d058cf8437/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 h1:4SPQljF/GJ8Q+QlCWMWxRBepub4DresnOm4eI2ebFGc= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.512 h1:1r0pVpVs5XrgibXeBW7SwD9kU9ceTEq+LHjPEwxkKU0= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.512/go.mod h1:Api2AkmMgGaSUAhmk76oaFObkoeCPc/bKAqcyplPODs= +github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs= +github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.654 h1:UpBbuyd0eqDkIfiuRmBGqdjXWd4Q7YwD9entykxwlnI= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.654/go.mod h1:CJJYa1ZMxjlN/NbXEwmejEnBkhi0DV+Yb3B2lxf+74o= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -131,34 +134,35 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/aws/aws-sdk-go v1.42.23/go.mod h1:gyRszuZ/icHmHAVE4gc/r+cfCmhA1AD+vqfWbgI+eHs= -github.com/aws/aws-sdk-go-v2 v1.20.1 h1:rZBf5DWr7YGrnlTK4kgDQGn1ltqOg5orCYb/UhOFZkg= -github.com/aws/aws-sdk-go-v2 v1.20.1/go.mod h1:NU06lETsFm8fUC6ZjhgDpVBcGZTFQ6XM+LZWZxMI4ac= -github.com/aws/aws-sdk-go-v2/config v1.18.33 h1:JKcw5SFxFW/rpM4mOPjv0VQ11E2kxW13F3exWOy7VZU= -github.com/aws/aws-sdk-go-v2/config v1.18.33/go.mod h1:hXO/l9pgY3K5oZJldamP0pbZHdPqqk+4/maa7DSD3cA= -github.com/aws/aws-sdk-go-v2/credentials v1.13.32 h1:lIH1eKPcCY1ylR4B6PkBGRWMHO3aVenOKJHWiS4/G2w= -github.com/aws/aws-sdk-go-v2/credentials v1.13.32/go.mod h1:lL8U3v/Y79YRG69WlAho0OHIKUXCyFvSXaIvfo81sls= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8 h1:DK/9C+UN/X+1+Wm8pqaDksQr2tSLzq+8X1/rI/ZxKEQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.8/go.mod h1:ce7BgLQfYr5hQFdy67oX2svto3ufGtm6oBvmsHScI1Q= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 h1:c8ed/T9T2K5I+h/JzmF5tpI46+OODQ74dzmdo+QnaMg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38/go.mod h1:qggunOChCMu9ZF/UkAfhTz25+U2rLVb3ya0Ua6TTfCA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 h1:hNeAAymUY5gu11WrrmFb3CVIp9Dar9hbo44yzzcQpzA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32/go.mod h1:0ZXSqrty4FtQ7p8TEuRde/SZm9X05KT18LAUlR40Ln0= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39 h1:fc0ukRAiP1syoSGZYu+DaE+FulSYhTiJ8WpVu5jElU4= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.39/go.mod h1:WLAW8PT7+JhjZfLSWe7WEJaJu0GNo0cKc2Zyo003RBs= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.112.0 h1:8I4NQ9BfrQATHzXKtBuu+jBdOVd2mBANqhbMOXfSIdA= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.112.0/go.mod h1:Ie0Kp61cLk223argiS+t8vO29SpbFIphzlPflIvYcv0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32 h1:dGAseBFEYxth10V23b5e2mAS+tX7oVbfYHD6dnDdAsg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.32/go.mod h1:4jwAWKEkCR0anWk5+1RbfSg1R5Gzld7NLiuaq5bTR/Y= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.2 h1:A2RlEMo4SJSwbNoUUgkxTAEMduAy/8wG3eB2b2lP4gY= -github.com/aws/aws-sdk-go-v2/service/sso v1.13.2/go.mod h1:ju+nNXUunfIFamXUIZQiICjnO/TPlOmWcYhZcSy7xaE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2 h1:OJELEgyaT2kmaBGZ+myyZbTTLobfe3ox3FSh5eYK9Qs= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.2/go.mod h1:ubDBBaDFs1GHijSOTi8ljppML15GLG0HxhILtbjNNYQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.21.2 h1:ympg1+Lnq33XLhcK/xTG4yZHPs1Oyxu+6DEWbl7qOzA= -github.com/aws/aws-sdk-go-v2/service/sts v1.21.2/go.mod h1:FQ/DQcOfESELfJi5ED+IPPAjI5xC6nxtSolVVB773jM= -github.com/aws/smithy-go v1.14.1 h1:EFKMUmH/iHMqLiwoEDx2rRjRQpI1YCn5jTysoaDujFs= -github.com/aws/smithy-go v1.14.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= +github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= +github.com/aws/aws-sdk-go-v2/config v1.26.3 h1:dKuc2jdp10y13dEEvPqWxqLoc0vF3Z9FC45MvuQSxOA= +github.com/aws/aws-sdk-go-v2/config v1.26.3/go.mod h1:Bxgi+DeeswYofcYO0XyGClwlrq3DZEXli0kLf4hkGA0= +github.com/aws/aws-sdk-go-v2/credentials v1.16.14 h1:mMDTwwYO9A0/JbOCOG7EOZHtYM+o7OfGWfu0toa23VE= +github.com/aws/aws-sdk-go-v2/credentials v1.16.14/go.mod h1:cniAUh3ErQPHtCQGPT5ouvSAQ0od8caTO9OOuufZOAE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.142.1 h1:tTAfm9YsKlmlv6ORgco838e0ZeAcGVRkgevseiYO0gU= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.142.1/go.mod h1:hIsHE0PaWAQakLCshKS7VKWMGXaqrAFp4m95s2W9E6c= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 h1:dGrs+Q/WzhsiUKh82SfTVN66QzyulXuMDTV/G8ZxOac= +github.com/aws/aws-sdk-go-v2/service/sso v1.18.6/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 h1:Yf2MIo9x+0tyv76GljxzqA3WtC5mw7NmazD2chwjxE4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= +github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U= +github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= +github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -186,6 +190,7 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/go-semver v0.1.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= @@ -195,7 +200,7 @@ github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pq github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creasty/defaults v1.5.2 h1:/VfB6uxpyp6h0fr7SPp7n8WJBoV8jfxQXPCnkVSjyls= github.com/creasty/defaults v1.5.2/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY= @@ -205,21 +210,23 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= -github.com/digitalocean/godo v1.102.0 h1:iXkLEHmrBi5n9TAbRcKM0182NwXYrp1xCj//7vMRFs4= -github.com/digitalocean/godo v1.102.0/go.mod h1:SaUYccN7r+CO1QtsbXGypAsgobDrmSfVMJESEfXgoEg= +github.com/digitalocean/godo v1.107.0 h1:P72IbmGFQvKOvyjVLyT59bmHxilA4E5hWi40rF4zNQc= +github.com/digitalocean/godo v1.107.0/go.mod h1:R6EmmWI8CT1+fCtjWY9UCB+L5uufuZH13wk3YhxycCs= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= +github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= +github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= -github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= +github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE= -github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.11.1 h1:S+9bSbua1z3FgCnV0KKOSSZ3mDthb5NyEPL5gEpCvyk= +github.com/emicklei/go-restful/v3 v3.11.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -229,22 +236,27 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= -github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= -github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc= +github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flatcar/container-linux-config-transpiler v0.9.4 h1:yXQ0NB8PeNrKJPrZvbv5/DV63PNhTqt8vaf8YxmX/RA= github.com/flatcar/container-linux-config-transpiler v0.9.4/go.mod h1:LxanhPvXkWgHG9PrkT4rX/p7YhUPdDGGsUdkNpV3L5U= github.com/flatcar/ignition v0.36.2 h1:xGHgScUe0P4Fkprjqv7L2CE58emiQgP833OCCn9z2v4= github.com/flatcar/ignition v0.36.2/go.mod h1:uk1tpzLFRXus4RrvzgMI+IqmmB8a/RGFSBlI+tMTbbA= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -270,30 +282,28 @@ github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= -github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= -github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= -github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= +github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= +github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= +github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.7 h1:JWrc1uc/P9cSomxfnsFSVWoE1FW6bNbrVPmpQYpCcR8= +github.com/go-openapi/swag v0.22.7/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= @@ -304,8 +314,8 @@ github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-playground/validator/v10 v10.10.1 h1:uA0+amWMiglNZKZ9FJRKUAe9U3RX91eVn1JYXMWt7ig= github.com/go-playground/validator/v10 v10.10.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= -github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= -github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= +github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= +github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= @@ -323,6 +333,7 @@ github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -403,24 +414,24 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.5 h1:8IYp3w9nysqv3JH+NJgXJzGbDHzLOTj43BmSkp+O7qg= -github.com/google/s2a-go v0.1.5/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= -github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gophercloud/gophercloud v1.5.0 h1:cDN6XFCLKiiqvYpjQLq9AiM7RDRbIC9450WpPH+yvXo= -github.com/gophercloud/gophercloud v1.5.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= +github.com/gophercloud/gophercloud v1.8.0 h1:TM3Jawprb2NrdOnvcHhWJalmKmAmOGgfZElM/3oBYCk= +github.com/gophercloud/gophercloud v1.8.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -433,8 +444,8 @@ github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9n github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= -github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= +github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -456,7 +467,6 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -478,6 +488,7 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/keploy/go-sdk v0.4.3 h1:dCsmfANlZH94It+JKWx8/JEEC6dn8W7KIRRKRZwCPZQ= @@ -515,8 +526,8 @@ github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++ github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= github.com/lestrrat-go/jwx v1.2.20/go.mod h1:tLE1XszaFgd7zaS5wHe4NxA+XVhu7xgdRvDpNyi3kNM= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/linode/linodego v1.20.1 h1:IW3SrZjRzrclYZnzFd80f8lSkTYAM7gVTJW0t7HnFKQ= -github.com/linode/linodego v1.20.1/go.mod h1:ggoWnJXssx9wPWNnR3x7WaOpOBOEhsPB/HO7iflF5qY= +github.com/linode/linodego v1.26.0 h1:2tOZ3Wxn4YvGBRgZi3Vz6dab+L16XUntJ9sJxh3ZBio= +github.com/linode/linodego v1.26.0/go.mod h1:kD7Bf1piWg/AXb9TA0ThAVwzR+GPf6r2PvbTbVk7PMA= github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -533,8 +544,8 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= @@ -575,27 +586,27 @@ github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042 github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.10.0 h1:sfUl4qgLdvkChZrWCYndY2EAu9BRIw1YphNAzy1VNWs= -github.com/onsi/ginkgo/v2 v2.10.0/go.mod h1:UDQOh5wbQUlMnkLfVaIUMtQ1Vus92oM+P2JX1aulgcE= +github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs= +github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= -github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= +github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/openshift/api v0.0.0-20230815201604-a2362cf53230 h1:PY2JBJdSkzTxathfsYMa/Mb2Xcw9YphTZ6IcvURayKk= -github.com/openshift/api v0.0.0-20230815201604-a2362cf53230/go.mod h1:yimSGmjsI+XF1mr+AKBs2//fSXIOhhetHGbMlBEfXbs= +github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f h1:3BMVfQpz1xe8MmJprp1+NL8hrpl9I04JVP9EczdCOqE= +github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= -github.com/packethost/packngo v0.30.0 h1:JVeTwbXXETsLTDQncUbYwIFpkOp/xevXrffM2HrFECI= -github.com/packethost/packngo v0.30.0/go.mod h1:BT/XcdwLVmeMtGPbovnxCpnI1s9ylSE1cs/7pq007NE= +github.com/packethost/packngo v0.31.0 h1:LLH90ardhULWbagBIc3I3nl2uU75io0a7AwY6hyi0S4= +github.com/packethost/packngo v0.31.0/go.mod h1:Io6VJqzkiqmIEQbpOjeIw9v8q9PfcTEq8TEY/tMQsfw= github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046 h1:zF+CUhv8LMpqTFFpECX6WF+yUWS2Bd1Nc1W+AczzqbY= github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046/go.mod h1:W/xTaqgJ2kJCwayvm3BF3bOj9ku0F5DjjYnZaioxnOk= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= @@ -618,27 +629,27 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.31.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -646,16 +657,17 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rollbar/rollbar-go v1.4.2/go.mod h1:kLQ9gP3WCRGrvJmF0ueO3wK9xWocej8GRX98D8sa39w= github.com/rollbar/rollbar-go/errors v0.0.0-20210929193720-32947096267e/go.mod h1:Ie0xEc1Cyj+T4XMO8s0Vf7pMfvSAAy1sb4AYc8aJsao= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.20 h1:a9hSJdJcd16e0HoMsnFvaHvxB3pxSD+SC7+CISp7xY0= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.20/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= +github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.22 h1:wJrcTdddKOI8TFxs8cemnhKP2EmKy3yfUKHj3ZdfzYo= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.22/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI= @@ -674,10 +686,10 @@ github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.5.1 h1:R+kOtfhWQE6TVQzY+4D7wJLBgkdVasCEFxSUBYBYIlA= -github.com/spf13/cast v1.5.1/go.mod h1:b9PdjNptOpzXr7Rq1q9gJML/2cdGQAo69NKzQ10KN48= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= @@ -697,6 +709,7 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinkerbell/lint-install v0.0.0-20211012174934-5ee5ab01db76/go.mod h1:0h2KsALaQLNkoVeV+G+HjBWWCnp0COFYhJdRd5WCQPM= github.com/tinkerbell/tink v0.8.0 h1:qgl/rglpO5Rvq6UKZd29O6X9mDgZZYgf841+Y0IYWak= @@ -719,12 +732,12 @@ github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8A github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vmware/go-vcloud-director/v2 v2.21.0 h1:zIONrJpM+Fj+rDyXmsRfMAn1sP5WAP87USL0T9GS4DY= github.com/vmware/go-vcloud-director/v2 v2.21.0/go.mod h1:QPxGFgrUcSyzy9IlpwDE4UNT3tsOy2047tJOPEJ4nlw= -github.com/vmware/govmomi v0.30.7 h1:YO8CcDpLJzmq6PK5/CBQbXyV21iCMh8SbdXt+xNkXp8= -github.com/vmware/govmomi v0.30.7/go.mod h1:epgoslm97rLECMV4D+08ORzUBEU7boFSepKjt7AYVGg= +github.com/vmware/govmomi v0.34.2 h1:o6ydkTVITOkpQU6HAf6tP5GvHFCNJlNUNlMsvFK77X4= +github.com/vmware/govmomi v0.34.2/go.mod h1:qWWT6n9mdCr/T9vySsoUqcI04sSEj4CqHXxtk/Y+Los= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= -github.com/vultr/govultr/v3 v3.3.1 h1:gn46pSL0A3pxBKZklzwtYgSW2Iq7dW1euDRBlOsIzTo= -github.com/vultr/govultr/v3 v3.3.1/go.mod h1:7NjuHeQv5vgUWR2H1sPc9D+xffrT5ql+kNi6R3yuwzo= +github.com/vultr/govultr/v3 v3.6.0 h1:WCXQwgdiZnGxG4CI+TTohE14V3jV6ikg/64fhDVdbIs= +github.com/vultr/govultr/v3 v3.6.0/go.mod h1:rt9v2x114jZmmLAE/h5N5jnxTmsK9ewwS2oQZ0UBQzM= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= @@ -737,8 +750,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.anx.io/go-anxcloud v0.5.3 h1:ZKdHC2QaVvN6VdzT1O/aBxkabE/OoNJNLNpOyRzLqcw= -go.anx.io/go-anxcloud v0.5.3/go.mod h1:XTIg137l88FfE42X0/aOgvSm/fcXxWVp8k4+x9//zaQ= +go.anx.io/go-anxcloud v0.6.2 h1:4FV9xtjilRny/TEBeqsqOPxm1i9UPlPjHRDK86fhFjc= +go.anx.io/go-anxcloud v0.6.2/go.mod h1:TW0KcKa1hlYEwCQ2YAFec07xtfX60psI/dmjJqRdmjY= go.keploy.io/server v0.1.8 h1:b50vAt1+WKMscYVP5Bm8gx/iSaR7mpHox8VpaxjrQ88= go.keploy.io/server v0.1.8/go.mod h1:ZqhwTZOBb+dzx5t30Wt6eUGI6kO5QizvPg6coNPtbow= go.mongodb.org/mongo-driver v1.8.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= @@ -753,16 +766,20 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0 h1:PzIubN4/sjByhDRHLviCjJuweBXWFZWhghjg7cS28+M= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.0/go.mod h1:Ct6zzQEuGK3WpJs2n4dn+wfJYzd/+hNnxMRTWjGn30M= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= -go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= -go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= -go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= -go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= -go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= -go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -773,6 +790,7 @@ go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= @@ -782,9 +800,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= -go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= go4.org v0.0.0-20160314031811-03efcb870d84/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= @@ -803,15 +820,18 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= @@ -821,6 +841,9 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= +golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -848,7 +871,9 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -895,7 +920,6 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -904,8 +928,10 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -919,8 +945,8 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= +golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -933,8 +959,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1006,18 +1033,21 @@ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1030,6 +1060,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1037,11 +1069,14 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1100,14 +1135,19 @@ golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpd golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= +golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1129,16 +1169,17 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.137.0 h1:QrKX6uNvzJLr0Fd3vWVqcyrcmFoYi036VUAsZbiF4+s= -google.golang.org/api v0.137.0/go.mod h1:4xyob8CxC+0GChNBvEUAk8VBKNvYOTWM9T3v3UfRxuY= +google.golang.org/api v0.155.0 h1:vBmGhCYs0djJttDNynWo44zosHlPvHmA0XiN2zP2DtA= +google.golang.org/api v0.155.0/go.mod h1:GI5qK5f40kCpHfPn6+YzGAByIKWv8ujFnmoWm7Igduk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1183,12 +1224,12 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d h1:VBu5YqKPv6XiJ199exd8Br+Aetz+o08F+PLMnwJQHAY= -google.golang.org/genproto v0.0.0-20230822172742-b8732ec3820d/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d h1:DoPTO70H+bcDXcd39vOqb2viZxgqeBeSGtZ55yZU4/Q= -google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d h1:uvYuEyMHKNt+lT4K3bN6fGswmK8qSvcreM3BwjDh+y4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917 h1:nz5NESFLZbJGPFxDT/HCn+V1mZ8JGNoY4nUpmW/Y2eg= +google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917/go.mod h1:pZqR+glSb11aJ+JQcczCvgf47+duRuzNSKqE8YAQnV0= +google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 h1:rcS6EyEaoCO52hQDupoSfrxI3R6C2Tq741is7X8OvnM= +google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917/go.mod h1:CmlNWB9lSezaYELKS5Ym1r44VrrbPUa7JTvw+6MbpJ0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 h1:6G8oQ016D88m1xAKljMlBOOGWDZkes4kMhgGFlf8WcQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917/go.mod h1:xtjpI3tXFPP051KaWnhvxkiubL/6dJ18vLVf7q2pTOU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1211,9 +1252,8 @@ google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/grpc/examples v0.0.0-20210728214646-ad0a2a847cdf/go.mod h1:bF8wuZSAZTcbF7ZPKrDI/qY52toTP/yxLpRRY4Eu9Js= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1229,8 +1269,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1264,6 +1304,7 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1275,23 +1316,23 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= -k8s.io/api v0.28.0 h1:3j3VPWmN9tTDI68NETBWlDiA9qOiGJ7sdKeufehBYsM= -k8s.io/api v0.28.0/go.mod h1:0l8NZJzB0i/etuWnIXcwfIv+xnDOhL3lLW919AWYDuY= -k8s.io/apiextensions-apiserver v0.28.0 h1:CszgmBL8CizEnj4sj7/PtLGey6Na3YgWyGCPONv7E9E= -k8s.io/apiextensions-apiserver v0.28.0/go.mod h1:uRdYiwIuu0SyqJKriKmqEN2jThIJPhVmOWETm8ud1VE= +k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A= +k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA= +k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0= +k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apimachinery v0.28.0 h1:ScHS2AG16UlYWk63r46oU3D5y54T53cVI5mMJwwqFNA= -k8s.io/apimachinery v0.28.0/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= +k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= +k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= -k8s.io/client-go v0.28.0 h1:ebcPRDZsCjpj62+cMk1eGNX1QkMdRmQ6lmz5BLoFWeM= -k8s.io/client-go v0.28.0/go.mod h1:0Asy9Xt3U98RypWJmU1ZrRAGKhP6NqDPmptlAzK2kMc= -k8s.io/cloud-provider v0.28.0 h1:BTIW7b757T+VXB5yqJeajPXsNOmeooopUgfzQueiWvk= -k8s.io/cloud-provider v0.28.0/go.mod h1:u0MGqdlutkTmCJyNrCzIMJ+OhrwQE9x5X8mBTN0R7us= +k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8= +k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38= +k8s.io/cloud-provider v0.29.0 h1:Qgk/jHsSKGRk/ltTlN6e7eaNuuamLROOzVBd0RPp94M= +k8s.io/cloud-provider v0.29.0/go.mod h1:gBCt7YYKFV4oUcJ/0xF9lS/9il4MxKunJ+ZKvh39WGo= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.28.0 h1:HQKy1enJrOeJlTlN4a6dU09wtmXaUvThC0irImfqyxI= -k8s.io/component-base v0.28.0/go.mod h1:Yyf3+ZypLfMydVzuLBqJ5V7Kx6WwDr/5cN+dFjw1FNk= +k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s= +k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= @@ -1302,40 +1343,42 @@ k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= -k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= +k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= -k8s.io/kube-openapi v0.0.0-20230811205723-7ac0aad8c58d h1:lGN9colvFFZIY0Guxkzdd73CoDXHIbhlQkfi7n/26Ak= -k8s.io/kube-openapi v0.0.0-20230811205723-7ac0aad8c58d/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= -k8s.io/kubelet v0.28.0 h1:H/3JAkLIungVF+WLpqrxhgJ4gzwsbN8VA8LOTYsEX3U= -k8s.io/kubelet v0.28.0/go.mod h1:i8jUg4ltbRusT3ExOhSAeqETuHdoHTZcTT2cPr9RTgc= +k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910 h1:1Rp/XEKP5uxPs6QrsngEHAxBjaAR78iJRiJq5Fi7LSU= +k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910/go.mod h1:Pa1PvrP7ACSkuX6I7KYomY6cmMA0Tx86waBhDUgoKPw= +k8s.io/kubelet v0.29.0 h1:SX5hlznTBcGIrS1scaf8r8p6m3e475KMifwt9i12iOk= +k8s.io/kubelet v0.29.0/go.mod h1:kvKS2+Bz2tgDOG1S1q0TH2z1DasNuVF+8p6Aw7xvKkI= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -kubevirt.io/api v1.0.0 h1:RBdXP5CDhE0v5qL2OUQdrYyRrHe/F68Z91GWqBDF6nw= -kubevirt.io/api v1.0.0/go.mod h1:CJ4vZsaWhVN3jNbyc9y3lIZhw8nUHbWjap0xHABQiqc= -kubevirt.io/containerized-data-importer-api v1.57.0 h1:IpRCUyDS0x7BaVa5q5MCzuWRAfvXT54GpEnNJke5hSE= -kubevirt.io/containerized-data-importer-api v1.57.0/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +kubevirt.io/api v1.1.1 h1:vt5bOpACArNFIudx1bcE1VeejQdh5wCd7Oz/uFBIkH8= +kubevirt.io/api v1.1.1/go.mod h1:CJ4vZsaWhVN3jNbyc9y3lIZhw8nUHbWjap0xHABQiqc= +kubevirt.io/containerized-data-importer-api v1.58.0 h1:l6bH2SrCUi14QAi1Mv1vzcrqZI0XYzrV1KLK6hiC0QI= +kubevirt.io/containerized-data-importer-api v1.58.0/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 h1:fZYvD3/Vnitfkx6IJxjLAk8ugnZQ7CXVYcRfkSKmuZY= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.15.1 h1:9UvgKD4ZJGcj24vefUFgZFP3xej/3igL9BsOUTb/+4c= -sigs.k8s.io/controller-runtime v0.15.1/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk= +sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= +sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= -sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index e16f7d27d..c4f81e5c6 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -858,7 +858,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. AccessModes: []corev1.PersistentVolumeAccessMode{ config.StorageAccessType, }, - Resources: corev1.ResourceRequirements{ + Resources: corev1.VolumeResourceRequirements{ Requests: pvcRequest, }, }, @@ -877,7 +877,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. AccessModes: []corev1.PersistentVolumeAccessMode{ config.StorageAccessType, }, - Resources: corev1.ResourceRequirements{ + Resources: corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{corev1.ResourceStorage: sd.Size}, }, }, diff --git a/pkg/node/poddeletion/pod_deletion.go b/pkg/node/poddeletion/pod_deletion.go index 980cb9b2d..af164fda5 100644 --- a/pkg/node/poddeletion/pod_deletion.go +++ b/pkg/node/poddeletion/pod_deletion.go @@ -189,10 +189,8 @@ func (vc *NodeVolumeAttachmentsCleanup) deletePods(ctx context.Context, log *zap } else if kerrors.IsTooManyRequests(err) { // PDB prevents pod deletion, return and make the controller retry later. return - } else { - errCh <- fmt.Errorf("error deleting pod %s/%s on node %s: %w", p.Namespace, p.Name, vc.nodeName, err) - return } + errCh <- fmt.Errorf("error deleting pod %s/%s on node %s: %w", p.Namespace, p.Name, vc.nodeName, err) } }(pod) } diff --git a/pkg/rhsm/subscription_manager_test.go b/pkg/rhsm/subscription_manager_test.go index 088203fe3..64736b951 100644 --- a/pkg/rhsm/subscription_manager_test.go +++ b/pkg/rhsm/subscription_manager_test.go @@ -77,6 +77,9 @@ func createTestingServer(pagination bool) *httptest.Server { processedRequest = 1 result string ) + + const resultPrefix = "{\"pagination\": {\"offset\": 0, \"limit\": 2,\"count\": 5}, \"body\": [" + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case authPath: @@ -86,17 +89,17 @@ func createTestingServer(pagination bool) *httptest.Server { if pagination { switch processedRequest { case 1: - result = "{\"pagination\": {\"offset\": 0, \"limit\": 2,\"count\": 5}, \"body\": [" + + result = resultPrefix + "{\"name\": \"test-machine-1\", \"uuid\": \"4a3ee8d7-337d-4cef-a20c-dda011f28f96\"}," + "{\"name\": \"test-machine-2\", \"uuid\": \"4a3ee8d7-337d-4cef-a20c-dda011f28f91\"}" + "]}" case 2: - result = "{\"pagination\": {\"offset\": 0, \"limit\": 2,\"count\": 5}, \"body\": [" + + result = resultPrefix + "{\"name\": \"test-machine-3\", \"uuid\": \"4a3ee8d7-337d-4cef-a20c-dda011f28f98\"}," + "{\"name\": \"test-machine-4\", \"uuid\": \"4a3ee8d7-337d-4cef-a20c-dda011f28f95\"}" + "]}" case 3: - result = "{\"pagination\": {\"offset\": 0, \"limit\": 2,\"count\": 5}, \"body\": [" + + result = resultPrefix + "{\"name\": \"test-machine-5\", \"uuid\": \"4a3ee8d7-337d-4cef-a20c-dda011f28f99\"}" + "]}" } diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index b74b09282..f4b2f0455 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -172,7 +172,16 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", + Kubelet: "1.28.5", + }, + }, + }, + { + name: "kubelet-v1.29-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.29.0", }, }, }, diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml index 8cc1660ba..7279309d8 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml @@ -321,6 +321,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml index aeb983125..ec34eeec3 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml @@ -321,6 +321,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml index d32468fcb..68dc04d43 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml @@ -321,6 +321,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index 96b3a92a6..60bb2fd32 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -338,6 +338,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index ccee1c339..859c8c46d 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -338,6 +338,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml index c974e8043..742531ab8 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml @@ -329,6 +329,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml index 587763ace..1f2c60b4b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml @@ -321,6 +321,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml index bbcad8969..c489d477a 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml @@ -143,7 +143,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -321,6 +321,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.29-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29-aws.yaml new file mode 100644 index 000000000..61dd8a9bd --- /dev/null +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.29-aws.yaml @@ -0,0 +1,454 @@ +#cloud-config + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + + mkdir -p /etc/systemd/system/containerd.service.d + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMaximumGCAge: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index 57a94030c..e6bd23c03 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -173,7 +173,16 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", + Kubelet: "1.28.5", + }, + }, + }, + { + name: "kubelet-v1.29-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.29.5", }, }, }, diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml index 1ad244638..4b50bd558 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml @@ -327,6 +327,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml index e12e0a730..d983f9261 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml @@ -327,6 +327,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml index 752862e7a..d9e56c166 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml @@ -335,6 +335,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index 5dbae2241..8697d4c3c 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -344,6 +344,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index 0032322de..053554a4b 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -344,6 +344,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml index cdac24c72..f8f811fc0 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml @@ -335,6 +335,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml index 17cc1663b..5cd009103 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml @@ -327,6 +327,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml index 984a3ab78..c043c9718 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -327,6 +327,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml new file mode 100644 index 000000000..e64a46ccb --- /dev/null +++ b/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml @@ -0,0 +1,460 @@ +#cloud-config + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + source /etc/os-release + if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMaximumGCAge: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index eb91a40ef..a787979d1 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -124,7 +124,7 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "ignition_v1.28.0", + name: "ignition_v1.29.0", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -140,7 +140,38 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", + Kubelet: "1.29.0", + }, + }, + ccProvider: &fakeCloudConfigProvider{ + name: "vsphere", + config: "{vsphere-config:true}", + err: nil, + }, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, + osConfig: &Config{ + DisableAutoUpdate: true, + ProvisioningUtility: Ignition, + }, + }, + { + name: "ignition_v1.28.5", + providerSpec: &providerconfigtypes.Config{ + CloudProvider: "vsphere", + SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, + CAPublicKey: "ssh-rsa AAABBB", + Network: &providerconfigtypes.NetworkConfig{ + CIDR: "192.168.81.4/24", + Gateway: "192.168.81.1", + DNS: providerconfigtypes.DNSConfig{ + Servers: []string{"8.8.8.8"}, + }, + }, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.28.5", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -248,7 +279,38 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.28.0", + name: "cloud-init_v1.29.0", + providerSpec: &providerconfigtypes.Config{ + CloudProvider: "anexia", + SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, + CAPublicKey: "ssh-rsa AAABBB", + Network: &providerconfigtypes.NetworkConfig{ + CIDR: "192.168.81.4/24", + Gateway: "192.168.81.1", + DNS: providerconfigtypes.DNSConfig{ + Servers: []string{"8.8.8.8"}, + }, + }, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.29.0", + }, + }, + ccProvider: &fakeCloudConfigProvider{ + name: "anexia", + config: "{anexia-config:true}", + err: nil, + }, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, + osConfig: &Config{ + DisableAutoUpdate: true, + ProvisioningUtility: CloudInit, + }, + }, + { + name: "cloud-init_v1.28.5", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -264,7 +326,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", + Kubelet: "1.28.5", }, }, ccProvider: &fakeCloudConfigProvider{ diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml index 8d8c941e1..cc927c495 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml @@ -195,6 +195,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml index fdc2dc0c1..027f2b4a8 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml @@ -1,520 +1,509 @@ #cloud-config users: -- name: core - ssh_authorized_keys: - - ssh-rsa AAABBB - - ssh-rsa CCCDDD - + - name: core + ssh_authorized_keys: + - ssh-rsa AAABBB + - ssh-rsa CCCDDD coreos: units: - - name: static-nic.network - content: | - [Match] - # Because of difficulty predicting specific NIC names on different cloud providers, - # we only support static addressing on VSphere. There should be a single NIC attached - # that we will match by name prefix 'en' which denotes ethernet devices. - Name=en* - - [Network] - DHCP=no - Address=192.168.81.4/24 - Gateway=192.168.81.1 - DNS=8.8.8.8 - - - name: update-engine.service - command: stop - mask: true - - name: locksmithd.service - command: stop - mask: true - - name: download-script.service - enable: true - command: start - content: | - [Unit] - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/download.sh - [Install] - WantedBy=multi-user.target - - - name: kubelet-healthcheck.service - enable: true - command: start - drop-ins: - - name: 40-download.conf + - name: static-nic.network + content: | + [Match] + # Because of difficulty predicting specific NIC names on different cloud providers, + # we only support static addressing on VSphere. There should be a single NIC attached + # that we will match by name prefix 'en' which denotes ethernet devices. + Name=en* + + [Network] + DHCP=no + Address=192.168.81.4/24 + Gateway=192.168.81.1 + DNS=8.8.8.8 + + - name: update-engine.service + command: stop + mask: true + - name: locksmithd.service + command: stop + mask: true + - name: download-script.service + enable: true + command: start content: | [Unit] - Requires=download-script.service - After=download-script.service - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/download.sh + [Install] + WantedBy=multi-user.target + + - name: kubelet-healthcheck.service + enable: true + command: start + drop-ins: + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet - - name: nodeip.service - enable: true - command: start - content: | - [Unit] - Description=Setup Kubelet Node IP Env - Requires=network-online.target - After=network-online.target + [Install] + WantedBy=multi-user.target - [Service] - ExecStart=/opt/bin/setup_net_env.sh - RemainAfterExit=yes - Type=oneshot - [Install] - WantedBy=multi-user.target - - - name: kubelet.service - enable: true - command: start - drop-ins: - - name: 10-nodeip.conf - content: | - [Service] - EnvironmentFile=/etc/kubernetes/nodeip.conf - - name: resolv.conf + - name: nodeip.service + enable: true + command: start content: | + [Unit] + Description=Setup Kubelet Node IP Env + Requires=network-online.target + After=network-online.target + [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - - name: 40-download.conf + ExecStart=/opt/bin/setup_net_env.sh + RemainAfterExit=yes + Type=oneshot + [Install] + WantedBy=multi-user.target + + - name: kubelet.service + enable: true + command: start + drop-ins: + - name: 10-nodeip.conf + content: | + [Service] + EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: resolv.conf + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + - name: 40-download.conf + content: | + [Unit] + Requires=download-script.service + After=download-script.service + - name: 50-rpc-statd.conf + content: | + [Unit] + Wants=rpc-statd.service content: | [Unit] - Requires=download-script.service - After=download-script.service - - name: 50-rpc-statd.conf + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=anexia \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + + - name: apply-sysctl-settings.service + enable: true + command: start content: | [Unit] - Wants=rpc-statd.service + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + ExecStart=/opt/bin/apply_sysctl_settings.sh + [Install] + WantedBy=multi-user.target + +write_files: + - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + permissions: "0644" + content: | + [Journal] + SystemMaxUse=5G + + - path: "/etc/kubernetes/kubelet.conf" + permissions: "0644" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + - path: /opt/load-kernel-modules.sh + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + - path: /etc/sysctl.d/k8s.conf + permissions: "0644" content: | - [Unit] - After=containerd.service - Requires=containerd.service + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + - path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + - path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" + permissions: "0755" + content: | + # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar + # so we enable IPv6 Router Advertisement here. + # See for details https://github.com/flatcar-linux/Flatcar/issues/384 + [Network] + IPv6AcceptRA=true - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ + - path: /etc/kubernetes/bootstrap-kubelet.conf + permissions: "0400" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + - path: /etc/kubernetes/cloud-config + permissions: "0400" + content: | + {anexia-config:true} + - path: /etc/kubernetes/pki/ca.crt + permissions: "0644" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + - path: /etc/hostname + permissions: "0600" + content: "node1" + + - path: /etc/ssh/sshd_config + permissions: "0600" + user: root + content: | + # Use most defaults for sshd configuration. + Subsystem sftp internal-sftp + ClientAliveInterval 180 + UseDNS no + UsePAM yes + PrintLastLog no # handled by PAM + PrintMotd no # handled by PAM + PasswordAuthentication no + ChallengeResponseAuthentication no + + - path: /opt/bin/download.sh + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v1.2.0}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + mkdir -p /etc/systemd/system/containerd.service.d + + cat < /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: /etc/sysctl.d/k8s.conf - permissions: "0644" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" - permissions: "0755" - content: | - # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar - # so we enable IPv6 Router Advertisement here. - # See for details https://github.com/flatcar-linux/Flatcar/issues/384 - [Network] - IPv6AcceptRA=true - -- path: /etc/kubernetes/bootstrap-kubelet.conf - permissions: "0400" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: /etc/kubernetes/cloud-config - permissions: "0400" - content: | - {anexia-config:true} - -- path: /etc/kubernetes/pki/ca.crt - permissions: "0644" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - -- path: /etc/hostname - permissions: "0600" - content: 'node1' - -- path: /etc/ssh/sshd_config - permissions: "0600" - user: root - content: | - # Use most defaults for sshd configuration. - Subsystem sftp internal-sftp - ClientAliveInterval 180 - UseDNS no - UsePAM yes - PrintLastLog no # handled by PAM - PrintMotd no # handled by PAM - PasswordAuthentication no - ChallengeResponseAuthentication no - -- path: /opt/bin/download.sh - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.2.0}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") - cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat < /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: /etc/sysctl.d/k8s.conf + permissions: "0644" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" + permissions: "0755" + content: | + # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar + # so we enable IPv6 Router Advertisement here. + # See for details https://github.com/flatcar-linux/Flatcar/issues/384 + [Network] + IPv6AcceptRA=true + +- path: /etc/kubernetes/bootstrap-kubelet.conf + permissions: "0400" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: /etc/kubernetes/cloud-config + permissions: "0400" + content: | + {anexia-config:true} + +- path: /etc/kubernetes/pki/ca.crt + permissions: "0644" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + +- path: /etc/hostname + permissions: "0600" + content: 'node1' + +- path: /etc/ssh/sshd_config + permissions: "0600" + user: root + content: | + # Use most defaults for sshd configuration. + Subsystem sftp internal-sftp + ClientAliveInterval 180 + UseDNS no + UsePAM yes + PrintLastLog no # handled by PAM + PrintMotd no # handled by PAM + PasswordAuthentication no + ChallengeResponseAuthentication no + +- path: /opt/bin/download.sh + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v1.2.0}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + mkdir -p /etc/systemd/system/containerd.service.d + + cat < /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: /etc/sysctl.d/k8s.conf + permissions: "0644" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" + permissions: "0755" + content: | + # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar + # so we enable IPv6 Router Advertisement here. + # See for details https://github.com/flatcar-linux/Flatcar/issues/384 + [Network] + IPv6AcceptRA=true + +- path: /etc/kubernetes/bootstrap-kubelet.conf + permissions: "0400" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: /etc/kubernetes/cloud-config + permissions: "0400" + content: | + {anexia-config:true} + +- path: /etc/kubernetes/pki/ca.crt + permissions: "0644" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + +- path: /etc/hostname + permissions: "0600" + content: 'node1' + +- path: /etc/ssh/sshd_config + permissions: "0600" + user: root + content: | + # Use most defaults for sshd configuration. + Subsystem sftp internal-sftp + ClientAliveInterval 180 + UseDNS no + UsePAM yes + PrintLastLog no # handled by PAM + PrintMotd no # handled by PAM + PasswordAuthentication no + ChallengeResponseAuthentication no + +- path: /opt/bin/download.sh + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v1.2.0}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + mkdir -p /etc/systemd/system/containerd.service.d + + cat < /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + systemctl disable disable-nm-cloud-setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMaximumGCAge: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + +rh_subscription: + username: "" + password: "" + auto-attach: false + +runcmd: +- systemctl enable --now setup.service +- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.29-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29-nutanix.yaml new file mode 100644 index 000000000..a2231d9ee --- /dev/null +++ b/pkg/userdata/rhel/testdata/kubelet-v1.29-nutanix.yaml @@ -0,0 +1,510 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + +hostname: node1 +fqdn: node1 + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + hostnamectl set-hostname node1 + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + iscsi-initiator-utils \ + ipvsadm + systemctl enable --now iscsid + + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + systemctl disable disable-nm-cloud-setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=nutanix \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMaximumGCAge: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +- path: "/opt/bin/disable-nm-cloud-setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then + systemctl stop nm-cloud-setup.timer + systemctl disable nm-cloud-setup.service + systemctl disable nm-cloud-setup.timer + reboot + fi + +- path: "/etc/systemd/system/disable-nm-cloud-setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup + +rh_subscription: + username: "" + password: "" + auto-attach: false + +runcmd: +- systemctl enable --now setup.service +- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index cde79dbce..9015a5e4d 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -148,7 +148,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -340,6 +340,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rockylinux/provider_test.go b/pkg/userdata/rockylinux/provider_test.go index b306379b4..3a9e7d596 100644 --- a/pkg/userdata/rockylinux/provider_test.go +++ b/pkg/userdata/rockylinux/provider_test.go @@ -106,7 +106,16 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", + Kubelet: "1.28.5", + }, + }, + }, + { + name: "kubelet-v1.29-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.29.0", }, }, }, diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml index f86374982..bdfb9bedc 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml @@ -334,6 +334,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml index 1b28cdae9..0ce7d5bf7 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml @@ -334,6 +334,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml index cc3360212..99d10681d 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml @@ -342,6 +342,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml index 9c5477924..bd4e68418 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml @@ -351,6 +351,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml index 11a34ded7..3797ad98d 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml @@ -351,6 +351,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml index e306a464c..131883af6 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml @@ -342,6 +342,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml index 8d243a220..5282cc402 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -334,6 +334,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.29-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29-aws.yaml new file mode 100644 index 000000000..cb322562c --- /dev/null +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.29-aws.yaml @@ -0,0 +1,467 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + tar \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMaximumGCAge: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index e0a976144..ae172d5a7 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -129,9 +129,10 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.26.9"), - semver.MustParse("v1.27.6"), - semver.MustParse("v1.28.2"), + semver.MustParse("v1.26.12"), + semver.MustParse("v1.27.9"), + semver.MustParse("v1.28.5"), + semver.MustParse("v1.29.0"), } var tests []userDataTestCase @@ -602,7 +603,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.26.12", }, }, ccProvider: &fakeCloudConfigProvider{ diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index fb5eaab31..cb8e939bd 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -410,6 +410,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml index 8013cb5c2..baa7cbba2 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml @@ -411,6 +411,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml index 0e2b57630..b0bbae3c0 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml @@ -411,6 +411,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index e3e4c794f..ae4df986c 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -400,6 +400,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml index 1a9628695..834cdb583 100644 --- a/pkg/userdata/ubuntu/testdata/docker.yaml +++ b/pkg/userdata/ubuntu/testdata/docker.yaml @@ -405,6 +405,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index 0dff8169e..a95022455 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -398,6 +398,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 9330fa523..9be9387bd 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -400,6 +400,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 0a01a0378..23911aad4 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -400,6 +400,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index 6a148f382..f1f76860a 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -153,7 +153,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.26.12}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -409,6 +409,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml index 57b85c21e..aed39b9ab 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml @@ -411,6 +411,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml index a38bfa3b8..28d09a5f2 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml @@ -411,6 +411,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index faa42aaaa..fa7016c7d 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -406,6 +406,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 934dbbf6c..18241af54 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -406,6 +406,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/version-1.26.12.yaml b/pkg/userdata/ubuntu/testdata/version-1.26.12.yaml new file mode 100644 index 000000000..16da2588e --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/version-1.26.12.yaml @@ -0,0 +1,459 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMaximumGCAge: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.27.6.yaml b/pkg/userdata/ubuntu/testdata/version-1.27.9.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.27.6.yaml rename to pkg/userdata/ubuntu/testdata/version-1.27.9.yaml index b89ceb594..41ae7b59f 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.27.6.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.27.9.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.9}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -398,6 +398,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/version-1.28.2.yaml b/pkg/userdata/ubuntu/testdata/version-1.28.5.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.28.2.yaml rename to pkg/userdata/ubuntu/testdata/version-1.28.5.yaml index 3e379099f..1ce969630 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.28.2.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.28.5.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -398,6 +398,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/version-1.26.9.yaml b/pkg/userdata/ubuntu/testdata/version-1.29.0.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.26.9.yaml rename to pkg/userdata/ubuntu/testdata/version-1.29.0.yaml index f35f5be94..0d62ccc34 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.26.9.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.29.0.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -398,6 +398,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index 13fdbb121..bc78ed1e5 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -417,6 +417,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 6f660c644..5b1c95607 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -424,6 +424,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index 8f34e2980..d418274a6 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -407,6 +407,7 @@ write_files: RotateKubeletServerCertificate: true fileCheckFrequency: 0s httpCheckFrequency: 0s + imageMaximumGCAge: 0s imageMinimumGCAge: 0s kind: KubeletConfiguration kubeReserved: diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 0fad8c668..c0f75ad41 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -84,8 +84,8 @@ const ( ) const ( - defaultKubernetesVersion = "1.27.6" - awsDefaultKubernetesVersion = "1.26.9" + defaultKubernetesVersion = "1.28.5" + awsDefaultKubernetesVersion = "1.26.12" defaultContainerRuntime = "containerd" ) @@ -344,7 +344,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.26. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.26.9", "1.27.6", "1.28.2"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.26.12", "1.27.9", "1.28.5", "1.29.0"))) runScenarios(t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -424,7 +424,7 @@ func TestAWSProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := Not(VersionSelector("1.27.6", "1.28.2")) + selector := Not(VersionSelector("1.27.9", "1.28.5", "1.29.0")) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -478,7 +478,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.6", "1.28.2"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.9", "1.28.5", "1.29.0"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -500,7 +500,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.6", "1.28.2"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.9", "1.28.5", "1.29.0"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 593fd56ab..6783aaf80 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,9 +33,10 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.26.9"), - semver.MustParse("v1.27.6"), - semver.MustParse("v1.28.2"), + semver.MustParse("v1.26.12"), + semver.MustParse("v1.27.9"), + semver.MustParse("v1.28.5"), + semver.MustParse("v1.29.0"), } operatingSystems = []providerconfigtypes.OperatingSystem{ diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml index db1768715..d92a6a034 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml @@ -55,4 +55,4 @@ spec: rhelOrganizationName: "" rhelActivationKey: "" versions: - kubelet: 1.26.6 + kubelet: 1.28.5 From 938522f3d9a42162f95a8aa41c465f26a9551d65 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 9 Feb 2024 15:10:03 +0500 Subject: [PATCH 373/489] Upgrade to Go 1.22 (#1759) * Upgrade to Go 1.22 Signed-off-by: Waleed Malik * Upgrade CRDs for OSM Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- .prow/e2e-features.yaml | 8 ++-- .prow/postsubmits.yaml | 4 +- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 16 ++++---- .prow/provider-azure.yaml | 6 +-- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 +- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 8 ++-- .prow/verify.yaml | 10 ++--- Dockerfile | 4 +- Makefile | 2 +- examples/operating-system-manager.yaml | 14 ++++--- go.mod | 22 +++++----- go.sum | 50 +++++++++++------------ hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- 25 files changed, 87 insertions(+), 87 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index fe59ab740..fc228ea77 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -119,7 +119,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 68d162289..43c25859e 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index d3462ad51..630cae547 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 2c6896ceb..deb7c19e5 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 9adfb1063..9ac0d6052 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -156,7 +156,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -187,7 +187,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -218,7 +218,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -249,7 +249,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index a055a5b3f..619645b13 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 393ef95fd..1d3bf0de8 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 0e4888a7a..1d803423f 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 592af7d08..1cf8cc5c9 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 204cfba35..e426630a8 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index a6f1204ce..ecc8741eb 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index dc900ef89..b76d3a799 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index bf40ac5a2..008e02404 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index c32083ad7..494e1b11c 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 42eb04cd7..7717473ce 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index e0abf8433..b84575cce 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 8af222d88..e49407684 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 261ba1c32..66f27dc22 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.21.5 + - image: golang:1.22.0 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.21.5 + - image: golang:1.22.0 command: - make args: @@ -63,7 +63,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golangci/golangci-lint:v1.54.1 + - image: golangci/golangci-lint:v1.56.0 command: - make args: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.21.5 + - image: golang:1.22.0 command: - make args: diff --git a/Dockerfile b/Dockerfile index eb4168adb..5651e7673 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.21.5 +ARG GO_VERSION=1.22.0 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . RUN make all -FROM alpine:3.17 +FROM alpine:3.19 RUN apk add --no-cache ca-certificates cdrkit diff --git a/Makefile b/Makefile index 82350088c..fe96b53b8 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.21.5 +GO_VERSION ?= 1.22.0 GOOS ?= $(shell go env GOOS) diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index 7f78d2c59..ce1648bb6 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -4,8 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.13.0 name: operatingsystemconfigs.operatingsystemmanager.k8c.io spec: group: operatingsystemmanager.k8c.io @@ -202,6 +201,7 @@ spec: - aws - azure - digitalocean + - edge - gce - hetzner - kubevirt @@ -415,8 +415,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.2 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.13.0 name: operatingsystemprofiles.operatingsystemmanager.k8c.io spec: group: operatingsystemmanager.k8c.io @@ -552,7 +551,8 @@ spec: supportedContainerRuntimes: description: SupportedContainerRuntimes represents the container - runtimes supported by the given OS + runtimes supported by the given OS. Docker has been deprecated + and is no-op. items: description: ContainerRuntimeSpec aggregates information about @@ -789,7 +789,8 @@ spec: supportedContainerRuntimes: description: SupportedContainerRuntimes represents the container - runtimes supported by the given OS + runtimes supported by the given OS. Docker has been deprecated + and is no-op. items: description: ContainerRuntimeSpec aggregates information about @@ -941,6 +942,7 @@ spec: - aws - azure - digitalocean + - edge - gce - hetzner - kubevirt diff --git a/go.mod b/go.mod index 22c6d6539..2baec178e 100644 --- a/go.mod +++ b/go.mod @@ -55,17 +55,17 @@ require ( google.golang.org/grpc v1.60.1 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.29.0 - k8s.io/apiextensions-apiserver v0.29.0 - k8s.io/apimachinery v0.29.0 - k8s.io/client-go v0.29.0 - k8s.io/cloud-provider v0.29.0 + k8s.io/api v0.29.1 + k8s.io/apiextensions-apiserver v0.29.1 + k8s.io/apimachinery v0.29.1 + k8s.io/client-go v0.29.1 + k8s.io/cloud-provider v0.29.1 k8s.io/klog v1.0.0 - k8s.io/kubelet v0.29.0 + k8s.io/kubelet v0.29.1 k8s.io/utils v0.0.0-20240102154912-e7106e64919e kubevirt.io/api v1.1.1 kubevirt.io/containerized-data-importer-api v1.58.0 - sigs.k8s.io/controller-runtime v0.16.3 + sigs.k8s.io/controller-runtime v0.17.0 sigs.k8s.io/yaml v1.4.0 ) @@ -106,7 +106,7 @@ require ( github.com/docker/distribution v2.8.3+incompatible // indirect github.com/emicklei/go-restful/v3 v3.11.1 // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.7.0 // indirect + github.com/evanphx/json-patch/v5 v5.8.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/flatcar/ignition v0.36.2 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect @@ -148,7 +148,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/ginkgo/v2 v2.13.2 // indirect + github.com/onsi/ginkgo/v2 v2.14.0 // indirect github.com/onsi/gomega v1.30.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f // indirect @@ -180,7 +180,7 @@ require ( golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.16.0 // indirect + golang.org/x/tools v0.16.1 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect @@ -191,7 +191,7 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/component-base v0.29.0 // indirect + k8s.io/component-base v0.29.1 // indirect k8s.io/klog/v2 v2.110.1 // indirect k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect diff --git a/go.sum b/go.sum index 2956f758d..0d21ddaa0 100644 --- a/go.sum +++ b/go.sum @@ -240,8 +240,8 @@ github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7 github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc= -github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= +github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flatcar/container-linux-config-transpiler v0.9.4 h1:yXQ0NB8PeNrKJPrZvbv5/DV63PNhTqt8vaf8YxmX/RA= @@ -586,8 +586,8 @@ github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042 github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.13.2 h1:Bi2gGVkfn6gQcjNjZJVO8Gf0FHzMPf2phUei9tejVMs= -github.com/onsi/ginkgo/v2 v2.13.2/go.mod h1:XStQ8QcGwLyF4HdfcZB8SFOS/MWCgDuXMSBe6zrvLgM= +github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= +github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= @@ -789,8 +789,8 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= -go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= @@ -872,8 +872,6 @@ golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1136,8 +1134,8 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= -golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1316,23 +1314,23 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= -k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A= -k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA= -k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0= -k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= +k8s.io/api v0.29.1 h1:DAjwWX/9YT7NQD4INu49ROJuZAAAP/Ijki48GUPzxqw= +k8s.io/api v0.29.1/go.mod h1:7Kl10vBRUXhnQQI8YR/R327zXC8eJ7887/+Ybta+RoQ= +k8s.io/apiextensions-apiserver v0.29.1 h1:S9xOtyk9M3Sk1tIpQMu9wXHm5O2MX6Y1kIpPMimZBZw= +k8s.io/apiextensions-apiserver v0.29.1/go.mod h1:zZECpujY5yTW58co8V2EQR4BD6A9pktVgHhvc0uLfeU= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= -k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= +k8s.io/apimachinery v0.29.1 h1:KY4/E6km/wLBguvCZv8cKTeOwwOBqFNjwJIdMkMbbRc= +k8s.io/apimachinery v0.29.1/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= -k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8= -k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38= -k8s.io/cloud-provider v0.29.0 h1:Qgk/jHsSKGRk/ltTlN6e7eaNuuamLROOzVBd0RPp94M= -k8s.io/cloud-provider v0.29.0/go.mod h1:gBCt7YYKFV4oUcJ/0xF9lS/9il4MxKunJ+ZKvh39WGo= +k8s.io/client-go v0.29.1 h1:19B/+2NGEwnFLzt0uB5kNJnfTsbV8w6TgQRz9l7ti7A= +k8s.io/client-go v0.29.1/go.mod h1:TDG/psL9hdet0TI9mGyHJSgRkW3H9JZk2dNEUS7bRks= +k8s.io/cloud-provider v0.29.1 h1:bDLpOSpysWrtU2PCkvyP2sUTwRBa6MGCmxt68CRRW/8= +k8s.io/cloud-provider v0.29.1/go.mod h1:u50Drm6AbuoKpsVbAstNiFHGgbSVHuJV4TWN5imdM2w= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s= -k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M= +k8s.io/component-base v0.29.1 h1:MUimqJPCRnnHsskTTjKD+IC1EHBbRCVyi37IoFBrkYw= +k8s.io/component-base v0.29.1/go.mod h1:fP9GFjxYrLERq1GcWWZAE3bqbNcDKDytn2srWuHTtKc= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= @@ -1350,8 +1348,8 @@ k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lV k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910 h1:1Rp/XEKP5uxPs6QrsngEHAxBjaAR78iJRiJq5Fi7LSU= k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910/go.mod h1:Pa1PvrP7ACSkuX6I7KYomY6cmMA0Tx86waBhDUgoKPw= -k8s.io/kubelet v0.29.0 h1:SX5hlznTBcGIrS1scaf8r8p6m3e475KMifwt9i12iOk= -k8s.io/kubelet v0.29.0/go.mod h1:kvKS2+Bz2tgDOG1S1q0TH2z1DasNuVF+8p6Aw7xvKkI= +k8s.io/kubelet v0.29.1 h1:cso8Dk8dymkj8q+EvW/aCbIYU2aOkH27gho48tYza/8= +k8s.io/kubelet v0.29.1/go.mod h1:hTl/naFcCVG1Ku17fMgj/krbheBwBkf3gnFhaboMx7E= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -1368,8 +1366,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= -sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= +sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= +sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 0afcaa361..8657d5ab6 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.21.5 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.22.0 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index cb13ab003..3709cc8d9 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.21-node-18-kind-0.20-9 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 containerize ./hack/verify-licenses.sh go mod vendor From 1fe4168d66ec6a5b5e5425eef5e0e94391d308c8 Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Fri, 16 Feb 2024 11:22:43 +0100 Subject: [PATCH 374/489] Update prow jobs and remove yamllint image (#1764) Signed-off-by: Marvin Beckers --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 16 ++++++++-------- .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 8 ++++---- .prow/verify.yaml | 10 +++++----- Makefile | 3 +++ hack/verify-licenses.sh | 2 +- 20 files changed, 43 insertions(+), 40 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index fc228ea77..7fd8eff78 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -119,7 +119,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 43c25859e..d94aa85a6 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index 630cae547..1c7a1af7c 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index deb7c19e5..deb91e618 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 9ac0d6052..e7a60d1b2 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -156,7 +156,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -187,7 +187,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -218,7 +218,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -249,7 +249,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 619645b13..f06fa7dd0 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 1d3bf0de8..0a2ee8929 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 1d803423f..986bb7faf 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 1cf8cc5c9..5c85b2939 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index e426630a8..7205e1492 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index ecc8741eb..ad712a605 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index b76d3a799..def0cb7de 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 008e02404..4c05da09b 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 494e1b11c..8213afc44 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 7717473ce..c9a33e4fc 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index b84575cce..bc09746d8 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index e49407684..71ea8ab3e 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 66f27dc22..edee2d747 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -83,11 +83,11 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/yamllint:0.1 + - image: quay.io/kubermatic/build:go-1.22-node-18-3 command: - - "sh" - - "-c" - - "yamllint -c .yamllint.conf ." + - make + args: + - yamllint resources: requests: memory: 32Mi @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 command: - ./hack/verify-licenses.sh resources: diff --git a/Makefile b/Makefile index fe96b53b8..20a574f84 100644 --- a/Makefile +++ b/Makefile @@ -67,6 +67,9 @@ clean: lint: golangci-lint run -v +yamllint: + yamllint -c .yamllint.conf . + .PHONY: docker-image docker-image: docker build --build-arg GO_VERSION=$(GO_VERSION) -t $(IMAGE_NAME) . diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 3709cc8d9..3646eec8f 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-2 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 containerize ./hack/verify-licenses.sh go mod vendor From 2397824a8275c8623544ae4173f8a2fc78bb9c4d Mon Sep 17 00:00:00 2001 From: Marvin Beckers Date: Mon, 19 Feb 2024 11:35:46 +0100 Subject: [PATCH 375/489] Remove TORCX_BINDIR from containerd binary call (#1760) Signed-off-by: Marvin Beckers --- pkg/containerruntime/containerd.go | 2 +- pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml | 2 +- pkg/userdata/flatcar/testdata/cloud-init_v1.28.5.yaml | 2 +- pkg/userdata/flatcar/testdata/cloud-init_v1.29.0.yaml | 2 +- pkg/userdata/flatcar/testdata/containerd.yaml | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.26.6.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.28.5.json | 2 +- pkg/userdata/flatcar/testdata/ignition_v1.29.0.json | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index 7b9d3469a..2b5250658 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -96,7 +96,7 @@ cat < Date: Mon, 19 Feb 2024 20:08:46 +0100 Subject: [PATCH 376/489] Support Edge Provider in Machine Controller (#1765) * support edge provider Signed-off-by: Moath Qasim Signed-off-by: Moath Qasim * fix linting Signed-off-by: Moath Qasim Signed-off-by: Moath Qasim --------- Signed-off-by: Moath Qasim --- pkg/cloudprovider/provider.go | 4 + pkg/cloudprovider/provider/edge/provider.go | 99 +++++++++++++++++++++ pkg/providerconfig/types/types.go | 2 + 3 files changed, 105 insertions(+) create mode 100644 pkg/cloudprovider/provider/edge/provider.go diff --git a/pkg/cloudprovider/provider.go b/pkg/cloudprovider/provider.go index 28161700c..4cd794157 100644 --- a/pkg/cloudprovider/provider.go +++ b/pkg/cloudprovider/provider.go @@ -26,6 +26,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/azure" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/digitalocean" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/edge" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/equinixmetal" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/fake" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/gce" @@ -85,6 +86,9 @@ var ( providerconfigtypes.CloudProviderFake: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return fake.New(cvr) }, + providerconfigtypes.CloudProviderEdge: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return edge.New(cvr) + }, providerconfigtypes.CloudProviderKubeVirt: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return kubevirt.New(cvr) }, diff --git a/pkg/cloudprovider/provider/edge/provider.go b/pkg/cloudprovider/provider/edge/provider.go new file mode 100644 index 000000000..b47f535af --- /dev/null +++ b/pkg/cloudprovider/provider/edge/provider.go @@ -0,0 +1,99 @@ +/* +Copyright 2024 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package edge + +import ( + "context" + "go.uber.org/zap" + + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" + cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "github.com/kubermatic/machine-controller/pkg/providerconfig" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +type provider struct{} + +type CloudProviderSpec struct{} + +type CloudProviderInstance struct{} + +func (f CloudProviderInstance) Name() string { + return "" +} + +func (f CloudProviderInstance) ID() string { + return "" +} + +func (f CloudProviderInstance) ProviderID() string { + return "" +} + +func (f CloudProviderInstance) Addresses() map[string]corev1.NodeAddressType { + return nil +} + +func (f CloudProviderInstance) Status() instance.Status { + return instance.StatusUnknown +} + +// New returns a edge cloud provider. +func New(_ *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return &provider{} +} + +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { + return spec, nil +} + +// Validate returns success or failure based according to its EdgeCloudProviderSpec. +func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, _ clusterv1alpha1.MachineSpec) error { + return nil +} + +func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { + return CloudProviderInstance{}, nil +} + +func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string, error) { + return "", "", nil +} + +// Create creates a cloud instance according to the given machine. +func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, _ string) (instance.Instance, error) { + return CloudProviderInstance{}, nil +} + +func (p *provider) Cleanup(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { + return true, nil +} + +func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ types.UID) error { + return nil +} + +func (p *provider) MachineMetricsLabels(_ *clusterv1alpha1.Machine) (map[string]string, error) { + return map[string]string{}, nil +} + +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { + return nil +} diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 046f4cd9f..6fb9e967d 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -63,6 +63,7 @@ const ( CloudProviderVultr CloudProvider = "vultr" CloudProviderVMwareCloudDirector CloudProvider = "vmware-cloud-director" CloudProviderFake CloudProvider = "fake" + CloudProviderEdge CloudProvider = "edge" CloudProviderAlibaba CloudProvider = "alibaba" CloudProviderAnexia CloudProvider = "anexia" CloudProviderScaleway CloudProvider = "scaleway" @@ -100,6 +101,7 @@ var ( CloudProviderVsphere, CloudProviderVMwareCloudDirector, CloudProviderFake, + CloudProviderEdge, CloudProviderAlibaba, CloudProviderAnexia, CloudProviderScaleway, From 8cef3ecaab1a55295bcb9247ee56a4fe5cf61ccd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Wed, 21 Feb 2024 15:46:50 +0100 Subject: [PATCH 377/489] Release branch release/v1.59 commit (#1766) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 470eb79aa..1e77e80a3 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ **Important Note: User data plugins for machine-controller are deprecated and will soon be removed. [Operating System Manager](https://github.com/kubermatic/operating-system-manager) is the successor of user data plugins. It's responsible for creating and managing the required configurations for worker nodes in a Kubernetes cluster with better modularity and extensibility. Please refer to [Operating System Manager][8] for more details.** + ## Table of Contents - [Kubermatic machine-controller](#kubermatic-machine-controller) From 4342ca29172ad17b0fd87ade441b3c607e7ad17c Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 7 Mar 2024 01:32:43 +0500 Subject: [PATCH 378/489] Drop support for kubernetes 1.26 (#1769) * Drop support for kubernetes 1.26 Signed-off-by: Waleed Malik * Update matrix for e2e tests Signed-off-by: Waleed Malik * Built in cloud providers are disabled in k8s 1.26 Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- .prow/provider-aws.yaml | 9 +- .prow/provider-gcp.yaml | 36 +- README.md | 2 - pkg/userdata/amzn2/provider_test.go | 29 +- ...yaml => kubelet-v1.29.2-aws-external.yaml} | 2 +- ...1.26-aws.yaml => kubelet-v1.29.2-aws.yaml} | 2 +- ...l => kubelet-v1.29.2-vsphere-mirrors.yaml} | 2 +- ...aml => kubelet-v1.29.2-vsphere-proxy.yaml} | 2 +- ...here.yaml => kubelet-v1.29.2-vsphere.yaml} | 2 +- pkg/userdata/centos/provider_test.go | 24 +- .../centos/testdata/kubelet-v1.26-aws.yaml | 448 ---------------- ...yaml => kubelet-v1.29.2-aws-external.yaml} | 2 +- ...26.6-aws.yaml => kubelet-v1.29.2-aws.yaml} | 2 +- ...anix.yaml => kubelet-v1.29.2-nutanix.yaml} | 2 +- ...l => kubelet-v1.29.2-vsphere-mirrors.yaml} | 2 +- ...aml => kubelet-v1.29.2-vsphere-proxy.yaml} | 2 +- ...here.yaml => kubelet-v1.29.2-vsphere.yaml} | 2 +- pkg/userdata/flatcar/provider_test.go | 22 +- ...t_v1.26.6.yaml => cloud-init_v1.29.2.yaml} | 2 +- pkg/userdata/flatcar/testdata/containerd.yaml | 2 +- ...ion_v1.26.6.json => ignition_v1.29.2.json} | 2 +- pkg/userdata/helper/common_test.go | 7 +- .../helper/download_binaries_script_test.go | 4 +- pkg/userdata/helper/kubelet_test.go | 8 +- ...lden => download_binaries_v1.27.11.golden} | 2 +- .../testdata/download_binaries_v1.28.7.golden | 17 + .../testdata/download_binaries_v1.29.2.golden | 17 + ...emd_unit_version-v1.27.11-external.golden} | 0 ...blet_systemd_unit_version-v1.27.11.golden} | 0 ...stemd_unit_version-v1.28.7-external.golden | 36 ++ ...kublet_systemd_unit_version-v1.28.7.golden | 35 ++ ...stemd_unit_version-v1.29.2-external.golden | 36 ++ ...kublet_systemd_unit_version-v1.29.2.golden | 35 ++ ... => safe_download_binaries_v1.29.2.golden} | 2 +- pkg/userdata/rhel/provider_test.go | 39 +- .../testdata/kubelet-v1.26-aws-external.yaml | 501 ------------------ .../rhel/testdata/kubelet-v1.26.6-aws.yaml | 501 ------------------ ...yaml => kubelet-v1.29.2-aws-external.yaml} | 2 +- ...1.26-aws.yaml => kubelet-v1.29.2-aws.yaml} | 2 +- ...l => kubelet-v1.29.2-vsphere-mirrors.yaml} | 2 +- ...aml => kubelet-v1.29.2-vsphere-proxy.yaml} | 2 +- ...here.yaml => kubelet-v1.29.2-vsphere.yaml} | 2 +- pkg/userdata/rockylinux/provider_test.go | 33 +- ...yaml => kubelet-v1.29.2-aws-external.yaml} | 2 +- ...26.6-aws.yaml => kubelet-v1.29.2-aws.yaml} | 2 +- ...anix.yaml => kubelet-v1.29.2-nutanix.yaml} | 2 +- ...l => kubelet-v1.29.2-vsphere-mirrors.yaml} | 2 +- ...aml => kubelet-v1.29.2-vsphere-proxy.yaml} | 2 +- ...here.yaml => kubelet-v1.29.2-vsphere.yaml} | 2 +- pkg/userdata/ubuntu/provider_test.go | 9 +- pkg/userdata/ubuntu/testdata/nutanix.yaml | 2 +- ...sion-1.26.12.yaml => version-1.27.11.yaml} | 2 +- .../testdata/version-1.28.7.yaml} | 233 ++++---- .../ubuntu/testdata/version-1.29.2.yaml | 459 ++++++++++++++++ test/e2e/provisioning/all_e2e_test.go | 12 +- test/e2e/provisioning/helper.go | 7 +- 56 files changed, 901 insertions(+), 1716 deletions(-) rename pkg/userdata/amzn2/testdata/{kubelet-v1.26.6-aws-external.yaml => kubelet-v1.29.2-aws-external.yaml} (99%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.26-aws.yaml => kubelet-v1.29.2-aws.yaml} (99%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.26.6-vsphere-mirrors.yaml => kubelet-v1.29.2-vsphere-mirrors.yaml} (99%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.26.6-vsphere-proxy.yaml => kubelet-v1.29.2-vsphere-proxy.yaml} (99%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.26.6-vsphere.yaml => kubelet-v1.29.2-vsphere.yaml} (99%) delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml rename pkg/userdata/centos/testdata/{kubelet-v1.26.6-aws-external.yaml => kubelet-v1.29.2-aws-external.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.26.6-aws.yaml => kubelet-v1.29.2-aws.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.26.6-nutanix.yaml => kubelet-v1.29.2-nutanix.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.26.6-vsphere-mirrors.yaml => kubelet-v1.29.2-vsphere-mirrors.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.26.6-vsphere-proxy.yaml => kubelet-v1.29.2-vsphere-proxy.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.26.6-vsphere.yaml => kubelet-v1.29.2-vsphere.yaml} (99%) rename pkg/userdata/flatcar/testdata/{cloud-init_v1.26.6.yaml => cloud-init_v1.29.2.yaml} (99%) rename pkg/userdata/flatcar/testdata/{ignition_v1.26.6.json => ignition_v1.29.2.json} (99%) rename pkg/userdata/helper/testdata/{download_binaries_v1.26.12.golden => download_binaries_v1.27.11.golden} (91%) create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.28.7.golden create mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.29.2.golden rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.26.12-external.golden => kublet_systemd_unit_version-v1.27.11-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.26.12.golden => kublet_systemd_unit_version-v1.27.11.golden} (100%) create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.7-external.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.7.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.2-external.golden create mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.2.golden rename pkg/userdata/helper/testdata/{safe_download_binaries_v1.26.6.golden => safe_download_binaries_v1.29.2.golden} (98%) delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml rename pkg/userdata/rhel/testdata/{kubelet-v1.26.6-aws-external.yaml => kubelet-v1.29.2-aws-external.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.26-aws.yaml => kubelet-v1.29.2-aws.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.26.6-vsphere-mirrors.yaml => kubelet-v1.29.2-vsphere-mirrors.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.26.6-vsphere-proxy.yaml => kubelet-v1.29.2-vsphere-proxy.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.26.6-vsphere.yaml => kubelet-v1.29.2-vsphere.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.26.6-aws-external.yaml => kubelet-v1.29.2-aws-external.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.26.6-aws.yaml => kubelet-v1.29.2-aws.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.26.6-nutanix.yaml => kubelet-v1.29.2-nutanix.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.26.6-vsphere-mirrors.yaml => kubelet-v1.29.2-vsphere-mirrors.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.26.6-vsphere-proxy.yaml => kubelet-v1.29.2-vsphere-proxy.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.26.6-vsphere.yaml => kubelet-v1.29.2-vsphere.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.26.12.yaml => version-1.27.11.yaml} (99%) rename pkg/userdata/{amzn2/testdata/kubelet-v1.26.6-aws.yaml => ubuntu/testdata/version-1.28.7.yaml} (90%) create mode 100644 pkg/userdata/ubuntu/testdata/version-1.29.2.yaml diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index e7a60d1b2..709f72dba 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -14,7 +14,8 @@ presubmits: - name: pull-machine-controller-e2e-aws - run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" + # In-tree CCM is not supported for AWS starting from k8s 1.27. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. + # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -46,7 +47,8 @@ presubmits: memory: 7Gi - name: pull-machine-controller-e2e-aws-spot-instance-legacy-userdata - always_run: true + # In-tree CCM is not supported for AWS starting from k8s 1.27. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. + # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -142,7 +144,8 @@ presubmits: memory: 7Gi - name: pull-machine-controller-e2e-aws-spot-instance - always_run: true + # In-tree CCM is not supported for AWS starting from k8s 1.27. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. + # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 5c85b2939..28aa9ef2a 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-gce - run_if_changed: "(pkg/cloudprovider/provider/gce/|pkg/userdata)" + always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -44,3 +44,37 @@ presubmits: cpu: 2 limits: memory: 7Gi + + - name: pull-machine-controller-e2e-gce-legacy-userdata + always_run: true + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-gce: "true" + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-rhel: "true" + preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" + spec: + containers: + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + command: + - "./hack/ci/run-e2e-tests.sh" + args: + - "TestGCEProvisioningE2E" + env: + - name: OPERATING_SYSTEM_MANAGER + value: "false" + - name: CLOUD_PROVIDER + value: gce + securityContext: + privileged: true + resources: + requests: + memory: 7Gi + cpu: 2 + limits: + memory: 7Gi diff --git a/README.md b/README.md index 1e77e80a3..f0e41d67f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ **Important Note: User data plugins for machine-controller are deprecated and will soon be removed. [Operating System Manager](https://github.com/kubermatic/operating-system-manager) is the successor of user data plugins. It's responsible for creating and managing the required configurations for worker nodes in a Kubernetes cluster with better modularity and extensibility. Please refer to [Operating System Manager][8] for more details.** - ## Table of Contents - [Kubermatic machine-controller](#kubermatic-machine-controller) @@ -47,7 +46,6 @@ Currently supported K8S versions are: - 1.29 - 1.28 - 1.27 -- 1.26 ### Community Providers diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index f4b2f0455..6d20a7b22 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -102,40 +102,40 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.26.6-aws", + name: "kubelet-v1.29.2-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, }, { - name: "kubelet-v1.26.6-aws-external", + name: "kubelet-v1.29.2-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.26.6-vsphere", + name: "kubelet-v1.29.2-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.26.6-vsphere-proxy", + name: "kubelet-v1.29.2-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), @@ -145,11 +145,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.26.6-vsphere-mirrors", + name: "kubelet-v1.29.2-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), @@ -158,15 +158,6 @@ func TestUserDataGeneration(t *testing.T) { registryMirrors: "https://registry.docker-cn.com/", pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, - { - name: "kubelet-v1.26-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.0", - }, - }, - }, { name: "kubelet-v1.28-aws", spec: clusterv1alpha1.MachineSpec{ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws-external.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws-external.yaml index ec34eeec3..f88e1edd9 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws-external.yaml @@ -143,7 +143,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws.yaml index 7279309d8..0cc636fb6 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws.yaml @@ -143,7 +143,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml index 60bb2fd32..9ac00af2a 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml @@ -156,7 +156,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-proxy.yaml index 859c8c46d..3ddf865fa 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-proxy.yaml @@ -156,7 +156,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere.yaml index 742531ab8..cc75ec35b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere.yaml @@ -148,7 +148,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index e6bd23c03..d93b87a2f 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -102,50 +102,50 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.26.6-aws", + name: "kubelet-v1.29.2-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, }, { - name: "kubelet-v1.26.6-nutanix", + name: "kubelet-v1.29.2-nutanix", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("nutanix"), }, { - name: "kubelet-v1.26.6-aws-external", + name: "kubelet-v1.29.2-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.26.6-vsphere", + name: "kubelet-v1.29.2-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.26.6-vsphere-proxy", + name: "kubelet-v1.29.2-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), @@ -155,11 +155,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.26.6-vsphere-mirrors", + name: "kubelet-v1.29.2-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml deleted file mode 100644 index 0ca4f01cc..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.26-aws.yaml +++ /dev/null @@ -1,448 +0,0 @@ -#cloud-config - -ssh_pwauth: false - -write_files: - - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - - path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - - path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - - path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - - - path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - - - path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - - - path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - - - path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - - - path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - - path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - - path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - - path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - - path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - - - path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - - - path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - - path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - -runcmd: - - systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws-external.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.29.2-aws-external.yaml index 4b50bd558..83e3124a4 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws-external.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.29.2-aws.yaml index d983f9261..67f9242a4 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-nutanix.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.29.2-nutanix.yaml index d9e56c166..f5f9f18cd 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.29.2-nutanix.yaml @@ -156,7 +156,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml index 8697d4c3c..b5401e2c4 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml @@ -162,7 +162,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-proxy.yaml index 053554a4b..54ba3c434 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-proxy.yaml @@ -162,7 +162,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere.yaml index f8f811fc0..5164e1e12 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere.yaml @@ -154,7 +154,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index a787979d1..faaa60a92 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -186,7 +186,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.26.6", + name: "ignition_v1.29.2", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -202,7 +202,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -217,7 +217,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.26.6", + name: "ignition_v1.29.2", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -233,7 +233,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -248,7 +248,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.26.6", + name: "ignition_v1.29.2", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -264,7 +264,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -341,7 +341,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.26.6", + name: "cloud-init_v1.29.2", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -357,7 +357,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -372,7 +372,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.26.6", + name: "cloud-init_v1.29.2", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -388,7 +388,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -415,7 +415,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, ccProvider: &fakeCloudConfigProvider{}, diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.29.2.yaml similarity index 99% rename from pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml rename to pkg/userdata/flatcar/testdata/cloud-init_v1.29.2.yaml index 5f55079e6..30c4bfa86 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.26.6.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.29.2.yaml @@ -432,7 +432,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/flatcar/testdata/containerd.yaml b/pkg/userdata/flatcar/testdata/containerd.yaml index b2ef391f1..184502438 100644 --- a/pkg/userdata/flatcar/testdata/containerd.yaml +++ b/pkg/userdata/flatcar/testdata/containerd.yaml @@ -411,7 +411,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/flatcar/testdata/ignition_v1.26.6.json b/pkg/userdata/flatcar/testdata/ignition_v1.29.2.json similarity index 99% rename from pkg/userdata/flatcar/testdata/ignition_v1.26.6.json rename to pkg/userdata/flatcar/testdata/ignition_v1.29.2.json index 2bf79b838..87202d61f 100644 --- a/pkg/userdata/flatcar/testdata/ignition_v1.26.6.json +++ b/pkg/userdata/flatcar/testdata/ignition_v1.29.2.json @@ -1 +1 @@ -{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMaximumGCAge%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.26.6%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20containerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file +{"ignition":{"config":{},"security":{"tls":{}},"timeouts":{},"version":"2.3.0"},"networkd":{"units":[{"contents":"[Match]\n# Because of difficulty predicting specific NIC names on different cloud providers,\n# we only support static addressing on VSphere. There should be a single NIC attached\n# that we will match by name prefix 'en' which denotes ethernet devices.\nName=en*\n\n[Network]\nDHCP=no\nAddress=192.168.81.4/24\nGateway=192.168.81.1\nDNS=8.8.8.8\n","name":"static-nic.network"}]},"passwd":{"users":[{"name":"core","sshAuthorizedKeys":["ssh-rsa AAABBB","ssh-rsa CCCDDD"]}]},"storage":{"files":[{"filesystem":"root","path":"/etc/systemd/journald.conf.d/max_disk_use.conf","contents":{"source":"data:,%5BJournal%5D%0ASystemMaxUse%3D5G%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/kubernetes/kubelet.conf","contents":{"source":"data:,apiVersion%3A%20kubelet.config.k8s.io%2Fv1beta1%0Aauthentication%3A%0A%20%20anonymous%3A%0A%20%20%20%20enabled%3A%20false%0A%20%20webhook%3A%0A%20%20%20%20cacheTTL%3A%200s%0A%20%20%20%20enabled%3A%20true%0A%20%20x509%3A%0A%20%20%20%20clientCAFile%3A%20%2Fetc%2Fkubernetes%2Fpki%2Fca.crt%0Aauthorization%3A%0A%20%20mode%3A%20Webhook%0A%20%20webhook%3A%0A%20%20%20%20cacheAuthorizedTTL%3A%200s%0A%20%20%20%20cacheUnauthorizedTTL%3A%200s%0AcgroupDriver%3A%20systemd%0AclusterDNS%3A%0A-%2010.10.10.10%0AclusterDomain%3A%20cluster.local%0AcontainerLogMaxSize%3A%20100Mi%0AcontainerRuntimeEndpoint%3A%20%22%22%0AcpuManagerReconcilePeriod%3A%200s%0AevictionHard%3A%0A%20%20imagefs.available%3A%2015%25%0A%20%20memory.available%3A%20100Mi%0A%20%20nodefs.available%3A%2010%25%0A%20%20nodefs.inodesFree%3A%205%25%0AevictionPressureTransitionPeriod%3A%200s%0AfeatureGates%3A%0A%20%20RotateKubeletServerCertificate%3A%20true%0AfileCheckFrequency%3A%200s%0AhttpCheckFrequency%3A%200s%0AimageMaximumGCAge%3A%200s%0AimageMinimumGCAge%3A%200s%0Akind%3A%20KubeletConfiguration%0AkubeReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0Alogging%3A%0A%20%20flushFrequency%3A%200%0A%20%20options%3A%0A%20%20%20%20json%3A%0A%20%20%20%20%20%20infoBufferSize%3A%20%220%22%0A%20%20verbosity%3A%200%0AmemorySwap%3A%20%7B%7D%0AnodeStatusReportFrequency%3A%200s%0AnodeStatusUpdateFrequency%3A%200s%0AprotectKernelDefaults%3A%20true%0ArotateCertificates%3A%20true%0AruntimeRequestTimeout%3A%200s%0AserverTLSBootstrap%3A%20true%0AshutdownGracePeriod%3A%200s%0AshutdownGracePeriodCriticalPods%3A%200s%0AstaticPodPath%3A%20%2Fetc%2Fkubernetes%2Fmanifests%0AstreamingConnectionIdleTimeout%3A%200s%0AsyncFrequency%3A%200s%0AsystemReserved%3A%0A%20%20cpu%3A%20200m%0A%20%20ephemeral-storage%3A%201Gi%0A%20%20memory%3A%20200Mi%0AtlsCipherSuites%3A%0A-%20TLS_AES_128_GCM_SHA256%0A-%20TLS_AES_256_GCM_SHA384%0A-%20TLS_CHACHA20_POLY1305_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305%0A-%20TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256%0A-%20TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384%0A-%20TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305%0AvolumePluginDir%3A%20%2Fvar%2Flib%2Fkubelet%2Fvolumeplugins%0AvolumeStatsAggPeriod%3A%200s%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/load-kernel-modules.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aset%20-euo%20pipefail%0A%0Amodprobe%20ip_vs%0Amodprobe%20ip_vs_rr%0Amodprobe%20ip_vs_wrr%0Amodprobe%20ip_vs_sh%0A%0Aif%20modinfo%20nf_conntrack_ipv4%20%26%3E%20%2Fdev%2Fnull%3B%20then%0A%20%20modprobe%20nf_conntrack_ipv4%0Aelse%0A%20%20modprobe%20nf_conntrack%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/sysctl.d/k8s.conf","contents":{"source":"data:,net.bridge.bridge-nf-call-ip6tables%20%3D%201%0Anet.bridge.bridge-nf-call-iptables%20%3D%201%0Akernel.panic_on_oops%20%3D%201%0Akernel.panic%20%3D%2010%0Anet.ipv4.ip_forward%20%3D%201%0Avm.overcommit_memory%20%3D%201%0Afs.inotify.max_user_watches%20%3D%201048576%0Afs.inotify.max_user_instances%20%3D%208192%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic_on_oops","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/kernel/panic","contents":{"source":"data:,10%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/proc/sys/vm/overcommit_memory","contents":{"source":"data:,1%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/opt/bin/setup_net_env.sh","contents":{"source":"data:,%23!%2Fusr%2Fbin%2Fenv%20bash%0Aechodate()%20%7B%0A%20%20echo%20%22%5B%24(date%20-Is)%5D%22%20%22%24%40%22%0A%7D%0A%0A%23%20get%20the%20default%20interface%20IP%20address%0ADEFAULT_IFC_IP%3D%24(ip%20-o%20%20route%20get%201%20%7C%20grep%20-oP%20%22src%20%5CK%5CS%2B%22)%0A%0A%23%20get%20the%20full%20hostname%0AFULL_HOSTNAME%3D%24(hostname%20-f)%0A%0Aif%20%5B%20-z%20%22%24%7BDEFAULT_IFC_IP%7D%22%20%5D%0Athen%0A%09echodate%20%22Failed%20to%20get%20IP%20address%20for%20the%20default%20route%20interface%22%0A%09exit%201%0Afi%0A%0A%23%20write%20the%20nodeip_env%20file%0A%23%20we%20need%20the%20line%20below%20because%20flatcar%20has%20the%20same%20string%20%22coreos%22%20in%20that%20file%0Aif%20grep%20-q%20coreos%20%2Fetc%2Fos-release%0Athen%0A%20%20echo%20-e%20%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5CnKUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%22%20%3E%20%2Fetc%2Fkubernetes%2Fnodeip.conf%0Aelif%20%5B%20!%20-d%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%20%5D%0Athen%0A%09echodate%20%22Can't%20find%20kubelet%20service%20extras%20directory%22%0A%09exit%201%0Aelse%0A%20%20echo%20-e%20%22%5BService%5D%5CnEnvironment%3D%5C%22KUBELET_NODE_IP%3D%24%7BDEFAULT_IFC_IP%7D%5C%22%5CnEnvironment%3D%5C%22KUBELET_HOSTNAME%3D%24%7BFULL_HOSTNAME%7D%5C%22%22%20%3E%20%2Fetc%2Fsystemd%2Fsystem%2Fkubelet.service.d%2Fnodeip.conf%0Afi%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/systemd/network/zz-default.network.d/ipv6-fix.conf","contents":{"source":"data:,%5BNetwork%5D%0AIPv6AcceptRA%3Dtrue%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/kubernetes/bootstrap-kubelet.conf","contents":{"source":"data:,apiVersion%3A%20v1%0Aclusters%3A%0A-%20cluster%3A%0A%20%20%20%20certificate-authority-data%3A%20LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t%0A%20%20%20%20server%3A%20https%3A%2F%2Fserver%3A443%0A%20%20name%3A%20%22%22%0Acontexts%3A%20null%0Acurrent-context%3A%20%22%22%0Akind%3A%20Config%0Apreferences%3A%20%7B%7D%0Ausers%3A%0A-%20name%3A%20%22%22%0A%20%20user%3A%0A%20%20%20%20token%3A%20my-token%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/cloud-config","contents":{"source":"data:,%7Bvsphere-config%3Atrue%7D%0A","verification":{}},"mode":256},{"filesystem":"root","path":"/etc/kubernetes/pki/ca.crt","contents":{"source":"data:,-----BEGIN%20CERTIFICATE-----%0AMIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV%0ABAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG%0AA1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3%0ADQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0%0ANjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG%0AcmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv%0Ac3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B%0AAQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS%0AR8Od0%2B9Q62Hyny%2BGFwMTb4A%2FKU8mssoHvcceSAAbwfbxFK%2F%2Bs51TobqUnORZrOoT%0AZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk%0AJfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS%2FPlPbUj2q7YnoVLposUBMlgUb%2FCykX3%0AmOoLb4yJJQyA%2FiST6ZxiIEj36D4yWZ5lg7YJl%2BUiiBQHGCnPdGyipqV06ex0heYW%0AcaiW8LWZSUQ93jQ%2BWVCH8hT7DQO1dmsvUmXlq%2FJeAlwQ%2FQIDAQABo4HgMIHdMB0G%0AA1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt%0AhS4P4U7vTfjByC569R7E6KF%2FpH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB%0AMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES%0AMBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv%0AbYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h%0AU9f9sNH0%2F6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k%2FXkDjQm%2B3lzjT0iGR4IxE%2FAo%0AeU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb%2FLnDUjs5Yj9brP0NWzXfYU4%0AUK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm%2Bje6voD%0A58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj%2Bqvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n%0AsH9BBH38%2FSzUmAN4QHSPy1gjqm00OAE8NaYDkh%2FbzE4d7mLGGMWp%2FWE3KPSu82HF%0AkPe6XoSbiLm%2Fkxk32T0%3D%0A-----END%20CERTIFICATE-----%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/hostname","contents":{"source":"data:,node1","verification":{}},"mode":384},{"filesystem":"root","group":{"id":0},"path":"/etc/ssh/sshd_config","user":{"id":0},"contents":{"source":"data:,%23%20Use%20most%20defaults%20for%20sshd%20configuration.%0ASubsystem%20sftp%20internal-sftp%0AClientAliveInterval%20180%0AUseDNS%20no%0AUsePAM%20yes%0APrintLastLog%20no%20%23%20handled%20by%20PAM%0APrintMotd%20no%20%23%20handled%20by%20PAM%0APasswordAuthentication%20no%0AChallengeResponseAuthentication%20no%0A","verification":{}},"mode":384},{"filesystem":"root","path":"/opt/bin/setup.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0A%23%20We%20stop%20these%20services%20here%20explicitly%20since%20masking%20only%20removes%20the%20symlinks%20for%20these%20services%20so%20that%20they%20can't%20be%20started.%0A%23%20But%20that%20wouldn't%20%22stop%22%20the%20already%20running%20services%20on%20the%20first%20boot.%0Asystemctl%20stop%20update-engine.service%0Asystemctl%20stop%20locksmithd.service%0Asystemctl%20disable%20setup.service%0A%0A%23%20Creates%20iscsi%20InitiatorName%20on%20Nutanix%20machines%20for%20CSI%20driver%20to%20attach%20volumes.%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/opt/bin/download.sh","contents":{"source":"data:,%23!%2Fbin%2Fbash%0Aset%20-xeuo%20pipefail%0A%0Aopt_bin%3D%2Fopt%2Fbin%0Ausr_local_bin%3D%2Fusr%2Flocal%2Fbin%0Acni_bin_dir%3D%2Fopt%2Fcni%2Fbin%0Amkdir%20-p%20%2Fetc%2Fcni%2Fnet.d%20%2Fetc%2Fkubernetes%2Fmanifests%20%22%24opt_bin%22%20%22%24cni_bin_dir%22%0Aarch%3D%24%7BHOST_ARCH-%7D%0Aif%20%5B%20-z%20%22%24arch%22%20%5D%0Athen%0Acase%20%24(uname%20-m)%20in%0Ax86_64)%0A%20%20%20%20arch%3D%22amd64%22%0A%20%20%20%20%3B%3B%0Aaarch64)%0A%20%20%20%20arch%3D%22arm64%22%0A%20%20%20%20%3B%3B%0A*)%0A%20%20%20%20echo%20%22unsupported%20CPU%20architecture%2C%20exiting%22%0A%20%20%20%20exit%201%0A%20%20%20%20%3B%3B%0Aesac%0Afi%0ACNI_VERSION%3D%22%24%7BCNI_VERSION%3A-v1.2.0%7D%22%0Acni_base_url%3D%22https%3A%2F%2Fgithub.com%2Fcontainernetworking%2Fplugins%2Freleases%2Fdownload%2F%24CNI_VERSION%22%0Acni_filename%3D%22cni-plugins-linux-%24arch-%24CNI_VERSION.tgz%22%0Acurl%20-Lfo%20%22%24cni_bin_dir%2F%24cni_filename%22%20%22%24cni_base_url%2F%24cni_filename%22%0Acni_sum%3D%24(curl%20-Lf%20%22%24cni_base_url%2F%24cni_filename.sha256%22)%0Acd%20%22%24cni_bin_dir%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cni_sum%22%0Atar%20xvf%20%22%24cni_filename%22%0Arm%20-f%20%22%24cni_filename%22%0Acd%20-%0ACRI_TOOLS_RELEASE%3D%22%24%7BCRI_TOOLS_RELEASE%3A-v1.27.0%7D%22%0Acri_tools_base_url%3D%22https%3A%2F%2Fgithub.com%2Fkubernetes-sigs%2Fcri-tools%2Freleases%2Fdownload%2F%24%7BCRI_TOOLS_RELEASE%7D%22%0Acri_tools_filename%3D%22crictl-%24%7BCRI_TOOLS_RELEASE%7D-linux-%24%7Barch%7D.tar.gz%22%0Acurl%20-Lfo%20%22%24opt_bin%2F%24cri_tools_filename%22%20%22%24cri_tools_base_url%2F%24cri_tools_filename%22%0Acri_tools_sum_value%3D%24(curl%20-Lf%20%22%24cri_tools_base_url%2F%24cri_tools_filename.sha256%22)%0Acri_tools_sum%3D%22%24cri_tools_sum_value%20%24cri_tools_filename%22%0Acd%20%22%24opt_bin%22%0Asha256sum%20-c%20%3C%3C%3C%22%24cri_tools_sum%22%0Atar%20xvf%20%22%24cri_tools_filename%22%0Arm%20-f%20%22%24cri_tools_filename%22%0Aln%20-sf%20%22%24opt_bin%2Fcrictl%22%20%22%24usr_local_bin%22%2Fcrictl%20%7C%7C%20echo%20%22symbolic%20link%20is%20skipped%22%0Acd%20-%0AKUBE_VERSION%3D%22%24%7BKUBE_VERSION%3A-v1.29.2%7D%22%0Akube_dir%3D%22%24opt_bin%2Fkubernetes-%24KUBE_VERSION%22%0Akube_base_url%3D%22https%3A%2F%2Fdl.k8s.io%2F%24KUBE_VERSION%2Fbin%2Flinux%2F%24arch%22%0Akube_sum_file%3D%22%24kube_dir%2Fsha256%22%0Amkdir%20-p%20%22%24kube_dir%22%0A%3A%20%3E%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20curl%20-Lfo%20%22%24kube_dir%2F%24bin%22%20%22%24kube_base_url%2F%24bin%22%0A%20%20%20%20chmod%20%2Bx%20%22%24kube_dir%2F%24bin%22%0A%20%20%20%20sum%3D%24(curl%20-Lf%20%22%24kube_base_url%2F%24bin.sha256%22)%0A%20%20%20%20echo%20%22%24sum%20%20%24kube_dir%2F%24bin%22%20%3E%3E%22%24kube_sum_file%22%0Adone%0Asha256sum%20-c%20%22%24kube_sum_file%22%0A%0Afor%20bin%20in%20kubelet%20kubeadm%20kubectl%3B%20do%0A%20%20%20%20ln%20-sf%20%22%24kube_dir%2F%24bin%22%20%22%24opt_bin%22%2F%24bin%0Adone%0A%0Aif%20%5B%5B%20!%20-x%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20%5D%5D%3B%20then%0A%20%20%20%20curl%20-Lfo%20%2Fopt%2Fbin%2Fhealth-monitor.sh%20https%3A%2F%2Fraw.githubusercontent.com%2Fkubermatic%2Fmachine-controller%2F7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde%2Fpkg%2Fuserdata%2Fscripts%2Fhealth-monitor.sh%0A%20%20%20%20chmod%20%2Bx%20%2Fopt%2Fbin%2Fhealth-monitor.sh%0Afi%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2Fenvironment.conf%20%2Fetc%2Fsystemd%2Fsystem%2Fdocker.service.d%2Fenvironment.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironmentFile%3D-%2Fetc%2Fenvironment%0AEOF%0A%0Amkdir%20-p%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%0A%0Acat%20%3C%3CEOF%20%7C%20tee%20%2Fetc%2Fsystemd%2Fsystem%2Fcontainerd.service.d%2F10-machine-controller.conf%0A%5BService%5D%0ARestart%3Dalways%0AEnvironment%3DCONTAINERD_CONFIG%3D%2Fetc%2Fcontainerd%2Fconfig.toml%0AExecStart%3D%0AExecStart%3D%2Fusr%2Fbin%2Fenv%20PATH%3D%5C%24%7BTORCX_BINDIR%7D%3A%5C%24%7BPATH%7D%20containerd%20--config%20%5C%24%7BCONTAINERD_CONFIG%7D%0AEOF%0A%0Asystemctl%20daemon-reload%0Asystemctl%20restart%20containerd%0A%0Asystemctl%20disable%20download-script.service%0A","verification":{}},"mode":493},{"filesystem":"root","path":"/etc/containerd/config.toml","contents":{"source":"data:,version%20%3D%202%0A%0A%5Bmetrics%5D%0Aaddress%20%3D%20%22127.0.0.1%3A1338%22%0A%0A%5Bplugins%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc%5D%0Aruntime_type%20%3D%20%22io.containerd.runc.v2%22%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.containerd.runtimes.runc.options%5D%0ASystemdCgroup%20%3D%20true%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors%5D%0A%5Bplugins.%22io.containerd.grpc.v1.cri%22.registry.mirrors.%22docker.io%22%5D%0Aendpoint%20%3D%20%5B%22https%3A%2F%2Fregistry-1.docker.io%22%5D%0A","verification":{}},"mode":420},{"filesystem":"root","path":"/etc/crictl.yaml","contents":{"source":"data:,runtime-endpoint%3A%20unix%3A%2F%2F%2Frun%2Fcontainerd%2Fcontainerd.sock%0A","verification":{}},"mode":420}]},"systemd":{"units":[{"mask":true,"name":"update-engine.service"},{"mask":true,"name":"locksmithd.service"},{"contents":"[Install]\nWantedBy=multi-user.target\n\n[Unit]\nRequires=network-online.target\nRequires=nodeip.service\nAfter=network-online.target\nAfter=nodeip.service\n\nDescription=Service responsible for configuring the flatcar machine\n\n[Service]\nType=oneshot\nRemainAfterExit=true\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/setup.sh\n","enabled":true,"name":"setup.service"},{"contents":"[Unit]\nRequires=network-online.target\nRequires=setup.service\nAfter=network-online.target\nAfter=setup.service\n[Service]\nType=oneshot\nEnvironmentFile=-/etc/environment\nExecStart=/opt/bin/download.sh\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"download-script.service"},{"contents":"[Unit]\nRequires=kubelet.service\nAfter=kubelet.service\n\n[Service]\nExecStart=/opt/bin/health-monitor.sh kubelet\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet-healthcheck.service"},{"contents":"[Unit]\nDescription=Setup Kubelet Node IP Env\nRequires=network-online.target\nAfter=network-online.target\n\n[Service]\nExecStart=/opt/bin/setup_net_env.sh\nRemainAfterExit=yes\nType=oneshot\n[Install]\nWantedBy=multi-user.target\n","enabled":true,"name":"nodeip.service"},{"contents":"[Unit]\nAfter=containerd.service\nRequires=containerd.service\n\nDescription=kubelet: The Kubernetes Node Agent\nDocumentation=https://kubernetes.io/docs/home/\n\n[Service]\nUser=root\nRestart=always\nStartLimitInterval=0\nRestartSec=10\nCPUAccounting=true\nMemoryAccounting=true\n\nEnvironment=\"PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/\"\nEnvironmentFile=-/etc/environment\n\nExecStartPre=/bin/bash /opt/load-kernel-modules.sh\n\nExecStartPre=/bin/bash /opt/bin/setup_net_env.sh\nExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \\\n --kubeconfig=/var/lib/kubelet/kubeconfig \\\n --config=/etc/kubernetes/kubelet.conf \\\n --cert-dir=/etc/kubernetes/pki \\\n --cloud-provider=vsphere \\\n --cloud-config=/etc/kubernetes/cloud-config \\\n --hostname-override=node1 \\\n --exit-on-lock-contention \\\n --lock-file=/tmp/kubelet.lock \\\n --container-runtime-endpoint=unix:///run/containerd/containerd.sock \\\n --node-ip ${KUBELET_NODE_IP}\n\n[Install]\nWantedBy=multi-user.target\n","dropins":[{"contents":"[Service]\nEnvironmentFile=/etc/kubernetes/nodeip.conf\n","name":"10-nodeip.conf"},{"contents":"[Service]\nEnvironment=\"KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf\"\n","name":"resolv.conf"},{"contents":"[Unit]\nRequires=download-script.service\nAfter=download-script.service\n","name":"40-download.conf"}],"enabled":true,"name":"kubelet.service"}]}} \ No newline at end of file diff --git a/pkg/userdata/helper/common_test.go b/pkg/userdata/helper/common_test.go index db213784b..71c2857f2 100644 --- a/pkg/userdata/helper/common_test.go +++ b/pkg/userdata/helper/common_test.go @@ -26,9 +26,8 @@ var update = flag.Bool("update", false, "update testdata files") var ( versions = []*semver.Version{ - semver.MustParse("v1.26.12"), - semver.MustParse("v1.27.9"), - semver.MustParse("v1.28.5"), - semver.MustParse("v1.29.0"), + semver.MustParse("v1.27.11"), + semver.MustParse("v1.28.7"), + semver.MustParse("v1.29.2"), } ) diff --git a/pkg/userdata/helper/download_binaries_script_test.go b/pkg/userdata/helper/download_binaries_script_test.go index 786e153ac..870469c37 100644 --- a/pkg/userdata/helper/download_binaries_script_test.go +++ b/pkg/userdata/helper/download_binaries_script_test.go @@ -42,9 +42,9 @@ func TestDownloadBinariesScript(t *testing.T) { } func TestSafeDownloadBinariesScript(t *testing.T) { - name := "safe_download_binaries_v1.26.6" + name := "safe_download_binaries_v1.29.2" t.Run(name, func(t *testing.T) { - script, err := SafeDownloadBinariesScript(zap.NewNop().Sugar(), "v1.26.6") + script, err := SafeDownloadBinariesScript(zap.NewNop().Sugar(), "v1.29.2") if err != nil { t.Error(err) } diff --git a/pkg/userdata/helper/kubelet_test.go b/pkg/userdata/helper/kubelet_test.go index 0b71d10d1..be2dc79b2 100644 --- a/pkg/userdata/helper/kubelet_test.go +++ b/pkg/userdata/helper/kubelet_test.go @@ -66,7 +66,7 @@ func TestKubeletSystemdUnit(t *testing.T) { tests = append(tests, []kubeletFlagTestCase{ { name: "multiple-dns-servers", - version: semver.MustParse("v1.26.6"), + version: semver.MustParse("v1.29.2"), dnsIPs: []net.IP{ net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), @@ -76,14 +76,14 @@ func TestKubeletSystemdUnit(t *testing.T) { }, { name: "cloud-provider-set", - version: semver.MustParse("v1.26.6"), + version: semver.MustParse("v1.29.2"), dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, hostname: "some-test-node", cloudProvider: "aws", }, { name: "pause-image-set", - version: semver.MustParse("v1.26.6"), + version: semver.MustParse("v1.29.2"), dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, hostname: "some-test-node", cloudProvider: "aws", @@ -91,7 +91,7 @@ func TestKubeletSystemdUnit(t *testing.T) { }, { name: "taints-set", - version: semver.MustParse("v1.26.6"), + version: semver.MustParse("v1.29.2"), dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, hostname: "some-test-node", cloudProvider: "aws", diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.26.12.golden b/pkg/userdata/helper/testdata/download_binaries_v1.27.11.golden similarity index 91% rename from pkg/userdata/helper/testdata/download_binaries_v1.26.12.golden rename to pkg/userdata/helper/testdata/download_binaries_v1.27.11.golden index fe50c1a8f..e5619f541 100644 --- a/pkg/userdata/helper/testdata/download_binaries_v1.26.12.golden +++ b/pkg/userdata/helper/testdata/download_binaries_v1.27.11.golden @@ -7,7 +7,7 @@ if [ ! -f /opt/cni/bin/loopback ]; then curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - fi if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.26.12/bin/linux/amd64/kubelet + curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.27.11/bin/linux/amd64/kubelet chmod +x /opt/bin/kubelet fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.28.7.golden b/pkg/userdata/helper/testdata/download_binaries_v1.28.7.golden new file mode 100644 index 000000000..103682b02 --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.28.7.golden @@ -0,0 +1,17 @@ +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - +fi +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.28.7/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.29.2.golden b/pkg/userdata/helper/testdata/download_binaries_v1.29.2.golden new file mode 100644 index 000000000..214208182 --- /dev/null +++ b/pkg/userdata/helper/testdata/download_binaries_v1.29.2.golden @@ -0,0 +1,17 @@ +mkdir -p /opt/bin/ +mkdir -p /var/lib/calico +mkdir -p /etc/kubernetes/manifests +mkdir -p /etc/cni/net.d +mkdir -p /opt/cni/bin +if [ ! -f /opt/cni/bin/loopback ]; then + curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - +fi +if [ ! -f /opt/bin/kubelet ]; then + curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.29.2/bin/linux/amd64/kubelet + chmod +x /opt/bin/kubelet +fi + +if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh +fi diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.12-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.11-external.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.12-external.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.11-external.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.12.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.11.golden similarity index 100% rename from pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.26.12.golden rename to pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.11.golden diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.7-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.7-external.golden new file mode 100644 index 000000000..50f4f5138 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.7-external.golden @@ -0,0 +1,36 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.7.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.7.golden new file mode 100644 index 000000000..e70567560 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.7.golden @@ -0,0 +1,35 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.2-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.2-external.golden new file mode 100644 index 000000000..50f4f5138 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.2-external.golden @@ -0,0 +1,36 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.2.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.2.golden new file mode 100644 index 000000000..e70567560 --- /dev/null +++ b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.2.golden @@ -0,0 +1,35 @@ +[Unit] +After=docker.service +Requires=docker.service + +Description=kubelet: The Kubernetes Node Agent +Documentation=https://kubernetes.io/docs/home/ + +[Service] +User=root +Restart=always +StartLimitInterval=0 +RestartSec=10 +CPUAccounting=true +MemoryAccounting=true + +Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" +EnvironmentFile=-/etc/environment + +ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + +ExecStartPre=/bin/bash /opt/disable-swap.sh + +ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh +ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=some-test-node \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --node-ip ${KUBELET_NODE_IP} + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/safe_download_binaries_v1.26.6.golden b/pkg/userdata/helper/testdata/safe_download_binaries_v1.29.2.golden similarity index 98% rename from pkg/userdata/helper/testdata/safe_download_binaries_v1.26.6.golden rename to pkg/userdata/helper/testdata/safe_download_binaries_v1.29.2.golden index 7a61bdb36..eb0c7e5bc 100644 --- a/pkg/userdata/helper/testdata/safe_download_binaries_v1.26.6.golden +++ b/pkg/userdata/helper/testdata/safe_download_binaries_v1.29.2.golden @@ -40,7 +40,7 @@ tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - -KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" +KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index 263611a1a..e972a47e5 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -120,40 +120,40 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "kubelet-v1.26.6-aws", + name: "kubelet-v1.29.2-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, }, { - name: "kubelet-v1.26.6-aws-external", + name: "kubelet-v1.29.2-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.26.6-vsphere", + name: "kubelet-v1.29.2-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.26.6-vsphere-proxy", + name: "kubelet-v1.29.2-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), @@ -163,11 +163,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.26.6-vsphere-mirrors", + name: "kubelet-v1.29.2-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), @@ -176,25 +176,6 @@ func TestUserDataGeneration(t *testing.T) { registryMirrors: "https://registry.docker-cn.com/", pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, - { - name: "kubelet-v1.26-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.0", - }, - }, - }, - { - name: "kubelet-v1.26-aws-external", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.0", - }, - }, - externalCloudProvider: true, - }, { name: "kubelet-v1.28-nutanix", spec: clusterv1alpha1.MachineSpec{ diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml deleted file mode 100644 index 8ea647e61..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws-external.yaml +++ /dev/null @@ -1,501 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=${KUBELET_HOSTNAME} \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml deleted file mode 100644 index 16e89d5ca..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws.yaml +++ /dev/null @@ -1,501 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-aws-external.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.29.2-aws-external.yaml index c18fc6ee3..ff0843c84 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-aws-external.yaml @@ -143,7 +143,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-aws.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.29.2-aws.yaml index fdf385829..856bcf147 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-aws.yaml @@ -143,7 +143,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml index ef101a03a..816d62598 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml @@ -157,7 +157,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-proxy.yaml index 1f12a40d8..94a100d05 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere-proxy.yaml @@ -157,7 +157,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere.yaml similarity index 99% rename from pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere.yaml index 506a105a5..9f95a1d66 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.29.2-vsphere.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/provider_test.go b/pkg/userdata/rockylinux/provider_test.go index 3a9e7d596..f5c883cb1 100644 --- a/pkg/userdata/rockylinux/provider_test.go +++ b/pkg/userdata/rockylinux/provider_test.go @@ -111,49 +111,40 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "kubelet-v1.29-aws", + name: "kubelet-v1.29.2-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.0", + Kubelet: "1.29.2", }, }, }, { - name: "kubelet-v1.26.6-aws", + name: "kubelet-v1.29.2-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", - }, - }, - }, - { - name: "kubelet-v1.26.6-aws-external", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.26.6-vsphere", + name: "kubelet-v1.29.2-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.26.6-vsphere-proxy", + name: "kubelet-v1.29.2-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), @@ -163,11 +154,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.26.6-vsphere-mirrors", + name: "kubelet-v1.29.2-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("vsphere"), @@ -177,11 +168,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.26.6-nutanix", + name: "kubelet-v1.29.2-nutanix", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.6", + Kubelet: "1.29.2", }, }, cloudProviderName: stringPtr("nutanix"), diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-aws-external.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-aws-external.yaml index bdfb9bedc..9d84e778d 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-aws-external.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-aws.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-aws.yaml index 0ce7d5bf7..10ed4d353 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-aws.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-nutanix.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-nutanix.yaml index 99d10681d..36101c601 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-nutanix.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml index bd4e68418..0f70ac398 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml @@ -157,7 +157,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-proxy.yaml index 3797ad98d..4a6d9a1ba 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-proxy.yaml @@ -157,7 +157,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere.yaml index 131883af6..a3c2c5a45 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.26.6-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index ae172d5a7..492c9fcd4 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -129,10 +129,9 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.26.12"), - semver.MustParse("v1.27.9"), - semver.MustParse("v1.28.5"), - semver.MustParse("v1.29.0"), + semver.MustParse("v1.27.11"), + semver.MustParse("v1.28.7"), + semver.MustParse("v1.29.2"), } var tests []userDataTestCase @@ -603,7 +602,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.26.12", + Kubelet: "1.29.0", }, }, ccProvider: &fakeCloudConfigProvider{ diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index f1f76860a..cea7dc61f 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -153,7 +153,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.12}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/ubuntu/testdata/version-1.26.12.yaml b/pkg/userdata/ubuntu/testdata/version-1.27.11.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.26.12.yaml rename to pkg/userdata/ubuntu/testdata/version-1.27.11.yaml index 16da2588e..8cf0a7d5c 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.26.12.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.27.11.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.26.12}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.11}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml b/pkg/userdata/ubuntu/testdata/version-1.28.7.yaml similarity index 90% rename from pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml rename to pkg/userdata/ubuntu/testdata/version-1.28.7.yaml index 68dc04d43..2b99ca5a9 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.26.6-aws.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.28.7.yaml @@ -1,7 +1,11 @@ #cloud-config +hostname: node1 + ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" write_files: @@ -41,61 +45,64 @@ write_files: fs.inotify.max_user_instances = 8192 -- path: /etc/selinux/config +- path: "/etc/default/grub.d/60-swap-accounting.cfg" content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - path: "/opt/bin/setup" permissions: "0755" content: | #!/bin/bash set -xeuo pipefail - - setenforce 0 || true + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw systemctl restart systemd-modules-load.service sysctl --system + apt-get update - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ ebtables \ ethtool \ - nfs-utils \ - bash-completion \ - sudo \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ socat \ - wget \ - curl \ + util-linux \ ipvsadm + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - mkdir -p /etc/systemd/system/containerd.service.d + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + cat < /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io/"] + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + containerRuntimeEndpoint: "" + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMaximumGCAge: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index c0f75ad41..95018f7b4 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -84,7 +84,7 @@ const ( ) const ( - defaultKubernetesVersion = "1.28.5" + defaultKubernetesVersion = "1.28.7" awsDefaultKubernetesVersion = "1.26.12" defaultContainerRuntime = "containerd" ) @@ -344,7 +344,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.26. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.26.12", "1.27.9", "1.28.5", "1.29.0"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.27.11", "1.28.7", "1.29.2"))) runScenarios(t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -424,7 +424,7 @@ func TestAWSProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := Not(VersionSelector("1.27.9", "1.28.5", "1.29.0")) + selector := Not(VersionSelector("1.27.11", "1.28.7", "1.29.2")) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -478,7 +478,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.9", "1.28.5", "1.29.0"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.11", "1.28.7", "1.29.2"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -500,7 +500,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.9", "1.28.5", "1.29.0"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.11", "1.28.7", "1.29.2"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -685,7 +685,7 @@ func TestGCEProvisioningE2E(t *testing.T) { } // Act. GCE does not support CentOS. - selector := OsSelector("ubuntu", "flatcar") + selector := And(OsSelector("ubuntu", "flatcar"), Not(VersionSelector("1.29.2"))) params := []string{ fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>=%s", safeBase64Encoding(googleServiceAccount)), } diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 6783aaf80..aff05972f 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,10 +33,9 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.26.12"), - semver.MustParse("v1.27.9"), - semver.MustParse("v1.28.5"), - semver.MustParse("v1.29.0"), + semver.MustParse("v1.27.11"), + semver.MustParse("v1.28.7"), + semver.MustParse("v1.29.2"), } operatingSystems = []providerconfigtypes.OperatingSystem{ From 9e086c43016ed5a41fd7b6adf4d541eb1bbaa345 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 7 Mar 2024 11:59:42 +0500 Subject: [PATCH 379/489] Upgrade to Go 1.22.1 (#1770) Signed-off-by: Waleed Malik --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 16 ++++++++-------- .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 4 ++-- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 8 ++++---- .prow/verify.yaml | 8 ++++---- Dockerfile | 2 +- Makefile | 2 +- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- 22 files changed, 43 insertions(+), 43 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 7fd8eff78..a550bf6fe 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -119,7 +119,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index d94aa85a6..dd380aa5a 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index 1c7a1af7c..7e6e8d709 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index deb91e618..defbcca80 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 709f72dba..3db39d632 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -95,7 +95,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -126,7 +126,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -159,7 +159,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -190,7 +190,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -221,7 +221,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -252,7 +252,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index f06fa7dd0..1cb8c860b 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 0a2ee8929..bed60b3bd 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 986bb7faf..059766bcc 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 28aa9ef2a..5f3721c08 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 7205e1492..b8655e8f5 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index ad712a605..adb808ae5 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index def0cb7de..12a0711ed 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 4c05da09b..31e53cf54 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 8213afc44..22ff68180 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index c9a33e4fc..0fa091d99 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index bc09746d8..8be8ec733 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 71ea8ab3e..8fe107ed3 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index edee2d747..0ba98100b 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.22.0 + - image: golang:1.22.1 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.22.0 + - image: golang:1.22.1 command: - make args: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.22.0 + - image: golang:1.22.1 command: - make args: diff --git a/Dockerfile b/Dockerfile index 5651e7673..68bd62d93 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.22.0 +ARG GO_VERSION=1.22.1 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index 20a574f84..c60377ee6 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.22.0 +GO_VERSION ?= 1.22.1 GOOS ?= $(shell go env GOOS) diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 8657d5ab6..5200ec615 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.22.0 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.22.1 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 3646eec8f..5465f0aef 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-18-kind-0.21-3 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 containerize ./hack/verify-licenses.sh go mod vendor From fb5b68d4617fbf0b589526facd405c7405e0b654 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 10:28:01 +0100 Subject: [PATCH 380/489] Bump google.golang.org/protobuf from 1.32.0 to 1.33.0 (#1771) Bumps google.golang.org/protobuf from 1.32.0 to 1.33.0. --- updated-dependencies: - dependency-name: google.golang.org/protobuf dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2baec178e..20523edf5 100644 --- a/go.mod +++ b/go.mod @@ -185,7 +185,7 @@ require ( google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect - google.golang.org/protobuf v1.32.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 0d21ddaa0..c94dc9cd6 100644 --- a/go.sum +++ b/go.sum @@ -1267,8 +1267,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= From 046343d124aab721a5035b957e448fb64234227a Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Thu, 21 Mar 2024 12:36:08 +0100 Subject: [PATCH 381/489] Bump dependencies (#1772) * bump dependencies * make it less likely those jobs drift away from the build image's toolkit * update to VCD API changes * lint: unused parameters * Wait() is deprecated in favor of WaitEx * lint: tautologies, impossible conditions (slightly changes VCD validation) * update deprecated azure SDK usage * ignore deprecation in hetzner SDK --- .golangci.yml | 4 + .prow/verify.yaml | 20 +- go.mod | 133 ++++---- go.sum | 283 +++++++++--------- .../provider/anexia/helper_test.go | 2 +- pkg/cloudprovider/provider/anexia/provider.go | 3 - .../provider/anexia/provider_test.go | 14 +- .../provider/azure/create_delete_resources.go | 10 +- .../provider/azure/get_client.go | 4 +- pkg/cloudprovider/provider/azure/provider.go | 12 +- pkg/cloudprovider/provider/gce/service.go | 2 +- .../provider/hetzner/provider.go | 2 + .../provider/openstack/provider.go | 2 +- .../provider/openstack/provider_test.go | 2 +- .../provider/vmwareclouddirector/helper.go | 17 +- pkg/cloudprovider/provider/vsphere/client.go | 3 - pkg/cloudprovider/provider/vsphere/helper.go | 4 +- .../provider/vsphere/provider.go | 10 +- pkg/cloudprovider/provider/vsphere/rule.go | 4 +- pkg/clusterinfo/configmap.go | 18 +- pkg/controller/machine/controller.go | 2 +- .../machinedeployment/controller.go | 2 +- pkg/controller/machinedeployment/sync.go | 8 +- pkg/controller/machineset/controller.go | 2 +- pkg/controller/nodecsrapprover/controller.go | 2 +- pkg/rhsm/util.go | 4 +- test/e2e/provisioning/all_e2e_test.go | 77 ++--- test/e2e/provisioning/deploymentscenario.go | 15 +- test/e2e/provisioning/helper.go | 11 +- test/e2e/provisioning/migrateuidscenario.go | 8 +- test/e2e/provisioning/verify.go | 58 ++-- 31 files changed, 366 insertions(+), 372 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index ea70b8697..028e270b7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -49,6 +49,10 @@ linters-settings: - { pkg: io/ioutil, desc: https://go.dev/doc/go1.16#ioutil } - { pkg: github.com/ghodss/yaml, desc: use sigs.k8s.io/yaml instead } + govet: + enable: + - nilness # find tautologies / impossible conditions + issues: exclude: - should have comment or be unexported diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 0ba98100b..40447e5c3 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.22.1 + - image: quay.io/kubermatic/build:go-1.22-node-18-5 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.22.1 + - image: quay.io/kubermatic/build:go-1.22-node-18-5 command: - make args: @@ -63,7 +63,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golangci/golangci-lint:v1.56.0 + - image: quay.io/kubermatic/build:go-1.22-node-18-5 command: - make args: @@ -83,7 +83,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-3 + - image: quay.io/kubermatic/build:go-1.22-node-18-5 command: - make args: @@ -102,7 +102,7 @@ presubmits: clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" spec: containers: - - image: docker.io/mvdan/shfmt:v3.3.1 + - image: quay.io/kubermatic/build:go-1.22-node-18-5 command: - "/bin/shfmt" args: @@ -130,7 +130,7 @@ presubmits: clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" spec: containers: - - image: quay.io/kubermatic-labs/boilerplate:v0.2.0 + - image: quay.io/kubermatic/build:go-1.22-node-18-5 command: - "./hack/verify-boilerplate.sh" resources: @@ -149,13 +149,13 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-18-5 command: - ./hack/verify-licenses.sh resources: requests: - memory: 2Gi - cpu: 2 + memory: 1Gi + cpu: 1 - name: pull-machine-controller-test always_run: true @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.22.1 + - image: quay.io/kubermatic/build:go-1.22-node-18-5 command: - make args: diff --git a/go.mod b/go.mod index 20523edf5..f14ee922f 100644 --- a/go.mod +++ b/go.mod @@ -6,74 +6,74 @@ toolchain go1.21.5 require ( cloud.google.com/go/logging v1.9.0 - cloud.google.com/go/monitoring v1.17.0 - github.com/Azure/azure-sdk-for-go v65.0.0+incompatible + cloud.google.com/go/monitoring v1.18.1 + github.com/Azure/azure-sdk-for-go v68.0.0+incompatible github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/BurntSushi/toml v1.3.2 github.com/Masterminds/semver/v3 v3.2.1 github.com/Masterminds/sprig/v3 v3.2.3 - github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240104224209-592a2a64f555 - github.com/aliyun/alibaba-cloud-sdk-go v1.62.654 - github.com/aws/aws-sdk-go-v2 v1.24.1 - github.com/aws/aws-sdk-go-v2/config v1.26.3 - github.com/aws/aws-sdk-go-v2/credentials v1.16.14 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.142.1 - github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 - github.com/aws/smithy-go v1.19.0 + github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240311170802-57efabf2d590 + github.com/aliyun/alibaba-cloud-sdk-go v1.62.695 + github.com/aws/aws-sdk-go-v2 v1.25.3 + github.com/aws/aws-sdk-go-v2/config v1.27.7 + github.com/aws/aws-sdk-go-v2/credentials v1.17.7 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.151.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.28.4 + github.com/aws/smithy-go v1.20.1 github.com/davecgh/go-spew v1.1.1 - github.com/digitalocean/godo v1.107.0 + github.com/digitalocean/godo v1.110.0 github.com/flatcar/container-linux-config-transpiler v0.9.4 github.com/go-logr/logr v1.4.1 github.com/go-logr/zapr v1.3.0 - github.com/go-test/deep v1.0.8 - github.com/google/uuid v1.5.0 - github.com/gophercloud/gophercloud v1.8.0 + github.com/go-test/deep v1.1.0 + github.com/google/uuid v1.6.0 + github.com/gophercloud/gophercloud v1.11.0 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb - github.com/hetznercloud/hcloud-go v1.39.0 - github.com/linode/linodego v1.26.0 + github.com/hetznercloud/hcloud-go v1.53.0 + github.com/linode/linodego v1.30.0 github.com/nutanix-cloud-native/prism-go-client v0.3.4 github.com/packethost/packngo v0.31.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 - github.com/prometheus/client_golang v1.18.0 - github.com/scaleway/scaleway-sdk-go v1.0.0-beta.22 + github.com/prometheus/client_golang v1.19.0 + github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25 github.com/sethvargo/go-password v0.2.0 github.com/spf13/pflag v1.0.5 github.com/tinkerbell/tink v0.8.0 - github.com/vmware/go-vcloud-director/v2 v2.21.0 - github.com/vmware/govmomi v0.34.2 - github.com/vultr/govultr/v3 v3.6.0 - go.anx.io/go-anxcloud v0.6.2 - go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.17.0 - golang.org/x/oauth2 v0.15.0 + github.com/vmware/go-vcloud-director/v2 v2.22.0 + github.com/vmware/govmomi v0.36.1 + github.com/vultr/govultr/v3 v3.6.4 + go.anx.io/go-anxcloud v0.6.4 + go.uber.org/zap v1.27.0 + golang.org/x/crypto v0.21.0 + golang.org/x/oauth2 v0.18.0 gomodules.xyz/jsonpatch/v2 v2.4.0 - google.golang.org/api v0.155.0 - google.golang.org/grpc v1.60.1 + google.golang.org/api v0.170.0 + google.golang.org/grpc v1.62.1 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.29.1 - k8s.io/apiextensions-apiserver v0.29.1 - k8s.io/apimachinery v0.29.1 - k8s.io/client-go v0.29.1 - k8s.io/cloud-provider v0.29.1 + k8s.io/api v0.29.3 + k8s.io/apiextensions-apiserver v0.29.3 + k8s.io/apimachinery v0.29.3 + k8s.io/client-go v0.29.3 + k8s.io/cloud-provider v0.29.3 k8s.io/klog v1.0.0 - k8s.io/kubelet v0.29.1 - k8s.io/utils v0.0.0-20240102154912-e7106e64919e - kubevirt.io/api v1.1.1 - kubevirt.io/containerized-data-importer-api v1.58.0 - sigs.k8s.io/controller-runtime v0.17.0 + k8s.io/kubelet v0.29.3 + k8s.io/utils v0.0.0-20240310230437-4693a0247e57 + kubevirt.io/api v1.2.0 + kubevirt.io/containerized-data-importer-api v1.58.1 + sigs.k8s.io/controller-runtime v0.17.2 sigs.k8s.io/yaml v1.4.0 ) require ( - cloud.google.com/go v0.111.0 // indirect - cloud.google.com/go/compute v1.23.3 // indirect + cloud.google.com/go v0.112.1 // indirect + cloud.google.com/go/compute v1.24.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/longrunning v0.5.4 // indirect + cloud.google.com/go/longrunning v0.5.5 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.29 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect @@ -88,14 +88,14 @@ require ( github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -119,7 +119,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -127,7 +127,7 @@ require ( github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.0 // indirect + github.com/googleapis/gax-go/v2 v2.12.2 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.5 // indirect github.com/hashicorp/go-version v1.6.0 // indirect @@ -141,15 +141,14 @@ require ( github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/ginkgo/v2 v2.14.0 // indirect - github.com/onsi/gomega v1.30.0 // indirect + github.com/onsi/ginkgo/v2 v2.16.0 // indirect + github.com/onsi/gomega v1.31.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect @@ -157,7 +156,7 @@ require ( github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046 // indirect github.com/peterhellberg/link v1.2.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect @@ -165,33 +164,33 @@ require ( github.com/spf13/cobra v1.8.0 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect - go.opentelemetry.io/otel v1.21.0 // indirect - go.opentelemetry.io/otel/metric v1.21.0 // indirect - go.opentelemetry.io/otel/trace v1.21.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect - golang.org/x/net v0.19.0 // indirect + golang.org/x/net v0.22.0 // indirect golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.16.0 // indirect - golang.org/x/term v0.16.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.16.1 // indirect + golang.org/x/tools v0.17.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect + google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/component-base v0.29.1 // indirect + k8s.io/component-base v0.29.3 // indirect k8s.io/klog/v2 v2.110.1 // indirect k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect diff --git a/go.sum b/go.sum index c94dc9cd6..d489d67d3 100644 --- a/go.sum +++ b/go.sum @@ -18,28 +18,28 @@ cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmW cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.111.0 h1:YHLKNupSD1KqjDbQ3+LVdQ81h/UJbJyZG203cEfnQgM= -cloud.google.com/go v0.111.0/go.mod h1:0mibmpKP1TyOOFYQY5izo0LnT+ecvOQ0Sg3OdmMiNRU= +cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= +cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= -cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= +cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= +cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= -cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= +cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= +cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= cloud.google.com/go/logging v1.9.0 h1:iEIOXFO9EmSiTjDmfpbRjOxECO7R8C7b8IXUGOj7xZw= cloud.google.com/go/logging v1.9.0/go.mod h1:1Io0vnZv4onoUnsVUQY3HZ3Igb1nBchky0A0y7BBBhE= -cloud.google.com/go/longrunning v0.5.4 h1:w8xEcbZodnA2BbW6sVirkkoC+1gP8wS57EUUgGS0GVg= -cloud.google.com/go/longrunning v0.5.4/go.mod h1:zqNVncI0BOP8ST6XQD1+VcvuShMmq7+xFSzOL++V0dI= -cloud.google.com/go/monitoring v1.17.0 h1:blrdvF0MkPPivSO041ihul7rFMhXdVp8Uq7F59DKXTU= -cloud.google.com/go/monitoring v1.17.0/go.mod h1:KwSsX5+8PnXv5NJnICZzW2R8pWTis8ypC4zmdRD63Tw= +cloud.google.com/go/longrunning v0.5.5 h1:GOE6pZFdSrTb4KAiKnXsJBtlE6mEyaW44oKyMILWnOg= +cloud.google.com/go/longrunning v0.5.5/go.mod h1:WV2LAxD8/rg5Z1cNW6FJ/ZpX4E4VnDnoTk0yawPBB7s= +cloud.google.com/go/monitoring v1.18.1 h1:0yvFXK+xQd95VKo6thndjwnJMno7c7Xw1CwMByg0B+8= +cloud.google.com/go/monitoring v1.18.1/go.mod h1:52hTzJ5XOUMRm7jYi7928aEdVxBEmGwA0EjNJXIBvt8= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -52,8 +52,8 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/99designs/gqlgen v0.15.1 h1:48bRXecwlCNTa/n2bMSp2rQsXNxwZ54QHbiULNf78ec= github.com/99designs/gqlgen v0.15.1/go.mod h1:nbeSjFkqphIqpZsYe1ULVz0yfH8hjpJdJIQoX/e0G2I= -github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= -github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= +github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= @@ -97,8 +97,8 @@ github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240104224209-592a2a64f555 h1:kWb9OISprBC94fTeagHWzz+TQOx5IrwQOY88JyEVNjc= -github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240104224209-592a2a64f555/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= +github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240311170802-57efabf2d590 h1:wvNejQUL/d0Z2n4DZfAtAQv+/fUFrFSkLj3X49ioDiM= +github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240311170802-57efabf2d590/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= github.com/PaesslerAG/gval v1.2.2 h1:Y7iBzhgE09IGTt5QgGQ2IdaYYYOU134YGHBThD+wm9E= github.com/PaesslerAG/gval v1.2.2/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac= @@ -122,8 +122,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs= github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.654 h1:UpBbuyd0eqDkIfiuRmBGqdjXWd4Q7YwD9entykxwlnI= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.654/go.mod h1:CJJYa1ZMxjlN/NbXEwmejEnBkhi0DV+Yb3B2lxf+74o= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.695 h1:Lk9qjMhhkzZaD4eyx23v0E2+4nAIfwreJ/ecKdaTU6E= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.695/go.mod h1:CJJYa1ZMxjlN/NbXEwmejEnBkhi0DV+Yb3B2lxf+74o= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -134,34 +134,34 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/aws/aws-sdk-go v1.42.23/go.mod h1:gyRszuZ/icHmHAVE4gc/r+cfCmhA1AD+vqfWbgI+eHs= -github.com/aws/aws-sdk-go-v2 v1.24.1 h1:xAojnj+ktS95YZlDf0zxWBkbFtymPeDP+rvUQIH3uAU= -github.com/aws/aws-sdk-go-v2 v1.24.1/go.mod h1:LNh45Br1YAkEKaAqvmE1m8FUx6a5b/V0oAKV7of29b4= -github.com/aws/aws-sdk-go-v2/config v1.26.3 h1:dKuc2jdp10y13dEEvPqWxqLoc0vF3Z9FC45MvuQSxOA= -github.com/aws/aws-sdk-go-v2/config v1.26.3/go.mod h1:Bxgi+DeeswYofcYO0XyGClwlrq3DZEXli0kLf4hkGA0= -github.com/aws/aws-sdk-go-v2/credentials v1.16.14 h1:mMDTwwYO9A0/JbOCOG7EOZHtYM+o7OfGWfu0toa23VE= -github.com/aws/aws-sdk-go-v2/credentials v1.16.14/go.mod h1:cniAUh3ErQPHtCQGPT5ouvSAQ0od8caTO9OOuufZOAE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11 h1:c5I5iH+DZcH3xOIMlz3/tCKJDaHFwYEmxvlh2fAcFo8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.11/go.mod h1:cRrYDYAMUohBJUtUnOhydaMHtiK/1NZ0Otc9lIb6O0Y= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10 h1:vF+Zgd9s+H4vOXd5BMaPWykta2a6Ih0AKLq/X6NYKn4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.10/go.mod h1:6BkRjejp/GR4411UGqkX8+wFMbFbqsUIimfK4XjOKR4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10 h1:nYPe006ktcqUji8S2mqXf9c/7NdiKriOwMvWQHgYztw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.10/go.mod h1:6UV4SZkVvmODfXKql4LCbaZUpF7HO2BX38FgBf9ZOLw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2 h1:GrSw8s0Gs/5zZ0SX+gX4zQjRnRsMJDJ2sLur1gRBhEM= -github.com/aws/aws-sdk-go-v2/internal/ini v1.7.2/go.mod h1:6fQQgfuGmw8Al/3M2IgIllycxV7ZW7WCdVSqfBeUiCY= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.142.1 h1:tTAfm9YsKlmlv6ORgco838e0ZeAcGVRkgevseiYO0gU= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.142.1/go.mod h1:hIsHE0PaWAQakLCshKS7VKWMGXaqrAFp4m95s2W9E6c= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4 h1:/b31bi3YVNlkzkBrm9LfpaKoaYZUxIAj4sHfOTmLfqw= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.10.4/go.mod h1:2aGXHFmbInwgP9ZfpmdIfOELL79zhdNYNmReK8qDfdQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10 h1:DBYTXwIGQSGs9w4jKm60F5dmCQ3EEruxdc0MFh+3EY4= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.10/go.mod h1:wohMUQiFdzo0NtxbBg0mSRGZ4vL3n0dKjLTINdcIino= -github.com/aws/aws-sdk-go-v2/service/sso v1.18.6 h1:dGrs+Q/WzhsiUKh82SfTVN66QzyulXuMDTV/G8ZxOac= -github.com/aws/aws-sdk-go-v2/service/sso v1.18.6/go.mod h1:+mJNDdF+qiUlNKNC3fxn74WWNN+sOiGOEImje+3ScPM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6 h1:Yf2MIo9x+0tyv76GljxzqA3WtC5mw7NmazD2chwjxE4= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.21.6/go.mod h1:ykf3COxYI0UJmxcfcxcVuz7b6uADi1FkiUz6Eb7AgM8= -github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 h1:NzO4Vrau795RkUdSHKEwiR01FaGzGOH1EETJ+5QHnm0= -github.com/aws/aws-sdk-go-v2/service/sts v1.26.7/go.mod h1:6h2YuIoxaMSCFf5fi1EgZAwdfkGMgDY+DVfa61uLe4U= -github.com/aws/smithy-go v1.19.0 h1:KWFKQV80DpP3vJrrA9sVAHQ5gc2z8i4EzrLhLlWXcBM= -github.com/aws/smithy-go v1.19.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +github.com/aws/aws-sdk-go-v2 v1.25.3 h1:xYiLpZTQs1mzvz5PaI6uR0Wh57ippuEthxS4iK5v0n0= +github.com/aws/aws-sdk-go-v2 v1.25.3/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I= +github.com/aws/aws-sdk-go-v2/config v1.27.7 h1:JSfb5nOQF01iOgxFI5OIKWwDiEXWTyTgg1Mm1mHi0A4= +github.com/aws/aws-sdk-go-v2/config v1.27.7/go.mod h1:PH0/cNpoMO+B04qET699o5W92Ca79fVtbUnvMIZro4I= +github.com/aws/aws-sdk-go-v2/credentials v1.17.7 h1:WJd+ubWKoBeRh7A5iNMnxEOs982SyVKOJD+K8HIezu4= +github.com/aws/aws-sdk-go-v2/credentials v1.17.7/go.mod h1:UQi7LMR0Vhvs+44w5ec8Q+VS+cd10cjwgHwiVkE0YGU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3 h1:p+y7FvkK2dxS+FEwRIDHDe//ZX+jDhP8HHE50ppj4iI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3/go.mod h1:/fYB+FZbDlwlAiynK9KDXlzZl3ANI9JkD0Uhz5FjNT4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3 h1:ifbIbHZyGl1alsAhPIYsHOg5MuApgqOvVeI8wIugXfs= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3/go.mod h1:oQZXg3c6SNeY6OZrDY+xHcF4VGIEoNotX2B4PrDeoJI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3 h1:Qvodo9gHG9F3E8SfYOspPeBt0bjSbsevK8WhRAUHcoY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3/go.mod h1:vCKrdLXtybdf/uQd/YfVR2r5pcbNuEYKzMQpcxmeSJw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.151.0 h1:gH571JR1hMfIER4zK457aNjCfi1FCuVwriKx0bAyw/I= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.151.0/go.mod h1:KNJMjsbzK97hci9ev2Vl/27GgUt3ZciRP4RGujAPF2I= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5 h1:K/NXvIftOlX+oGgWGIa3jDyYLDNsdVhsjHmsBH2GLAQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5/go.mod h1:cl9HGLV66EnCmMNzq4sYOti+/xo8w34CsgzVtm2GgsY= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 h1:XOPfar83RIRPEzfihnp+U6udOveKZJvPQ76SKWrLRHc= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.2/go.mod h1:Vv9Xyk1KMHXrR3vNQe8W5LMFdTjSeWk0gBZBzvf3Qa0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 h1:pi0Skl6mNl2w8qWZXcdOyg197Zsf4G97U7Sso9JXGZE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2/go.mod h1:JYzLoEVeLXk+L4tn1+rrkfhkxl6mLDEVaDSvGq9og90= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.4 h1:Ppup1nVNAOWbBOrcoOxaxPeEnSFB2RnnQdguhXpmeQk= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.4/go.mod h1:+K1rNPVyGxkRuv9NNiaZ4YhBFuyw2MMA9SlIJ1Zlpz8= +github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw= +github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -189,8 +189,6 @@ github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= -github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/coreos/go-semver v0.1.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= @@ -210,8 +208,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= -github.com/digitalocean/godo v1.107.0 h1:P72IbmGFQvKOvyjVLyT59bmHxilA4E5hWi40rF4zNQc= -github.com/digitalocean/godo v1.107.0/go.mod h1:R6EmmWI8CT1+fCtjWY9UCB+L5uufuZH13wk3YhxycCs= +github.com/digitalocean/godo v1.110.0 h1:EY+rewWCYrUNOPbk9wI2Ytf0TBSRTJcZ6BINCb5dfmQ= +github.com/digitalocean/godo v1.110.0/go.mod h1:R6EmmWI8CT1+fCtjWY9UCB+L5uufuZH13wk3YhxycCs= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= @@ -235,8 +233,6 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= -github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -320,8 +316,9 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= +github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/goccy/go-json v0.9.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -365,8 +362,8 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -420,18 +417,18 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= -github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUhuHF+DA= +github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gophercloud/gophercloud v1.8.0 h1:TM3Jawprb2NrdOnvcHhWJalmKmAmOGgfZElM/3oBYCk= -github.com/gophercloud/gophercloud v1.8.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= +github.com/gophercloud/gophercloud v1.11.0 h1:ls0O747DIq1D8SUHc7r2vI8BFbMLeLFuENaAIfEx7OM= +github.com/gophercloud/gophercloud v1.11.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -452,8 +449,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb h1:tsEKRC3PU9rMw18w/uAptoijhgG4EvlA5kfJPtwrMDk= github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= -github.com/hetznercloud/hcloud-go v1.39.0 h1:RUlzI458nGnPR6dlcZlrsGXYC1hQlFbKdm8tVtEQQB0= -github.com/hetznercloud/hcloud-go v1.39.0/go.mod h1:mepQwR6va27S3UQthaEPGS86jtzSY9xWL1e9dyxXpgA= +github.com/hetznercloud/hcloud-go v1.53.0 h1:xThhlJc6MdpvDAqVB7bAw+nAQuCpQMwsf3yanCis4rM= +github.com/hetznercloud/hcloud-go v1.53.0/go.mod h1:VzDWThl47lOnZXY0q5/LPFD+M62pfe/52TV+mOrpp9Q= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= @@ -467,6 +464,8 @@ github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= +github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -526,8 +525,8 @@ github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++ github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= github.com/lestrrat-go/jwx v1.2.20/go.mod h1:tLE1XszaFgd7zaS5wHe4NxA+XVhu7xgdRvDpNyi3kNM= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/linode/linodego v1.26.0 h1:2tOZ3Wxn4YvGBRgZi3Vz6dab+L16XUntJ9sJxh3ZBio= -github.com/linode/linodego v1.26.0/go.mod h1:kD7Bf1piWg/AXb9TA0ThAVwzR+GPf6r2PvbTbVk7PMA= +github.com/linode/linodego v1.30.0 h1:6HJli+LX7NGu+Sne2G+ux790EkVOWOV/SR4mK3jcs6k= +github.com/linode/linodego v1.30.0/go.mod h1:/46h/XpmWi//oSA92GX2p3FIxb8HbX7grslPPQalR2o= github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -544,8 +543,6 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= @@ -586,16 +583,16 @@ github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042 github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.14.0 h1:vSmGj2Z5YPb9JwCWT6z6ihcUvDhuXLc3sJiqd3jMKAY= -github.com/onsi/ginkgo/v2 v2.14.0/go.mod h1:JkUdW7JkN0V6rFvsHcJ478egV3XH9NxpD27Hal/PhZw= +github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM= +github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= -github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= +github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f h1:3BMVfQpz1xe8MmJprp1+NL8hrpl9I04JVP9EczdCOqE= @@ -629,8 +626,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= -github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -641,8 +638,8 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.31.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -666,8 +663,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.22 h1:wJrcTdddKOI8TFxs8cemnhKP2EmKy3yfUKHj3ZdfzYo= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.22/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25 h1:/8rfZAdFfafRXOgz+ZpMZZWZ5pYggCY9t7e/BvjaBHM= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI= @@ -708,8 +705,8 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinkerbell/lint-install v0.0.0-20211012174934-5ee5ab01db76/go.mod h1:0h2KsALaQLNkoVeV+G+HjBWWCnp0COFYhJdRd5WCQPM= github.com/tinkerbell/tink v0.8.0 h1:qgl/rglpO5Rvq6UKZd29O6X9mDgZZYgf841+Y0IYWak= @@ -730,14 +727,14 @@ github.com/vektah/gqlparser/v2 v2.2.0 h1:bAc3slekAAJW6sZTi07aGq0OrfaCjj4jxARAaC7 github.com/vektah/gqlparser/v2 v2.2.0/go.mod h1:i3mQIGIrbK2PD1RrCeMTlVbkF2FJ6WkU1KJlJlC+3F4= github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= -github.com/vmware/go-vcloud-director/v2 v2.21.0 h1:zIONrJpM+Fj+rDyXmsRfMAn1sP5WAP87USL0T9GS4DY= -github.com/vmware/go-vcloud-director/v2 v2.21.0/go.mod h1:QPxGFgrUcSyzy9IlpwDE4UNT3tsOy2047tJOPEJ4nlw= -github.com/vmware/govmomi v0.34.2 h1:o6ydkTVITOkpQU6HAf6tP5GvHFCNJlNUNlMsvFK77X4= -github.com/vmware/govmomi v0.34.2/go.mod h1:qWWT6n9mdCr/T9vySsoUqcI04sSEj4CqHXxtk/Y+Los= +github.com/vmware/go-vcloud-director/v2 v2.22.0 h1:i1yFCoQZl/mTKViWLpT8mC9tlOAbupip703K0q1gQT0= +github.com/vmware/go-vcloud-director/v2 v2.22.0/go.mod h1:QPxGFgrUcSyzy9IlpwDE4UNT3tsOy2047tJOPEJ4nlw= +github.com/vmware/govmomi v0.36.1 h1:+E/nlfteQ8JvC0xhuKAfpnMsuIeGeGj7rJwqENUcWm8= +github.com/vmware/govmomi v0.36.1/go.mod h1:mtGWtM+YhTADHlCgJBiskSRPOZRsN9MSjPzaZLte/oQ= github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= -github.com/vultr/govultr/v3 v3.6.0 h1:WCXQwgdiZnGxG4CI+TTohE14V3jV6ikg/64fhDVdbIs= -github.com/vultr/govultr/v3 v3.6.0/go.mod h1:rt9v2x114jZmmLAE/h5N5jnxTmsK9ewwS2oQZ0UBQzM= +github.com/vultr/govultr/v3 v3.6.4 h1:unvY9eXlBw667ECQZDbBDOIaWB8wkk6Bx+yB0IMKXJ4= +github.com/vultr/govultr/v3 v3.6.4/go.mod h1:rt9v2x114jZmmLAE/h5N5jnxTmsK9ewwS2oQZ0UBQzM= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= @@ -750,8 +747,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.anx.io/go-anxcloud v0.6.2 h1:4FV9xtjilRny/TEBeqsqOPxm1i9UPlPjHRDK86fhFjc= -go.anx.io/go-anxcloud v0.6.2/go.mod h1:TW0KcKa1hlYEwCQ2YAFec07xtfX60psI/dmjJqRdmjY= +go.anx.io/go-anxcloud v0.6.4 h1:SaFqYHFZC96PNt0cp7bX+4khAWg1u1hUdSt11R++fn8= +go.anx.io/go-anxcloud v0.6.4/go.mod h1:aattNBzzaDFtPRU/eTsNK1lDdTFa8QUXal+w1SQPCF0= go.keploy.io/server v0.1.8 h1:b50vAt1+WKMscYVP5Bm8gx/iSaR7mpHox8VpaxjrQ88= go.keploy.io/server v0.1.8/go.mod h1:ZqhwTZOBb+dzx5t30Wt6eUGI6kO5QizvPg6coNPtbow= go.mongodb.org/mongo-driver v1.8.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= @@ -766,20 +763,20 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= -go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= -go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= -go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= -go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= -go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= -go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= +go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= -go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= -go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -800,8 +797,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go4.org v0.0.0-20160314031811-03efcb870d84/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= @@ -826,8 +823,8 @@ golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -928,8 +925,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -943,8 +940,8 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= -golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1035,8 +1032,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -1044,8 +1041,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1134,8 +1131,8 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= -golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1167,8 +1164,8 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.155.0 h1:vBmGhCYs0djJttDNynWo44zosHlPvHmA0XiN2zP2DtA= -google.golang.org/api v0.155.0/go.mod h1:GI5qK5f40kCpHfPn6+YzGAByIKWv8ujFnmoWm7Igduk= +google.golang.org/api v0.170.0 h1:zMaruDePM88zxZBG+NG8+reALO2rfLhe/JShitLyT48= +google.golang.org/api v0.170.0/go.mod h1:/xql9M2btF85xac/VAm4PsLMTLVGUOpq4BE9R8jyNy8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1222,12 +1219,12 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917 h1:nz5NESFLZbJGPFxDT/HCn+V1mZ8JGNoY4nUpmW/Y2eg= -google.golang.org/genproto v0.0.0-20240102182953-50ed04b92917/go.mod h1:pZqR+glSb11aJ+JQcczCvgf47+duRuzNSKqE8YAQnV0= -google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 h1:rcS6EyEaoCO52hQDupoSfrxI3R6C2Tq741is7X8OvnM= -google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917/go.mod h1:CmlNWB9lSezaYELKS5Ym1r44VrrbPUa7JTvw+6MbpJ0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 h1:6G8oQ016D88m1xAKljMlBOOGWDZkes4kMhgGFlf8WcQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917/go.mod h1:xtjpI3tXFPP051KaWnhvxkiubL/6dJ18vLVf7q2pTOU= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= +google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 h1:9IZDv+/GcI6u+a4jRFRLxQs0RUCfavGfoOgEW6jpkI0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1250,8 +1247,8 @@ google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= -google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/grpc/examples v0.0.0-20210728214646-ad0a2a847cdf/go.mod h1:bF8wuZSAZTcbF7ZPKrDI/qY52toTP/yxLpRRY4Eu9Js= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1314,23 +1311,23 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= -k8s.io/api v0.29.1 h1:DAjwWX/9YT7NQD4INu49ROJuZAAAP/Ijki48GUPzxqw= -k8s.io/api v0.29.1/go.mod h1:7Kl10vBRUXhnQQI8YR/R327zXC8eJ7887/+Ybta+RoQ= -k8s.io/apiextensions-apiserver v0.29.1 h1:S9xOtyk9M3Sk1tIpQMu9wXHm5O2MX6Y1kIpPMimZBZw= -k8s.io/apiextensions-apiserver v0.29.1/go.mod h1:zZECpujY5yTW58co8V2EQR4BD6A9pktVgHhvc0uLfeU= +k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= +k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= +k8s.io/apiextensions-apiserver v0.29.3 h1:9HF+EtZaVpFjStakF4yVufnXGPRppWFEQ87qnO91YeI= +k8s.io/apiextensions-apiserver v0.29.3/go.mod h1:po0XiY5scnpJfFizNGo6puNU6Fq6D70UJY2Cb2KwAVc= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apimachinery v0.29.1 h1:KY4/E6km/wLBguvCZv8cKTeOwwOBqFNjwJIdMkMbbRc= -k8s.io/apimachinery v0.29.1/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= +k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= +k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= -k8s.io/client-go v0.29.1 h1:19B/+2NGEwnFLzt0uB5kNJnfTsbV8w6TgQRz9l7ti7A= -k8s.io/client-go v0.29.1/go.mod h1:TDG/psL9hdet0TI9mGyHJSgRkW3H9JZk2dNEUS7bRks= -k8s.io/cloud-provider v0.29.1 h1:bDLpOSpysWrtU2PCkvyP2sUTwRBa6MGCmxt68CRRW/8= -k8s.io/cloud-provider v0.29.1/go.mod h1:u50Drm6AbuoKpsVbAstNiFHGgbSVHuJV4TWN5imdM2w= +k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= +k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= +k8s.io/cloud-provider v0.29.3 h1:y39hNq0lrPD1qmqQ2ykwMJGeWF9LsepVkR2a4wskwLc= +k8s.io/cloud-provider v0.29.3/go.mod h1:daDV1WkAO6pTrdsn7v8TpN/q9n75ExUC4RJDl7vlPKk= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.29.1 h1:MUimqJPCRnnHsskTTjKD+IC1EHBbRCVyi37IoFBrkYw= -k8s.io/component-base v0.29.1/go.mod h1:fP9GFjxYrLERq1GcWWZAE3bqbNcDKDytn2srWuHTtKc= +k8s.io/component-base v0.29.3 h1:Oq9/nddUxlnrCuuR2K/jp6aflVvc0uDvxMzAWxnGzAo= +k8s.io/component-base v0.29.3/go.mod h1:Yuj33XXjuOk2BAaHsIGHhCKZQAgYKhqIxIjIr2UXYio= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= @@ -1348,17 +1345,17 @@ k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lV k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910 h1:1Rp/XEKP5uxPs6QrsngEHAxBjaAR78iJRiJq5Fi7LSU= k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910/go.mod h1:Pa1PvrP7ACSkuX6I7KYomY6cmMA0Tx86waBhDUgoKPw= -k8s.io/kubelet v0.29.1 h1:cso8Dk8dymkj8q+EvW/aCbIYU2aOkH27gho48tYza/8= -k8s.io/kubelet v0.29.1/go.mod h1:hTl/naFcCVG1Ku17fMgj/krbheBwBkf3gnFhaboMx7E= +k8s.io/kubelet v0.29.3 h1:X9h0ZHzc+eUeNTaksbN0ItHyvGhQ7Z0HPjnQD2oHdwU= +k8s.io/kubelet v0.29.3/go.mod h1:jDiGuTkFOUynyBKzOoC1xRSWlgAZ9UPcTYeFyjr6vas= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= -k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -kubevirt.io/api v1.1.1 h1:vt5bOpACArNFIudx1bcE1VeejQdh5wCd7Oz/uFBIkH8= -kubevirt.io/api v1.1.1/go.mod h1:CJ4vZsaWhVN3jNbyc9y3lIZhw8nUHbWjap0xHABQiqc= -kubevirt.io/containerized-data-importer-api v1.58.0 h1:l6bH2SrCUi14QAi1Mv1vzcrqZI0XYzrV1KLK6hiC0QI= -kubevirt.io/containerized-data-importer-api v1.58.0/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= +k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0gQBEuevE/AaBsHY= +k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +kubevirt.io/api v1.2.0 h1:1f8XQLPl4BuHPsc6SHTPnYSYeDxucKCQGa8CdrGJSRc= +kubevirt.io/api v1.2.0/go.mod h1:SbeR9ma4EwnaOZEUkh/lNz0kzYm5LPpEDE30vKXC5Zg= +kubevirt.io/containerized-data-importer-api v1.58.1 h1:Zbf0pCvxb4fBvtMR6uI2OIJZ4UfwFxripzOLMO4HPbI= +kubevirt.io/containerized-data-importer-api v1.58.1/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 h1:fZYvD3/Vnitfkx6IJxjLAk8ugnZQ7CXVYcRfkSKmuZY= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= @@ -1366,8 +1363,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= -sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= +sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeGOUvw0= +sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/pkg/cloudprovider/provider/anexia/helper_test.go b/pkg/cloudprovider/provider/anexia/helper_test.go index c4d41e02a..ff5a1f2be 100644 --- a/pkg/cloudprovider/provider/anexia/helper_test.go +++ b/pkg/cloudprovider/provider/anexia/helper_test.go @@ -128,7 +128,7 @@ func hookableReconcileContext(locationID string, templateID string, hook func(*r }, }, ProviderData: &cloudprovidertypes.ProviderData{ - Update: func(m *clusterv1alpha1.Machine, mods ...cloudprovidertypes.MachineModifier) error { + Update: func(*clusterv1alpha1.Machine, ...cloudprovidertypes.MachineModifier) error { return nil }, }, diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index c861e1eef..ea1017797 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -474,9 +474,6 @@ func (p *provider) Get(ctx context.Context, log *zap.SugaredLogger, machine *clu vsphereAPI := vsphere.NewAPI(cli) status := getProviderStatus(log, machine) - if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, "failed to get machine status: %v", err) - } if status.InstanceID == "" && status.ProvisioningID == "" { return nil, cloudprovidererrors.ErrInstanceNotFound diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index 376610dff..bae7a23c1 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -146,7 +146,7 @@ func TestAnexiaProvider(t *testing.T) { }, } - testhelper.Mux.HandleFunc("/api/ipam/v1/address/reserve/ip/count.json", func(writer http.ResponseWriter, request *http.Request) { + testhelper.Mux.HandleFunc("/api/ipam/v1/address/reserve/ip/count.json", func(writer http.ResponseWriter, _ *http.Request) { err := json.NewEncoder(writer).Encode(address.ReserveRandomSummary{ Data: []address.ReservedIP{ { @@ -242,7 +242,7 @@ func TestAnexiaProvider(t *testing.T) { provider := New(nil).(*provider) for _, testCase := range testCases { - templateID, err := resolveTemplateID(context.TODO(), a, testCase.config, provider.configVarResolver, "foo") + templateID, err := resolveTemplateID(context.Background(), a, testCase.config, provider.configVarResolver, "foo") if testCase.expectedError != "" { if err != nil { testhelper.AssertErr(t, err) @@ -410,7 +410,7 @@ func TestUpdateStatus(t *testing.T) { machine.Status.ProviderStatus = &runtime.RawExtension{Raw: providerStatusJSON} called := false - err = updateMachineStatus(machine, providerStatus, func(paramMachine *v1alpha1.Machine, modifier ...cloudprovidertypes.MachineModifier) error { + err = updateMachineStatus(machine, providerStatus, func(paramMachine *v1alpha1.Machine, _ ...cloudprovidertypes.MachineModifier) error { called = true testhelper.AssertEquals(t, machine, paramMachine) status := getProviderStatus(zap.NewNop().Sugar(), machine) @@ -423,13 +423,13 @@ func TestUpdateStatus(t *testing.T) { } func Test_anexiaErrorToTerminalError(t *testing.T) { - forbiddenMockHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + forbiddenMockHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusForbidden) _, err := w.Write([]byte(`{"error": {"code": 403}}`)) testhelper.AssertNoErr(t, err) }) - unauthorizedMockHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + unauthorizedMockHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusUnauthorized) _, err := w.Write([]byte(`{"error": {"code": 401}}`)) testhelper.AssertNoErr(t, err) @@ -438,7 +438,7 @@ func Test_anexiaErrorToTerminalError(t *testing.T) { legacyClientRun := func(url string) error { client, err := client.New(client.BaseURL(url), client.IgnoreMissingToken(), client.ParseEngineErrors(true)) testhelper.AssertNoErr(t, err) - _, err = core.NewAPI(client).Location().List(context.TODO(), 1, 1, "", "") + _, err = core.NewAPI(client).Location().List(context.Background(), 1, 1, "", "") return err } @@ -448,7 +448,7 @@ func Test_anexiaErrorToTerminalError(t *testing.T) { client.IgnoreMissingToken(), )) testhelper.AssertNoErr(t, err) - return client.Get(context.TODO(), &corev1.Location{Identifier: "foo"}) + return client.Get(context.Background(), &corev1.Location{Identifier: "foo"}) } testCases := []struct { diff --git a/pkg/cloudprovider/provider/azure/create_delete_resources.go b/pkg/cloudprovider/provider/azure/create_delete_resources.go index 242fd765f..f98c76304 100644 --- a/pkg/cloudprovider/provider/azure/create_delete_resources.go +++ b/pkg/cloudprovider/provider/azure/create_delete_resources.go @@ -20,8 +20,8 @@ import ( "context" "fmt" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/latest/compute/mgmt/compute" + "github.com/Azure/azure-sdk-for-go/profiles/latest/network/mgmt/network" "github.com/Azure/go-autorest/autorest/azure/auth" "github.com/Azure/go-autorest/autorest/to" "go.uber.org/zap" @@ -343,7 +343,7 @@ func createOrUpdateNetworkInterface(ctx context.Context, log *zap.SugaredLogger, Name: to.StringPtr("ip-config-1"), InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ Subnet: &subnet, - PrivateIPAllocationMethod: network.IPAllocationMethodDynamic, + PrivateIPAllocationMethod: network.Dynamic, PublicIPAddress: publicIP, Primary: to.BoolPtr(true), }, @@ -353,11 +353,11 @@ func createOrUpdateNetworkInterface(ctx context.Context, log *zap.SugaredLogger, *ifSpec.InterfacePropertiesFormat.IPConfigurations = append(*ifSpec.InterfacePropertiesFormat.IPConfigurations, network.InterfaceIPConfiguration{ Name: to.StringPtr("ip-config-2"), InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ - PrivateIPAllocationMethod: network.IPAllocationMethodDynamic, + PrivateIPAllocationMethod: network.Dynamic, Subnet: &subnet, PublicIPAddress: publicIPv6, Primary: to.BoolPtr(false), - PrivateIPAddressVersion: network.IPVersionIPv6, + PrivateIPAddressVersion: network.IPv6, }, }) } diff --git a/pkg/cloudprovider/provider/azure/get_client.go b/pkg/cloudprovider/provider/azure/get_client.go index a4ee34021..c79a0eeba 100644 --- a/pkg/cloudprovider/provider/azure/get_client.go +++ b/pkg/cloudprovider/provider/azure/get_client.go @@ -19,8 +19,8 @@ package azure import ( "fmt" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/latest/compute/mgmt/compute" + "github.com/Azure/azure-sdk-for-go/profiles/latest/network/mgmt/network" "github.com/Azure/go-autorest/autorest/azure/auth" ) diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index b6a90b70a..9d857c074 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -27,8 +27,8 @@ import ( "unicode" "unicode/utf8" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" + "github.com/Azure/azure-sdk-for-go/profiles/latest/compute/mgmt/compute" + "github.com/Azure/azure-sdk-for-go/profiles/latest/network/mgmt/network" "github.com/Azure/go-autorest/autorest/to" gocache "github.com/patrickmn/go-cache" "go.uber.org/zap" @@ -618,13 +618,13 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * }); err != nil { return nil, err } - publicIP, err = createOrUpdatePublicIPAddress(ctx, log, publicIPName(ifaceName(machine)), network.IPVersionIPv4, sku, network.IPAllocationMethodStatic, machine.UID, config) + publicIP, err = createOrUpdatePublicIPAddress(ctx, log, publicIPName(ifaceName(machine)), network.IPv4, sku, network.Static, machine.UID, config) if err != nil { return nil, fmt.Errorf("failed to create public IP: %w", err) } if ipFamily.IsDualstack() { - publicIPv6, err = createOrUpdatePublicIPAddress(ctx, log, publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodStatic, machine.UID, config) + publicIPv6, err = createOrUpdatePublicIPAddress(ctx, log, publicIPv6Name(ifaceName(machine)), network.IPv6, sku, network.Static, machine.UID, config) if err != nil { return nil, fmt.Errorf("failed to create public IP: %w", err) } @@ -1143,14 +1143,14 @@ func (p *provider) MigrateUID(ctx context.Context, log *zap.SugaredLogger, machi if kuberneteshelper.HasFinalizer(machine, finalizerPublicIPv6) { sku = network.PublicIPAddressSkuNameStandard - _, err = createOrUpdatePublicIPAddress(ctx, log, publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodDynamic, newUID, config) + _, err = createOrUpdatePublicIPAddress(ctx, log, publicIPv6Name(ifaceName(machine)), network.IPv6, sku, network.Dynamic, newUID, config) if err != nil { return fmt.Errorf("failed to update UID on public IP: %w", err) } } if kuberneteshelper.HasFinalizer(machine, finalizerPublicIP) { - _, err = createOrUpdatePublicIPAddress(ctx, log, publicIPName(ifaceName(machine)), network.IPVersionIPv4, sku, network.IPAllocationMethodStatic, newUID, config) + _, err = createOrUpdatePublicIPAddress(ctx, log, publicIPName(ifaceName(machine)), network.IPv4, sku, network.Static, newUID, config) if err != nil { return fmt.Errorf("failed to update UID on public IP: %w", err) } diff --git a/pkg/cloudprovider/provider/gce/service.go b/pkg/cloudprovider/provider/gce/service.go index 4c1dd9570..fb15961c7 100644 --- a/pkg/cloudprovider/provider/gce/service.go +++ b/pkg/cloudprovider/provider/gce/service.go @@ -150,7 +150,7 @@ func (svc *service) waitOperation(ctx context.Context, refreshOperation func() ( var op *compute.Operation var err error - return wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, false, func(ctx context.Context) (bool, error) { + return wait.PollUntilContextTimeout(ctx, pollInterval, pollTimeout, false, func(_ context.Context) (bool, error) { op, err = refreshOperation() if err != nil { return false, err diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index a35d899fe..9e3a7d1aa 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -228,6 +228,7 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus } if c.Image != "" { + //nolint:staticcheck // We do not have the architecture available here. if _, _, err = client.Image.Get(ctx, c.Image); err != nil { return fmt.Errorf("failed to get image: %w", err) } @@ -350,6 +351,7 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * serverCreateOpts.Firewalls = append(serverCreateOpts.Firewalls, &hcloud.ServerCreateFirewall{Firewall: *n}) } + //nolint:staticcheck // We do not have the architecture available here. image, _, err := client.Image.Get(ctx, c.Image) if err != nil { return nil, hzErrorToTerminalError(err, "failed to get image") diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index a5b24ef8f..78f381b0d 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -690,7 +690,7 @@ func waitForPort(ctx context.Context, instanceLog *zap.SugaredLogger, netClient started := time.Now() instanceLog.Info("Waiting for the port to become active...") - portIsReady := func(c context.Context) (bool, error) { + portIsReady := func(context.Context) (bool, error) { port, err := getInstancePort(netClient, serverID, networkID) if err != nil { tErr := osErrorToTerminalError(instanceLog, err, fmt.Sprintf("failed to get current instance port %s", serverID)) diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index ad650c433..8222076f1 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -273,7 +273,7 @@ func TestCreateServer(t *testing.T) { // Note that configVarResolver is not used in this test as the getConfigFunc is mocked. configVarResolver: providerconfig.NewConfigVarResolver(context.Background(), fakectrlruntimeclient.NewClientBuilder().Build()), // mock client config getter - clientGetter: func(c *Config) (*gophercloud.ProviderClient, error) { + clientGetter: func(*Config) (*gophercloud.ProviderClient, error) { pc := client.ServiceClient() // endpoint locator used to redirect to local test endpoint pc.ProviderClient.EndpointLocator = func(_ gophercloud.EndpointOpts) (string, error) { diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go index 419e4e817..bd11b5010 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go @@ -89,11 +89,10 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * if sizingPolicy == nil { return fmt.Errorf("sizing policy '%s' doesn't exist", *c.SizingPolicy) } - if computePolicy == nil { - computePolicy = &types.ComputePolicy{} - } - computePolicy.VmSizingPolicy = &vcdapitypes.Reference{ - HREF: sizingPolicy.VdcComputePolicy.ID, + computePolicy = &types.ComputePolicy{ + VmSizingPolicy: &vcdapitypes.Reference{ + HREF: sizingPolicy.VdcComputePolicy.ID, + }, } } @@ -121,9 +120,7 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * } } if storageProfile == nil { - if err != nil { - return fmt.Errorf("failed to get storage profile '%s': %w", *c.StorageProfile, err) - } + return fmt.Errorf("failed to get storage profile '%s'", *c.StorageProfile) } } @@ -207,7 +204,9 @@ func recomposeComputeAndDisk(config *Config, vm *govcd.VM) (*govcd.VM, error) { needsDiskRecomposition = true } if config.DiskIOPS != nil && *config.DiskIOPS > 0 { - vmSpecSection.DiskSection.DiskSettings[i].Iops = ptr.To(*config.DiskIOPS) + vmSpecSection.DiskSection.DiskSettings[i].IopsAllocation = &vcdapitypes.IopsResource{ + Reservation: *config.DiskIOPS, + } needsDiskRecomposition = true } if config.DiskBusType != nil && *config.DiskBusType != "" { diff --git a/pkg/cloudprovider/provider/vsphere/client.go b/pkg/cloudprovider/provider/vsphere/client.go index 4a2e688eb..e1e45b7e9 100644 --- a/pkg/cloudprovider/provider/vsphere/client.go +++ b/pkg/cloudprovider/provider/vsphere/client.go @@ -50,9 +50,6 @@ func NewSession(ctx context.Context, config *Config) (*Session, error) { if err != nil { return nil, err } - if err != nil { - return nil, err - } client := &govmomi.Client{ Client: vim25Client, diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go index 522e77ff9..a700f744e 100644 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ b/pkg/cloudprovider/provider/vsphere/helper.go @@ -100,7 +100,7 @@ func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, return nil, fmt.Errorf("failed to clone template vm: %w", err) } - if err := clonedVMTask.Wait(ctx); err != nil { + if err := clonedVMTask.WaitEx(ctx); err != nil { return nil, fmt.Errorf("error when waiting for result of clone task: %w", err) } @@ -211,7 +211,7 @@ func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, if err != nil { return nil, fmt.Errorf("failed to reconfigure the VM: %w", err) } - if err := reconfigureTask.Wait(ctx); err != nil { + if err := reconfigureTask.WaitEx(ctx); err != nil { return nil, fmt.Errorf("error when waiting for result of the reconfigure task: %w", err) } diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 7a13cd43f..c3452bde0 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -409,7 +409,7 @@ func (p *provider) create(ctx context.Context, log *zap.SugaredLogger, machine * if vmErr != nil { return nil, fmt.Errorf("failed to destroy vm %s after failing upload and attach userdata iso: %w / %w", virtualMachine.Name(), err, vmErr) } - if vmErr := destroyTask.Wait(ctx); vmErr != nil { + if vmErr := destroyTask.WaitEx(ctx); vmErr != nil { return nil, fmt.Errorf("failed to destroy vm %s after failing upload and attach userdata iso: %w / %w", virtualMachine.Name(), err, vmErr) } return nil, machineInvalidConfigurationTerminalError(fmt.Errorf("failed to upload and attach userdata iso: %w", err)) @@ -421,7 +421,7 @@ func (p *provider) create(ctx context.Context, log *zap.SugaredLogger, machine * return nil, fmt.Errorf("failed to power on machine: %w", err) } - if err := powerOnTask.Wait(ctx); err != nil { + if err := powerOnTask.WaitEx(ctx); err != nil { return nil, fmt.Errorf("error when waiting for vm powerOn task: %w", err) } @@ -464,7 +464,7 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine if err != nil { return false, fmt.Errorf("failed to poweroff vm %s: %w", virtualMachine.Name(), err) } - if err = powerOffTask.Wait(ctx); err != nil { + if err = powerOffTask.WaitEx(ctx); err != nil { return false, fmt.Errorf("failed to poweroff vm %s: %w", virtualMachine.Name(), err) } } @@ -503,7 +503,7 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine if err != nil { return false, fmt.Errorf("failed to destroy vm %s: %w", virtualMachine.Name(), err) } - if err := destroyTask.Wait(ctx); err != nil { + if err := destroyTask.WaitEx(ctx); err != nil { return false, fmt.Errorf("failed to destroy vm %s: %w", virtualMachine.Name(), err) } @@ -557,7 +557,7 @@ func (p *provider) Get(ctx context.Context, log *zap.SugaredLogger, machine *clu if err != nil { return nil, fmt.Errorf("failed to power on instance that was in state %q: %w", powerState, err) } - if err := powerOnTask.Wait(ctx); err != nil { + if err := powerOnTask.WaitEx(ctx); err != nil { return nil, fmt.Errorf("failed waiting for instance to be powered on: %w", err) } // We must return here because the vendored code for determining if the guest diff --git a/pkg/cloudprovider/provider/vsphere/rule.go b/pkg/cloudprovider/provider/vsphere/rule.go index e4dcf2e8b..8df7f3811 100644 --- a/pkg/cloudprovider/provider/vsphere/rule.go +++ b/pkg/cloudprovider/provider/vsphere/rule.go @@ -100,7 +100,7 @@ func (p *provider) createOrUpdateVMAntiAffinityRule(ctx context.Context, session return err } - err = task.Wait(ctx) + err = task.WaitEx(ctx) if err != nil { return err } @@ -172,7 +172,7 @@ func removeVMAntiAffinityRule(ctx context.Context, session *Session, clusterPath if err != nil { return err } - return task.Wait(ctx) + return task.WaitEx(ctx) } func findClusterAntiAffinityRuleByName(ctx context.Context, cluster *object.ClusterComputeResource, name string) (*types.ClusterAntiAffinityRuleSpec, error) { diff --git a/pkg/clusterinfo/configmap.go b/pkg/clusterinfo/configmap.go index 27bb5bef7..116e39416 100644 --- a/pkg/clusterinfo/configmap.go +++ b/pkg/clusterinfo/configmap.go @@ -97,15 +97,6 @@ func (p *KubeconfigProvider) buildKubeconfigFromEndpoint(ctx context.Context) (* return nil, errors.New("could not parse ip from ") } - getSecurePort := func(endpointSubset corev1.EndpointSubset) *corev1.EndpointPort { - for _, p := range subset.Ports { - if p.Name == securePortName { - return &p - } - } - return nil - } - port := getSecurePort(subset) if port == nil { return nil, errors.New("no secure port in the subset") @@ -129,6 +120,15 @@ func (p *KubeconfigProvider) buildKubeconfigFromEndpoint(ctx context.Context) (* }, nil } +func getSecurePort(endpointSubset corev1.EndpointSubset) *corev1.EndpointPort { + for _, p := range endpointSubset.Ports { + if p.Name == securePortName { + return &p + } + } + return nil +} + func getCAData(config *rest.Config) ([]byte, error) { if len(config.TLSClientConfig.CAData) > 0 { return config.TLSClientConfig.CAData, nil diff --git a/pkg/controller/machine/controller.go b/pkg/controller/machine/controller.go index 0a0b1d301..0bf4b2013 100644 --- a/pkg/controller/machine/controller.go +++ b/pkg/controller/machine/controller.go @@ -226,7 +226,7 @@ func Add( c, err := controller.New(ControllerName, mgr, controller.Options{ Reconciler: reconciler, MaxConcurrentReconciles: numWorkers, - LogConstructor: func(request *reconcile.Request) logr.Logger { + LogConstructor: func(*reconcile.Request) logr.Logger { // we log ourselves return zapr.NewLogger(zap.NewNop()) }, diff --git a/pkg/controller/machinedeployment/controller.go b/pkg/controller/machinedeployment/controller.go index 745fa1e76..f37fee3fb 100644 --- a/pkg/controller/machinedeployment/controller.go +++ b/pkg/controller/machinedeployment/controller.go @@ -80,7 +80,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) err // Create a new controller. c, err := controller.New(controllerName, mgr, controller.Options{ Reconciler: r, - LogConstructor: func(request *reconcile.Request) logr.Logger { + LogConstructor: func(*reconcile.Request) logr.Logger { // we log ourselves return zapr.NewLogger(zap.NewNop()) }, diff --git a/pkg/controller/machinedeployment/sync.go b/pkg/controller/machinedeployment/sync.go index 9313d950a..e9e9f071a 100644 --- a/pkg/controller/machinedeployment/sync.go +++ b/pkg/controller/machinedeployment/sync.go @@ -115,8 +115,8 @@ func (r *ReconcileMachineDeployment) getNewMachineSet(ctx context.Context, log * } // Apply revision annotation from existingNewMS if it is missing from the deployment. - err := r.updateMachineDeployment(ctx, d, func(innerDeployment *clusterv1alpha1.MachineDeployment) { - dutil.SetDeploymentRevision(d, msCopy.Annotations[dutil.RevisionAnnotation]) + err := r.updateMachineDeployment(ctx, d, func(md *clusterv1alpha1.MachineDeployment) { + dutil.SetDeploymentRevision(md, msCopy.Annotations[dutil.RevisionAnnotation]) }) return msCopy, err } @@ -209,8 +209,8 @@ func (r *ReconcileMachineDeployment) getNewMachineSet(ctx context.Context, log * log.Debugw("Created new MachineSet", "machineset", client.ObjectKeyFromObject(createdMS)) } - err = r.updateMachineDeployment(ctx, d, func(innerDeployment *clusterv1alpha1.MachineDeployment) { - dutil.SetDeploymentRevision(d, newRevision) + err = r.updateMachineDeployment(ctx, d, func(md *clusterv1alpha1.MachineDeployment) { + dutil.SetDeploymentRevision(md, newRevision) }) return createdMS, err diff --git a/pkg/controller/machineset/controller.go b/pkg/controller/machineset/controller.go index 6301c3819..8d6ca3f10 100644 --- a/pkg/controller/machineset/controller.go +++ b/pkg/controller/machineset/controller.go @@ -83,7 +83,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) err // Create a new controller. c, err := controller.New(controllerName, mgr, controller.Options{ Reconciler: r, - LogConstructor: func(request *reconcile.Request) logr.Logger { + LogConstructor: func(*reconcile.Request) logr.Logger { // we log ourselves return zapr.NewLogger(zap.NewNop()) }, diff --git a/pkg/controller/nodecsrapprover/controller.go b/pkg/controller/nodecsrapprover/controller.go index e5ceda524..3f9178063 100644 --- a/pkg/controller/nodecsrapprover/controller.go +++ b/pkg/controller/nodecsrapprover/controller.go @@ -85,7 +85,7 @@ func Add(mgr manager.Manager, log *zap.SugaredLogger) error { cntrl, err := controller.New(ControllerName, mgr, controller.Options{ Reconciler: rec, - LogConstructor: func(request *reconcile.Request) logr.Logger { + LogConstructor: func(*reconcile.Request) logr.Logger { // we log ourselves return zapr.NewLogger(zap.NewNop()) }, diff --git a/pkg/rhsm/util.go b/pkg/rhsm/util.go index 7fbf8f781..ed45078dc 100644 --- a/pkg/rhsm/util.go +++ b/pkg/rhsm/util.go @@ -30,7 +30,7 @@ const ( func AddRHELSubscriptionFinalizer(machine *v1alpha1.Machine, update types.MachineUpdater) error { if !kuberneteshelper.HasFinalizer(machine, RedhatSubscriptionFinalizer) { if err := update(machine, func(m *v1alpha1.Machine) { - machine.Finalizers = append(m.Finalizers, RedhatSubscriptionFinalizer) + m.Finalizers = append(m.Finalizers, RedhatSubscriptionFinalizer) }); err != nil { return err } @@ -43,7 +43,7 @@ func AddRHELSubscriptionFinalizer(machine *v1alpha1.Machine, update types.Machin func RemoveRHELSubscriptionFinalizer(machine *v1alpha1.Machine, update types.MachineUpdater) error { if kuberneteshelper.HasFinalizer(machine, RedhatSubscriptionFinalizer) { if err := update(machine, func(m *v1alpha1.Machine) { - machine.Finalizers = kuberneteshelper.RemoveFinalizer(machine.Finalizers, RedhatSubscriptionFinalizer) + m.Finalizers = kuberneteshelper.RemoveFinalizer(m.Finalizers, RedhatSubscriptionFinalizer) }); err != nil { return err } diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 95018f7b4..b949a7e53 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -99,8 +99,11 @@ func TestInvalidObjectsGetRejected(t *testing.T) { {osName: "flatcar", executor: verifyCreateMachineFails}, } + ctx := context.Background() + for i, test := range tests { - testScenario(t, + testScenario(ctx, + t, test, fmt.Sprintf("invalid-machine-%v", i), nil, @@ -139,6 +142,7 @@ func TestCustomCAsAreApplied(t *testing.T) { } testScenario( + context.Background(), t, scenario{ name: "ca-test", @@ -146,12 +150,12 @@ func TestCustomCAsAreApplied(t *testing.T) { kubernetesVersion: versions[0].String(), osName: string(providerconfigtypes.OperatingSystemUbuntu), - executor: func(kubeConfig, manifestPath string, parameters []string, d time.Duration) error { - if err := updateMachineControllerForCustomCA(kubeConfig); err != nil { + executor: func(ctx context.Context, kubeConfig, manifestPath string, parameters []string, d time.Duration) error { + if err := updateMachineControllerForCustomCA(ctx, kubeConfig); err != nil { return fmt.Errorf("failed to add CA: %w", err) } - return verifyCreateMachineFails(kubeConfig, manifestPath, parameters, d) + return verifyCreateMachineFails(ctx, kubeConfig, manifestPath, parameters, d) }, }, "dummy-machine", @@ -161,7 +165,7 @@ func TestCustomCAsAreApplied(t *testing.T) { ) } -func updateMachineControllerForCustomCA(kubeconfig string) error { +func updateMachineControllerForCustomCA(ctx context.Context, kubeconfig string) error { cfg, err := clientcmd.BuildConfigFromFlags("", kubeconfig) if err != nil { return fmt.Errorf("Error building kubeconfig: %w", err) @@ -172,7 +176,6 @@ func updateMachineControllerForCustomCA(kubeconfig string) error { return fmt.Errorf("failed to create Client: %w", err) } - ctx := context.Background() ns := metav1.NamespaceSystem // create intentionally valid but useless CA bundle @@ -304,7 +307,7 @@ func TestKubevirtProvisioningE2E(t *testing.T) { fmt.Sprintf("<< KUBECONFIG_BASE64 >>=%s", safeBase64Encoding(kubevirtKubeconfig)), } - runScenarios(t, selector, params, kubevirtManifest, fmt.Sprintf("kubevirt-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, kubevirtManifest, fmt.Sprintf("kubevirt-%s", *testRunIdentifier)) } // safeBase64Encoding takes a value and encodes it with base64 @@ -345,7 +348,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { // In-tree cloud provider is not supported from Kubernetes v1.26. selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.27.11", "1.28.7", "1.29.2"))) - runScenarios(t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } func TestOpenstackProjectAuthProvisioningE2E(t *testing.T) { @@ -382,7 +385,7 @@ func TestOpenstackProjectAuthProvisioningE2E(t *testing.T) { kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } - testScenario(t, scenario, *testRunIdentifier, params, OSManifestProjectAuth, false) + testScenario(context.Background(), t, scenario, *testRunIdentifier, params, OSManifestProjectAuth, false) } // TestDigitalOceanProvisioning - a test suite that exercises digital ocean provider @@ -402,7 +405,7 @@ func TestDigitalOceanProvisioningE2E(t *testing.T) { // act params := []string{fmt.Sprintf("<< DIGITALOCEAN_TOKEN >>=%s", doToken)} - runScenarios(t, selector, params, DOManifest, fmt.Sprintf("do-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, DOManifest, fmt.Sprintf("do-%s", *testRunIdentifier)) } // TestAWSProvisioning - a test suite that exercises AWS provider @@ -432,7 +435,7 @@ func TestAWSProvisioningE2E(t *testing.T) { fmt.Sprintf("<< PROVISIONING_UTILITY >>=%s", provisioningUtility), } - runScenarios(t, selector, params, AWSManifest, fmt.Sprintf("aws-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, AWSManifest, fmt.Sprintf("aws-%s", *testRunIdentifier)) } // TestAWSAssumeRoleProvisioning - a test suite that exercises AWS provider @@ -462,7 +465,7 @@ func TestAWSAssumeRoleProvisioningE2E(t *testing.T) { kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } - testScenario(t, scenario, *testRunIdentifier, params, AWSManifest, false) + testScenario(context.Background(), t, scenario, *testRunIdentifier, params, AWSManifest, false) } // TestAWSSpotInstanceProvisioning - a test suite that exercises AWS provider @@ -485,7 +488,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { fmt.Sprintf("<< AWS_SECRET_ACCESS_KEY >>=%s", awsSecret), fmt.Sprintf("<< PROVISIONING_UTILITY >>=%s", flatcar.Ignition), } - runScenarios(t, selector, params, AWSSpotInstanceManifest, fmt.Sprintf("aws-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, AWSSpotInstanceManifest, fmt.Sprintf("aws-%s", *testRunIdentifier)) } // TestAWSARMProvisioningE2E - a test suite that exercises AWS provider for arm machines @@ -507,7 +510,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { fmt.Sprintf("<< AWS_SECRET_ACCESS_KEY >>=%s", awsSecret), fmt.Sprintf("<< PROVISIONING_UTILITY >>=%s", flatcar.Ignition), } - runScenarios(t, selector, params, AWSManifestARM, fmt.Sprintf("aws-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, AWSManifestARM, fmt.Sprintf("aws-%s", *testRunIdentifier)) } func TestAWSFlatcarCoreOSCloudInit8ProvisioningE2E(t *testing.T) { @@ -528,7 +531,7 @@ func TestAWSFlatcarCoreOSCloudInit8ProvisioningE2E(t *testing.T) { // We would like to test flatcar with CoreOS-cloud-init selector := OsSelector("flatcar") - runScenarios(t, selector, params, AWSManifest, fmt.Sprintf("aws-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, AWSManifest, fmt.Sprintf("aws-%s", *testRunIdentifier)) } func TestAWSCentOS8ProvisioningE2E(t *testing.T) { @@ -551,7 +554,7 @@ func TestAWSCentOS8ProvisioningE2E(t *testing.T) { // We would like to test CentOS8 image only in this test as the other images are tested in TestAWSProvisioningE2E selector := OsSelector("centos") - runScenarios(t, selector, params, AWSManifest, fmt.Sprintf("aws-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, AWSManifest, fmt.Sprintf("aws-%s", *testRunIdentifier)) } // TestAWSEbsEncryptionEnabledProvisioningE2E - a test suite that exercises AWS provider with ebs encryption enabled @@ -578,7 +581,7 @@ func TestAWSEbsEncryptionEnabledProvisioningE2E(t *testing.T) { kubernetesVersion: awsDefaultKubernetesVersion, executor: verifyCreateAndDelete, } - testScenario(t, scenario, fmt.Sprintf("aws-%s", *testRunIdentifier), params, AWSEBSEncryptedManifest, false) + testScenario(context.Background(), t, scenario, fmt.Sprintf("aws-%s", *testRunIdentifier), params, AWSEBSEncryptedManifest, false) } // TestAzureProvisioningE2E - a test suite that exercises Azure provider @@ -606,7 +609,7 @@ func TestAzureProvisioningE2E(t *testing.T) { fmt.Sprintf("<< AZURE_OS_DISK_SKU >>=%s", "Standard_LRS"), fmt.Sprintf("<< AZURE_DATA_DISK_SKU >>=%s", "Standard_LRS"), } - runScenarios(t, selector, params, AzureManifest, fmt.Sprintf("azure-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, AzureManifest, fmt.Sprintf("azure-%s", *testRunIdentifier)) } // TestAzureCustomImageReferenceProvisioningE2E - a test suite that exercises Azure provider @@ -633,7 +636,7 @@ func TestAzureCustomImageReferenceProvisioningE2E(t *testing.T) { fmt.Sprintf("<< AZURE_OS_DISK_SKU >>=%s", "Standard_LRS"), fmt.Sprintf("<< AZURE_DATA_DISK_SKU >>=%s", "Standard_LRS"), } - runScenarios(t, selector, params, AzureCustomImageReferenceManifest, fmt.Sprintf("azure-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, AzureCustomImageReferenceManifest, fmt.Sprintf("azure-%s", *testRunIdentifier)) } // TestAzureRedhatSatelliteProvisioningE2E - a test suite that exercises Azure provider @@ -669,7 +672,7 @@ func TestAzureRedhatSatelliteProvisioningE2E(t *testing.T) { executor: verifyCreateAndDelete, } - testScenario(t, scenario, *testRunIdentifier, params, AzureRedhatSatelliteManifest, false) + testScenario(context.Background(), t, scenario, *testRunIdentifier, params, AzureRedhatSatelliteManifest, false) } // TestGCEProvisioningE2E - a test suite that exercises Google Cloud provider @@ -690,7 +693,7 @@ func TestGCEProvisioningE2E(t *testing.T) { fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>=%s", safeBase64Encoding(googleServiceAccount)), } - runScenarios(t, selector, params, GCEManifest, fmt.Sprintf("gce-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, GCEManifest, fmt.Sprintf("gce-%s", *testRunIdentifier)) } // TestHetznerProvisioning - a test suite that exercises Hetzner provider @@ -708,7 +711,7 @@ func TestHetznerProvisioningE2E(t *testing.T) { // act params := []string{fmt.Sprintf("<< HETZNER_TOKEN >>=%s", hzToken)} - runScenarios(t, selector, params, HZManifest, fmt.Sprintf("hz-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, HZManifest, fmt.Sprintf("hz-%s", *testRunIdentifier)) } // TestEquinixMetalProvisioningE2E - a test suite that exercises Equinix Metal provider @@ -734,7 +737,7 @@ func TestEquinixMetalProvisioningE2E(t *testing.T) { fmt.Sprintf("<< METAL_AUTH_TOKEN >>=%s", token), fmt.Sprintf("<< METAL_PROJECT_ID >>=%s", projectID), } - runScenarios(t, selector, params, EquinixMetalManifest, fmt.Sprintf("equinixmetal-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, EquinixMetalManifest, fmt.Sprintf("equinixmetal-%s", *testRunIdentifier)) } func TestAlibabaProvisioningE2E(t *testing.T) { @@ -758,7 +761,7 @@ func TestAlibabaProvisioningE2E(t *testing.T) { fmt.Sprintf("<< ALIBABA_ACCESS_KEY_ID >>=%s", accessKeyID), fmt.Sprintf("<< ALIBABA_ACCESS_KEY_SECRET >>=%s", accessKeySecret), } - runScenarios(t, selector, params, alibabaManifest, fmt.Sprintf("alibaba-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, alibabaManifest, fmt.Sprintf("alibaba-%s", *testRunIdentifier)) } // TestLinodeProvisioning - a test suite that exercises Linode provider @@ -779,7 +782,7 @@ func TestLinodeProvisioningE2E(t *testing.T) { // act params := []string{fmt.Sprintf("<< LINODE_TOKEN >>=%s", linodeToken)} - runScenarios(t, selector, params, LinodeManifest, fmt.Sprintf("linode-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, LinodeManifest, fmt.Sprintf("linode-%s", *testRunIdentifier)) } func getVMwareCloudDirectorTestParams(t *testing.T) []string { @@ -811,7 +814,7 @@ func TestVMwareCloudDirectorProvisioningE2E(t *testing.T) { selector := OsSelector("ubuntu") params := getVMwareCloudDirectorTestParams(t) - runScenarios(t, selector, params, VMwareCloudDirectorManifest, fmt.Sprintf("vcd-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, VMwareCloudDirectorManifest, fmt.Sprintf("vcd-%s", *testRunIdentifier)) } func getVSphereTestParams(t *testing.T) []string { @@ -841,7 +844,7 @@ func TestVsphereProvisioningE2E(t *testing.T) { selector := Not(OsSelector("amzn2", "centos")) params := getVSphereTestParams(t) - runScenarios(t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) } // TestVsphereMultipleNICProvisioning - is the same as the TestVsphereProvisioning suit but has multiple networks attached to the VMs. @@ -852,7 +855,7 @@ func TestVsphereMultipleNICProvisioningE2E(t *testing.T) { selector := OsSelector("ubuntu") params := getVSphereTestParams(t) - runScenarios(t, selector, params, VSPhereMultipleNICManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, VSPhereMultipleNICManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) } // TestVsphereDatastoreClusterProvisioning - is the same as the TestVsphereProvisioning suite but specifies a DatastoreCluster @@ -863,7 +866,7 @@ func TestVsphereDatastoreClusterProvisioningE2E(t *testing.T) { selector := OsSelector("ubuntu", "centos", "rhel", "flatcar") params := getVSphereTestParams(t) - runScenarios(t, selector, params, VSPhereDSCManifest, fmt.Sprintf("vs-dsc-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, VSPhereDSCManifest, fmt.Sprintf("vs-dsc-%s", *testRunIdentifier)) } // TestVsphereResourcePoolProvisioning - creates a machine deployment using a @@ -881,7 +884,7 @@ func TestVsphereResourcePoolProvisioningE2E(t *testing.T) { executor: verifyCreateAndDelete, } - testScenario(t, scenario, *testRunIdentifier, params, VSPhereResourcePoolManifest, false) + testScenario(context.Background(), t, scenario, *testRunIdentifier, params, VSPhereResourcePoolManifest, false) } // TestScalewayProvisioning - a test suite that exercises scaleway provider @@ -917,7 +920,7 @@ func TestScalewayProvisioningE2E(t *testing.T) { fmt.Sprintf("<< SCW_SECRET_KEY >>=%s", scwSecretKey), fmt.Sprintf("<< SCW_DEFAULT_PROJECT_ID >>=%s", scwProjectID), } - runScenarios(t, selector, params, ScalewayManifest, fmt.Sprintf("scw-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, ScalewayManifest, fmt.Sprintf("scw-%s", *testRunIdentifier)) } func getNutanixTestParams(t *testing.T) []string { @@ -955,7 +958,7 @@ func TestNutanixProvisioningE2E(t *testing.T) { // location, thus possibly blocking access a HTTP proxy if it is configured. selector := And(OsSelector("ubuntu", "centos"), Not(NameSelector("migrateUID"))) params := getNutanixTestParams(t) - runScenarios(t, selector, params, nutanixManifest, fmt.Sprintf("nx-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, nutanixManifest, fmt.Sprintf("nx-%s", *testRunIdentifier)) } func TestOpenNebulaProvisioningE2E(t *testing.T) { @@ -992,7 +995,7 @@ func TestOpenNebulaProvisioningE2E(t *testing.T) { } selector := OsSelector("rockylinux", "flatcar") - runScenarios(t, selector, params, openNebulaManifest, fmt.Sprintf("one-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, openNebulaManifest, fmt.Sprintf("one-%s", *testRunIdentifier)) } // TestUbuntuProvisioningWithUpgradeE2E will create an instance from an old Ubuntu 1604 @@ -1030,7 +1033,7 @@ func TestUbuntuProvisioningWithUpgradeE2E(t *testing.T) { executor: verifyCreateAndDelete, } - testScenario(t, scenario, *testRunIdentifier, params, OSUpgradeManifest, false) + testScenario(context.Background(), t, scenario, *testRunIdentifier, params, OSUpgradeManifest, false) } // TestDeploymentControllerUpgradesMachineE2E verifies the machineDeployment controller correctly @@ -1054,7 +1057,7 @@ func TestDeploymentControllerUpgradesMachineE2E(t *testing.T) { kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateUpdateAndDelete, } - testScenario(t, scenario, *testRunIdentifier, params, HZManifest, false) + testScenario(context.Background(), t, scenario, *testRunIdentifier, params, HZManifest, false) } func TestAnexiaProvisioningE2E(t *testing.T) { @@ -1077,7 +1080,7 @@ func TestAnexiaProvisioningE2E(t *testing.T) { fmt.Sprintf("<< ANEXIA_LOCATION_ID >>=%s", locationID), } - runScenarios(t, selector, params, anexiaManifest, fmt.Sprintf("anexia-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, anexiaManifest, fmt.Sprintf("anexia-%s", *testRunIdentifier)) } // TestVultrProvisioning - a test suite that exercises Vultr provider @@ -1095,5 +1098,5 @@ func TestVultrProvisioningE2E(t *testing.T) { // act params := []string{fmt.Sprintf("<< VULTR_API_KEY >>=%s", apiKey)} - runScenarios(t, selector, params, vultrManifest, fmt.Sprintf("vlt-%s", *testRunIdentifier)) + runScenarios(context.Background(), t, selector, params, vultrManifest, fmt.Sprintf("vlt-%s", *testRunIdentifier)) } diff --git a/test/e2e/provisioning/deploymentscenario.go b/test/e2e/provisioning/deploymentscenario.go index 3ec6e182b..12933296b 100644 --- a/test/e2e/provisioning/deploymentscenario.go +++ b/test/e2e/provisioning/deploymentscenario.go @@ -29,12 +29,11 @@ import ( "k8s.io/klog" ) -func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []string, timeout time.Duration) error { +func verifyCreateUpdateAndDelete(ctx context.Context, kubeConfig, manifestPath string, parameters []string, timeout time.Duration) error { client, machineDeployment, err := prepareMachineDeployment(kubeConfig, manifestPath, parameters) if err != nil { return err } - ctx := context.Background() // This test inherently relies on replicas being one so we enforce that machineDeployment.Spec.Replicas = getInt32Ptr(1) @@ -44,7 +43,7 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s return fmt.Errorf("failed to verify creation of node for MachineDeployment: %w", err) } - if err := updateMachineDeployment(machineDeployment, client, func(md *clusterv1alpha1.MachineDeployment) { + if err := updateMachineDeployment(ctx, machineDeployment, client, func(md *clusterv1alpha1.MachineDeployment) { md.Spec.Template.Labels["testUpdate"] = "true" }); err != nil { return fmt.Errorf("failed to update MachineDeployment %s after modifying it: %w", machineDeployment.Name, err) @@ -53,7 +52,7 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s klog.Infof("Waiting for second MachineSet to appear after updating MachineDeployment %s", machineDeployment.Name) var machineSets []clusterv1alpha1.MachineSet if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, false, func(ctx context.Context) (bool, error) { - machineSets, err = getMatchingMachineSets(machineDeployment, client) + machineSets, err = getMatchingMachineSets(ctx, machineDeployment, client) if err != nil { return false, err } @@ -82,7 +81,7 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s } var machines []clusterv1alpha1.Machine if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, false, func(ctx context.Context) (bool, error) { - machines, err = getMatchingMachinesForMachineset(&newestMachineSet, client) + machines, err = getMatchingMachinesForMachineset(ctx, &newestMachineSet, client) if err != nil { return false, err } @@ -113,7 +112,7 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s if *machineSet.Spec.Replicas != int32(0) { return false, nil } - machines, err := getMatchingMachinesForMachineset(machineSet, client) + machines, err := getMatchingMachinesForMachineset(ctx, machineSet, client) if err != nil { return false, err } @@ -124,7 +123,7 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s klog.Infof("Old MachineSet %s got scaled down and has no associated machines anymore", oldMachineSet.Name) klog.Infof("Setting replicas of MachineDeployment %s to 0 and waiting until it has no associated machines", machineDeployment.Name) - if err := updateMachineDeployment(machineDeployment, client, func(md *clusterv1alpha1.MachineDeployment) { + if err := updateMachineDeployment(ctx, machineDeployment, client, func(md *clusterv1alpha1.MachineDeployment) { md.Spec.Replicas = getInt32Ptr(0) }); err != nil { return fmt.Errorf("failed to update replicas of MachineDeployment %s: %w", machineDeployment.Name, err) @@ -133,7 +132,7 @@ func verifyCreateUpdateAndDelete(kubeConfig, manifestPath string, parameters []s klog.Infof("Waiting for MachineDeployment %s to not have any associated machines", machineDeployment.Name) if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, timeout, false, func(ctx context.Context) (bool, error) { - machines, err := getMatchingMachines(machineDeployment, client) + machines, err := getMatchingMachines(ctx, machineDeployment, client) return len(machines) == 0, err }); err != nil { return err diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index aff05972f..a0f2f0eb0 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -17,6 +17,7 @@ limitations under the License. package provisioning import ( + "context" "fmt" "os" "path/filepath" @@ -181,7 +182,7 @@ func (v *version) Match(testCase scenario) bool { return false } -func runScenarios(st *testing.T, selector Selector, testParams []string, manifestPath string, cloudProvider string) { +func runScenarios(ctx context.Context, st *testing.T, selector Selector, testParams []string, manifestPath string, cloudProvider string) { for _, testCase := range scenarios { if selector != nil && !selector.Match(testCase) { fmt.Printf("Skipping test %s\n", testCase.name) @@ -189,16 +190,16 @@ func runScenarios(st *testing.T, selector Selector, testParams []string, manifes } st.Run(testCase.name, func(it *testing.T) { - testScenario(it, testCase, cloudProvider, testParams, manifestPath, true) + testScenario(ctx, it, testCase, cloudProvider, testParams, manifestPath, true) }) } } // scenarioExecutor represents an executor for a given scenario // args: kubeConfig, maifestPath, scenarioParams, timeout -type scenarioExecutor func(string, string, []string, time.Duration) error +type scenarioExecutor func(context.Context, string, string, []string, time.Duration) error -func testScenario(t *testing.T, testCase scenario, cloudProvider string, testParams []string, manifestPath string, parallelize bool) { +func testScenario(ctx context.Context, t *testing.T, testCase scenario, cloudProvider string, testParams []string, manifestPath string, parallelize bool) { if parallelize { t.Parallel() } @@ -294,7 +295,7 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar // we decided to keep this time lower that the global timeout to prevent the following: // the global timeout is set to 20 minutes and the verify tool waits up to 60 hours for a machine to show up. // thus one faulty scenario prevents from showing the results for the whole group, which is confusing because it looks like all tests are broken. - if err := testCase.executor(kubeConfig, manifestPath, scenarioParams, 35*time.Minute); err != nil { + if err := testCase.executor(ctx, kubeConfig, manifestPath, scenarioParams, 35*time.Minute); err != nil { t.Errorf("verify failed due to error=%v", err) } } diff --git a/test/e2e/provisioning/migrateuidscenario.go b/test/e2e/provisioning/migrateuidscenario.go index 54beb379f..d846fa3e9 100644 --- a/test/e2e/provisioning/migrateuidscenario.go +++ b/test/e2e/provisioning/migrateuidscenario.go @@ -40,7 +40,7 @@ import ( fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) -func verifyMigrateUID(_, manifestPath string, parameters []string, _ time.Duration) error { +func verifyMigrateUID(ctx context.Context, _, manifestPath string, parameters []string, _ time.Duration) error { log := zap.NewNop().Sugar() // prepare the manifest @@ -72,7 +72,7 @@ func verifyMigrateUID(_, manifestPath string, parameters []string, _ time.Durati Build() providerData := &cloudprovidertypes.ProviderData{ - Update: cloudprovidertypes.GetMachineUpdater(context.Background(), fakeClient), + Update: cloudprovidertypes.GetMachineUpdater(ctx, fakeClient), Client: fakeClient, } @@ -80,7 +80,7 @@ func verifyMigrateUID(_, manifestPath string, parameters []string, _ time.Durati if err != nil { return fmt.Errorf("failed to get provideSpec: %w", err) } - skg := providerconfig.NewConfigVarResolver(context.Background(), fakeClient) + skg := providerconfig.NewConfigVarResolver(ctx, fakeClient) prov, err := cloudprovider.ForProvider(providerSpec.CloudProvider, skg) if err != nil { return fmt.Errorf("failed to get cloud provider %q: %w", providerSpec.CloudProvider, err) @@ -91,8 +91,6 @@ func verifyMigrateUID(_, manifestPath string, parameters []string, _ time.Durati } machine.Spec = defaultedSpec - ctx := context.Background() - // Step 0: Create instance with old UID maxTries := 15 for i := 0; i < maxTries; i++ { diff --git a/test/e2e/provisioning/verify.go b/test/e2e/provisioning/verify.go index 5fdd7813c..1c74cae08 100644 --- a/test/e2e/provisioning/verify.go +++ b/test/e2e/provisioning/verify.go @@ -44,31 +44,29 @@ const ( machineReadyCheckPeriod = 15 * time.Second ) -func verifyCreateMachineFails(kubeConfig, manifestPath string, parameters []string, _ time.Duration) error { +func verifyCreateMachineFails(ctx context.Context, kubeConfig, manifestPath string, parameters []string, _ time.Duration) error { client, machine, err := prepareMachine(kubeConfig, manifestPath, parameters) if err != nil { return err } - if err := client.Create(context.Background(), machine); err != nil { + if err := client.Create(ctx, machine); err != nil { return nil } return fmt.Errorf("expected create of Machine %s to fail but succeeded", machine.Name) } -func verifyCreateAndDelete(kubeConfig, manifestPath string, parameters []string, timeout time.Duration) error { +func verifyCreateAndDelete(ctx context.Context, kubeConfig, manifestPath string, parameters []string, timeout time.Duration) error { client, machineDeployment, err := prepareMachineDeployment(kubeConfig, manifestPath, parameters) if err != nil { return err } - ctx := context.Background() - machineDeployment, err = createAndAssure(ctx, machineDeployment, client, timeout) if err != nil { return fmt.Errorf("failed to verify creation of node for MachineDeployment: %w", err) } - if err := deleteAndAssure(machineDeployment, client, timeout); err != nil { + if err := deleteAndAssure(ctx, machineDeployment, client, timeout); err != nil { return fmt.Errorf("Failed to verify if a machine/node has been created/deleted, due to: \n%w", err) } @@ -142,7 +140,7 @@ func prepare(kubeConfig, manifestPath string, parameters []string) (ctrlruntimec func createAndAssure(ctx context.Context, machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Client, timeout time.Duration) (*clusterv1alpha1.MachineDeployment, error) { // we expect that no node for machine exists in the cluster - err := assureNodeForMachineDeployment(machineDeployment, client, false) + err := assureNodeForMachineDeployment(ctx, machineDeployment, client, false) if err != nil { return nil, fmt.Errorf("failed to perform the verification, incorrect cluster state detected %w", err) } @@ -170,7 +168,7 @@ func createAndAssure(ctx context.Context, machineDeployment *clusterv1alpha1.Mac var pollErr error err = wait.PollUntilContextTimeout(ctx, machineReadyCheckPeriod, timeout, false, func(ctx context.Context) (bool, error) { - pollErr = assureNodeForMachineDeployment(machineDeployment, client, true) + pollErr = assureNodeForMachineDeployment(ctx, machineDeployment, client, true) if pollErr == nil { return true, nil } @@ -183,7 +181,7 @@ func createAndAssure(ctx context.Context, machineDeployment *clusterv1alpha1.Mac klog.Infof("Waiting for node of MachineDeployment %s to become ready", machineDeployment.Name) err = wait.PollUntilContextTimeout(ctx, machineReadyCheckPeriod, timeout, false, func(ctx context.Context) (bool, error) { - machines, pollErr := getMatchingMachines(machineDeployment, client) + machines, pollErr := getMatchingMachines(ctx, machineDeployment, client) if pollErr != nil || len(machines) < 1 { return false, nil } @@ -230,21 +228,21 @@ func hasMachineReadyNode(ctx context.Context, machine *clusterv1alpha1.Machine, return false, nil } -func deleteAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Client, timeout time.Duration) error { +func deleteAndAssure(ctx context.Context, machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Client, timeout time.Duration) error { klog.Infof("Starting to clean up MachineDeployment %s", machineDeployment.Name) // We first scale down to 0, because once the machineSets are deleted we can not // match machines anymore and we do want to verify not only the node is gone but also // the instance at the cloud provider - if err := updateMachineDeployment(machineDeployment, client, func(md *clusterv1alpha1.MachineDeployment) { + if err := updateMachineDeployment(ctx, machineDeployment, client, func(md *clusterv1alpha1.MachineDeployment) { md.Spec.Replicas = getInt32Ptr(0) }); err != nil { return fmt.Errorf("failed to update replicas of MachineDeployment %s: %w", machineDeployment.Name, err) } // Ensure machines are gone - if err := wait.PollUntilContextTimeout(context.Background(), machineReadyCheckPeriod, timeout, false, func(ctx context.Context) (bool, error) { - ownedMachines, err := getMatchingMachines(machineDeployment, client) + if err := wait.PollUntilContextTimeout(ctx, machineReadyCheckPeriod, timeout, false, func(ctx context.Context) (bool, error) { + ownedMachines, err := getMatchingMachines(ctx, machineDeployment, client) if err != nil { return false, err } @@ -257,11 +255,11 @@ func deleteAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien } klog.V(2).Infof("Deleting MachineDeployment %s", machineDeployment.Name) - if err := client.Delete(context.Background(), machineDeployment); err != nil { + if err := client.Delete(ctx, machineDeployment); err != nil { return fmt.Errorf("failed to remove MachineDeployment %s, due to %w", machineDeployment.Name, err) } - return wait.PollUntilContextTimeout(context.Background(), machineReadyCheckPeriod, timeout, false, func(ctx context.Context) (bool, error) { - err := client.Get(context.Background(), types.NamespacedName{Namespace: machineDeployment.Namespace, Name: machineDeployment.Name}, &clusterv1alpha1.MachineDeployment{}) + return wait.PollUntilContextTimeout(ctx, machineReadyCheckPeriod, timeout, false, func(ctx context.Context) (bool, error) { + err := client.Get(ctx, types.NamespacedName{Namespace: machineDeployment.Namespace, Name: machineDeployment.Name}, &clusterv1alpha1.MachineDeployment{}) if kerrors.IsNotFound(err) { return true, nil } @@ -271,8 +269,8 @@ func deleteAndAssure(machineDeployment *clusterv1alpha1.MachineDeployment, clien // assureNodeForMachineDeployment according to shouldExists parameter check if a node for machine exists in the system or not // this method examines OwnerReference of each node. -func assureNodeForMachineDeployment(machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Client, shouldExist bool) error { - machines, err := getMatchingMachines(machineDeployment, client) +func assureNodeForMachineDeployment(ctx context.Context, machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Client, shouldExist bool) error { + machines, err := getMatchingMachines(ctx, machineDeployment, client) if err != nil { return fmt.Errorf("failed to list Machines: %w", err) } @@ -299,7 +297,7 @@ func assureNodeForMachineDeployment(machineDeployment *clusterv1alpha1.MachineDe } nodes := &corev1.NodeList{} - if err := client.List(context.Background(), nodes); err != nil { + if err := client.List(ctx, nodes); err != nil { return fmt.Errorf("failed to list Nodes: %w", err) } @@ -348,15 +346,15 @@ func readAndModifyManifest(pathToManifest string, keyValuePairs []string) (strin } // getMatchingMachines returns all machines that are owned by the passed machineDeployment. -func getMatchingMachines(machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Client) ([]clusterv1alpha1.Machine, error) { - matchingMachineSets, err := getMatchingMachineSets(machineDeployment, client) +func getMatchingMachines(ctx context.Context, machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Client) ([]clusterv1alpha1.Machine, error) { + matchingMachineSets, err := getMatchingMachineSets(ctx, machineDeployment, client) if err != nil { return nil, err } klog.V(2).Infof("Found %v matching MachineSets for %s", len(matchingMachineSets), machineDeployment.Name) var matchingMachines []clusterv1alpha1.Machine for _, machineSet := range matchingMachineSets { - machinesForMachineSet, err := getMatchingMachinesForMachineset(&machineSet, client) + machinesForMachineSet, err := getMatchingMachinesForMachineset(ctx, &machineSet, client) if err != nil { return nil, fmt.Errorf("failed to get matching Machines for MachineSet %s: %w", machineSet.Name, err) } @@ -366,9 +364,9 @@ func getMatchingMachines(machineDeployment *clusterv1alpha1.MachineDeployment, c return matchingMachines, nil } -func getMatchingMachinesForMachineset(machineSet *clusterv1alpha1.MachineSet, client ctrlruntimeclient.Client) ([]clusterv1alpha1.Machine, error) { +func getMatchingMachinesForMachineset(ctx context.Context, machineSet *clusterv1alpha1.MachineSet, client ctrlruntimeclient.Client) ([]clusterv1alpha1.Machine, error) { allMachines := &clusterv1alpha1.MachineList{} - if err := client.List(context.Background(), allMachines, &ctrlruntimeclient.ListOptions{Namespace: machineSet.Namespace}); err != nil { + if err := client.List(ctx, allMachines, &ctrlruntimeclient.ListOptions{Namespace: machineSet.Namespace}); err != nil { return nil, fmt.Errorf("failed to list Machines: %w", err) } var matchingMachines []clusterv1alpha1.Machine @@ -381,12 +379,12 @@ func getMatchingMachinesForMachineset(machineSet *clusterv1alpha1.MachineSet, cl } // getMatchingMachineSets returns all machineSets that are owned by the passed machineDeployment. -func getMatchingMachineSets(machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Reader) ([]clusterv1alpha1.MachineSet, error) { +func getMatchingMachineSets(ctx context.Context, machineDeployment *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Reader) ([]clusterv1alpha1.MachineSet, error) { // Ensure we actually have an object from the KubeAPI and not just the result of the yaml parsing, as the latter // can not be the owner of anything due to missing UID. if machineDeployment.ResourceVersion == "" { nn := types.NamespacedName{Namespace: machineDeployment.Namespace, Name: machineDeployment.Name} - if err := client.Get(context.Background(), nn, machineDeployment); err != nil { + if err := client.Get(ctx, nn, machineDeployment); err != nil { if !kerrors.IsNotFound(err) { return nil, fmt.Errorf("failed to get MachineDeployment %s: %w", nn.Name, err) } @@ -394,7 +392,7 @@ func getMatchingMachineSets(machineDeployment *clusterv1alpha1.MachineDeployment } } allMachineSets := &clusterv1alpha1.MachineSetList{} - if err := client.List(context.Background(), allMachineSets, &ctrlruntimeclient.ListOptions{Namespace: machineDeployment.Namespace}); err != nil { + if err := client.List(ctx, allMachineSets, &ctrlruntimeclient.ListOptions{Namespace: machineDeployment.Namespace}); err != nil { return nil, fmt.Errorf("failed to list MachineSets: %w", err) } var matchingMachineSets []clusterv1alpha1.MachineSet @@ -410,17 +408,17 @@ func getInt32Ptr(i int32) *int32 { return &i } -func updateMachineDeployment(md *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Client, modify func(*clusterv1alpha1.MachineDeployment)) error { +func updateMachineDeployment(ctx context.Context, md *clusterv1alpha1.MachineDeployment, client ctrlruntimeclient.Client, modify func(*clusterv1alpha1.MachineDeployment)) error { // Store Namespace and Name here because after an error md will be nil name := md.Name namespace := md.Namespace return retry.RetryOnConflict(retry.DefaultBackoff, func() error { md := &clusterv1alpha1.MachineDeployment{} - if err := client.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: name}, md); err != nil { + if err := client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, md); err != nil { return err } modify(md) - return client.Update(context.Background(), md) + return client.Update(ctx, md) }) } From 7e3d16b51279382213390ad164e2b500a7269c5c Mon Sep 17 00:00:00 2001 From: Akash Gautam Date: Wed, 10 Apr 2024 19:28:34 +0530 Subject: [PATCH 382/489] Update golang to 1.22.2 (#1776) * update golang to 1.22.2 Signed-off-by: Akash Gautam * fix test failures post golang 1.22.2 upgrade Signed-off-by: Akash Gautam --------- Signed-off-by: Akash Gautam --- .golangci.yml | 9 ++++----- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 16 ++++++++-------- .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 4 ++-- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 8 ++++---- .prow/verify.yaml | 18 +++++++++--------- Dockerfile | 2 +- Makefile | 2 +- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- 23 files changed, 52 insertions(+), 53 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 028e270b7..01ec59870 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,11 +1,7 @@ run: - deadline: 20m + timeout: 20m build-tags: - e2e - skip-dirs: - - pkg/client - - pkg/machines - linters: enable: - asciicheck @@ -69,3 +65,6 @@ issues: - 'cyclomatic complexity 31 of func `\(\*provider\)\.Validate` is high' - 'cyclomatic complexity 33 of func `\(\*provider\)\.Create` is high' - 'cyclomatic complexity 32 of func `\(\*Reconciler\)\.ensureInstanceExistsForMachine` is high' + exclude-dirs: + - pkg/client + - pkg/machines diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index a550bf6fe..68189470c 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -119,7 +119,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index dd380aa5a..f90e89d3f 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index 7e6e8d709..05dfe490c 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index defbcca80..58094e08d 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 3db39d632..9022cc691 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 env: - name: OPERATING_SYSTEM_MANAGER value: "false" @@ -95,7 +95,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -126,7 +126,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -159,7 +159,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -190,7 +190,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -221,7 +221,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -252,7 +252,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 1cb8c860b..c70fd17f4 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index bed60b3bd..d73e043f5 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 059766bcc..75bbbc3ee 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 5f3721c08..27862120d 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index b8655e8f5..a16858989 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index adb808ae5..26e1986ab 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 12a0711ed..c82c40a3b 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 31e53cf54..0c2398c08 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 22ff68180..4402359f5 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 0fa091d99..c7d567d0e 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 8be8ec733..e8cf27955 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 8fe107ed3..3cb5b6d1d 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 40447e5c3..e406805c5 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-3 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-3 command: - make args: @@ -63,7 +63,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-3 command: - make args: @@ -83,7 +83,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-3 command: - make args: @@ -102,9 +102,9 @@ presubmits: clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-3 command: - - "/bin/shfmt" + - "/usr/local/bin/shfmt" args: # -l list files whose formatting differs from shfmt's # -d error with a diff when the formatting differs @@ -130,7 +130,7 @@ presubmits: clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-3 command: - "./hack/verify-boilerplate.sh" resources: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-3 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-18-5 + - image: quay.io/kubermatic/build:go-1.22-node-20-3 command: - make args: diff --git a/Dockerfile b/Dockerfile index 68bd62d93..0ce3d2494 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.22.1 +ARG GO_VERSION=1.22.2 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index c60377ee6..88d409470 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.22.1 +GO_VERSION ?= 1.22.2 GOOS ?= $(shell go env GOOS) diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 5200ec615..c90dbad7e 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.22.1 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.22.2 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 5465f0aef..c90bd7108 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-18-kind-0.22-5 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 containerize ./hack/verify-licenses.sh go mod vendor From f339b66aae4f8d259a189fc088e3c49bce1cf6e7 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 18 Apr 2024 20:02:03 +0500 Subject: [PATCH 383/489] Support for Kubernetes 1.30 (#1777) * Update to k8s 1.30 APIs Signed-off-by: Waleed Malik * In-tree support has been dropped for Azure and vSphere Signed-off-by: Waleed Malik * Update fixtures Signed-off-by: Waleed Malik * Default version in machine deployment examples has been raised to v1.29.4 Signed-off-by: Waleed Malik * Update fixtures Signed-off-by: Waleed Malik * Disable AWS E2E tests Signed-off-by: Waleed Malik * Bump 1.28 to latest patch 1.28.9 for E2E Signed-off-by: Waleed Malik * Remove filter for GCE Signed-off-by: Waleed Malik * Fix logic to populate cloud-provider and cloud-config flags Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- .prow/provider-aws.yaml | 6 +- README.md | 1 + examples/alibaba-machinedeployment.yaml | 2 +- examples/anexia-machinedeployment.yaml | 2 +- examples/aws-machinedeployment.yaml | 2 +- examples/azure-machinedeployment.yaml | 2 +- examples/digitalocean-machinedeployment.yaml | 2 +- examples/equinixmetal-machinedeployment.yaml | 2 +- examples/gce-machinedeployment.yaml | 2 +- examples/hetzner-machinedeployment.yaml | 2 +- examples/kubevirt-machinedeployment.yaml | 2 +- examples/linode-machinedeployment.yaml | 2 +- examples/nutanix-machinedeployment.yaml | 2 +- examples/opennebula-machinedeployment.yaml | 2 +- examples/openstack-machinedeployment.yaml | 2 +- examples/scaleway-machinedeployment.yaml | 2 +- ...ware-cloud-director-machinedeployment.yaml | 2 +- ...e-datastore-cluster-machinedeployment.yaml | 2 +- examples/vsphere-machinedeployment.yaml | 2 +- examples/vultr-machinedeployment.yaml | 2 +- go.mod | 36 +- go.sum | 61 +- pkg/userdata/amzn2/provider_test.go | 22 +- .../amzn2/testdata/kubelet-v1.27-aws.yaml | 2 + .../amzn2/testdata/kubelet-v1.28-aws.yaml | 4 +- .../amzn2/testdata/kubelet-v1.29-aws.yaml | 4 +- ...yaml => kubelet-v1.30.0-aws-external.yaml} | 4 +- ...29.2-aws.yaml => kubelet-v1.30.0-aws.yaml} | 6 +- ...l => kubelet-v1.30.0-vsphere-mirrors.yaml} | 6 +- ...aml => kubelet-v1.30.0-vsphere-proxy.yaml} | 6 +- ...here.yaml => kubelet-v1.30.0-vsphere.yaml} | 6 +- pkg/userdata/centos/provider_test.go | 28 +- .../centos/testdata/kubelet-v1.27-aws.yaml | 2 + .../centos/testdata/kubelet-v1.28-aws.yaml | 4 +- .../centos/testdata/kubelet-v1.29-aws.yaml | 6 +- ...yaml => kubelet-v1.30.0-aws-external.yaml} | 4 +- ...29.2-aws.yaml => kubelet-v1.30.0-aws.yaml} | 6 +- ...anix.yaml => kubelet-v1.30.0-nutanix.yaml} | 6 +- ...l => kubelet-v1.30.0-vsphere-mirrors.yaml} | 6 +- ...aml => kubelet-v1.30.0-vsphere-proxy.yaml} | 6 +- ...here.yaml => kubelet-v1.30.0-vsphere.yaml} | 6 +- pkg/userdata/flatcar/provider_test.go | 30 +- .../flatcar/testdata/cloud-init_v1.28.0.yaml | 982 +++++++++--------- .../flatcar/testdata/cloud-init_v1.29.0.yaml | 4 +- .../flatcar/testdata/cloud-init_v1.29.2.yaml | 521 ---------- ...t_v1.28.5.yaml => cloud-init_v1.30.0.yaml} | 6 +- pkg/userdata/flatcar/testdata/containerd.yaml | 4 +- .../flatcar/testdata/ignition_v1.28.0.json | 232 +---- .../flatcar/testdata/ignition_v1.28.5.json | 1 - .../flatcar/testdata/ignition_v1.29.0.json | 2 +- .../flatcar/testdata/ignition_v1.29.2.json | 1 - .../flatcar/testdata/ignition_v1.30.0.json | 1 + pkg/userdata/helper/common_test.go | 7 +- .../helper/download_binaries_script_test.go | 4 +- pkg/userdata/helper/kubelet.go | 24 +- pkg/userdata/helper/kubelet_test.go | 8 +- pkg/userdata/helper/template_functions.go | 1 - ...olden => download_binaries_v1.27.0.golden} | 2 +- .../download_binaries_v1.27.11.golden | 17 - ...olden => download_binaries_v1.28.0.golden} | 2 +- .../testdata/download_binaries_v1.28.7.golden | 17 - ...olden => download_binaries_v1.30.0.golden} | 2 +- ...let_systemd_unit_cloud-provider-set.golden | 2 - ...kublet_systemd_unit_pause-image-set.golden | 2 - .../kublet_systemd_unit_taints-set.golden | 2 - ...temd_unit_version-v1.27.0-external.golden} | 0 ...ublet_systemd_unit_version-v1.27.0.golden} | 0 ...temd_unit_version-v1.28.0-external.golden} | 0 ...ublet_systemd_unit_version-v1.28.0.golden} | 0 ...stemd_unit_version-v1.28.7-external.golden | 36 - ...kublet_systemd_unit_version-v1.28.7.golden | 35 - ...stemd_unit_version-v1.29.2-external.golden | 36 - ...kublet_systemd_unit_version-v1.29.2.golden | 35 - ...temd_unit_version-v1.30.0-external.golden} | 0 ...ublet_systemd_unit_version-v1.30.0.golden} | 0 ... => safe_download_binaries_v1.30.0.golden} | 2 +- pkg/userdata/rhel/provider_test.go | 24 +- .../rhel/testdata/kubelet-v1.28-aws.yaml | 4 +- .../rhel/testdata/kubelet-v1.28-nutanix.yaml | 4 +- .../rhel/testdata/kubelet-v1.29-aws.yaml | 4 +- .../rhel/testdata/kubelet-v1.29-nutanix.yaml | 4 +- ...yaml => kubelet-v1.30.0-aws-external.yaml} | 4 +- ...29.2-aws.yaml => kubelet-v1.30.0-aws.yaml} | 6 +- ...l => kubelet-v1.30.0-vsphere-mirrors.yaml} | 6 +- ...aml => kubelet-v1.30.0-vsphere-proxy.yaml} | 6 +- ...here.yaml => kubelet-v1.30.0-vsphere.yaml} | 6 +- .../rhel/testdata/pod-cidr-azure-rhel.yaml | 4 +- pkg/userdata/rockylinux/provider_test.go | 26 +- .../testdata/kubelet-v1.28-aws.yaml | 4 +- .../testdata/kubelet-v1.29.2-aws.yaml | 467 --------- ...yaml => kubelet-v1.30.0-aws-external.yaml} | 4 +- ...1.29-aws.yaml => kubelet-v1.30.0-aws.yaml} | 6 +- ...anix.yaml => kubelet-v1.30.0-nutanix.yaml} | 6 +- ...l => kubelet-v1.30.0-vsphere-mirrors.yaml} | 6 +- ...aml => kubelet-v1.30.0-vsphere-proxy.yaml} | 6 +- ...here.yaml => kubelet-v1.30.0-vsphere.yaml} | 6 +- pkg/userdata/ubuntu/provider_test.go | 9 +- pkg/userdata/ubuntu/testdata/containerd.yaml | 4 +- .../digitalocean-dualstack-IPv6+IPv4.yaml | 4 +- .../testdata/digitalocean-dualstack.yaml | 4 +- .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 4 +- pkg/userdata/ubuntu/testdata/docker.yaml | 4 +- .../kubelet-version-without-v-prefix.yaml | 2 + .../ubuntu/testdata/multiple-dns-servers.yaml | 4 +- .../ubuntu/testdata/multiple-ssh-keys.yaml | 2 + pkg/userdata/ubuntu/testdata/nutanix.yaml | 4 +- .../openstack-dualstack-IPv6+IPv4.yaml | 4 +- .../ubuntu/testdata/openstack-dualstack.yaml | 4 +- .../openstack-overwrite-cloud-config.yaml | 2 + pkg/userdata/ubuntu/testdata/openstack.yaml | 6 +- ...ersion-1.27.9.yaml => version-1.27.0.yaml} | 4 +- ...ersion-1.28.5.yaml => version-1.28.0.yaml} | 4 +- .../ubuntu/testdata/version-1.28.7.yaml | 459 -------- .../ubuntu/testdata/version-1.29.0.yaml | 2 + .../ubuntu/testdata/version-1.29.2.yaml | 459 -------- ...rsion-1.27.11.yaml => version-1.30.0.yaml} | 4 +- .../ubuntu/testdata/vsphere-mirrors.yaml | 2 + .../ubuntu/testdata/vsphere-proxy.yaml | 2 + pkg/userdata/ubuntu/testdata/vsphere.yaml | 2 + test/e2e/provisioning/all_e2e_test.go | 27 +- test/e2e/provisioning/helper.go | 7 +- ...hinedeployment-azure-redhat-satellite.yaml | 2 +- 122 files changed, 840 insertions(+), 3086 deletions(-) rename pkg/userdata/amzn2/testdata/{kubelet-v1.29.2-aws-external.yaml => kubelet-v1.30.0-aws-external.yaml} (99%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.29.2-aws.yaml => kubelet-v1.30.0-aws.yaml} (99%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.29.2-vsphere-mirrors.yaml => kubelet-v1.30.0-vsphere-mirrors.yaml} (99%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.29.2-vsphere-proxy.yaml => kubelet-v1.30.0-vsphere-proxy.yaml} (99%) rename pkg/userdata/amzn2/testdata/{kubelet-v1.29.2-vsphere.yaml => kubelet-v1.30.0-vsphere.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.29.2-aws-external.yaml => kubelet-v1.30.0-aws-external.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.29.2-aws.yaml => kubelet-v1.30.0-aws.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.29.2-nutanix.yaml => kubelet-v1.30.0-nutanix.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.29.2-vsphere-mirrors.yaml => kubelet-v1.30.0-vsphere-mirrors.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.29.2-vsphere-proxy.yaml => kubelet-v1.30.0-vsphere-proxy.yaml} (99%) rename pkg/userdata/centos/testdata/{kubelet-v1.29.2-vsphere.yaml => kubelet-v1.30.0-vsphere.yaml} (99%) delete mode 100644 pkg/userdata/flatcar/testdata/cloud-init_v1.29.2.yaml rename pkg/userdata/flatcar/testdata/{cloud-init_v1.28.5.yaml => cloud-init_v1.30.0.yaml} (99%) delete mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.28.5.json delete mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.29.2.json create mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.30.0.json rename pkg/userdata/helper/testdata/{download_binaries_v1.29.2.golden => download_binaries_v1.27.0.golden} (91%) delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.27.11.golden rename pkg/userdata/helper/testdata/{download_binaries_v1.27.9.golden => download_binaries_v1.28.0.golden} (91%) delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.28.7.golden rename pkg/userdata/helper/testdata/{download_binaries_v1.28.5.golden => download_binaries_v1.30.0.golden} (91%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.27.11-external.golden => kublet_systemd_unit_version-v1.27.0-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.27.11.golden => kublet_systemd_unit_version-v1.27.0.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.27.9-external.golden => kublet_systemd_unit_version-v1.28.0-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.27.9.golden => kublet_systemd_unit_version-v1.28.0.golden} (100%) delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.7-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.7.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.2-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.2.golden rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.28.5-external.golden => kublet_systemd_unit_version-v1.30.0-external.golden} (100%) rename pkg/userdata/helper/testdata/{kublet_systemd_unit_version-v1.28.5.golden => kublet_systemd_unit_version-v1.30.0.golden} (100%) rename pkg/userdata/helper/testdata/{safe_download_binaries_v1.29.2.golden => safe_download_binaries_v1.30.0.golden} (98%) rename pkg/userdata/rhel/testdata/{kubelet-v1.29.2-aws-external.yaml => kubelet-v1.30.0-aws-external.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.29.2-aws.yaml => kubelet-v1.30.0-aws.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.29.2-vsphere-mirrors.yaml => kubelet-v1.30.0-vsphere-mirrors.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.29.2-vsphere-proxy.yaml => kubelet-v1.30.0-vsphere-proxy.yaml} (99%) rename pkg/userdata/rhel/testdata/{kubelet-v1.29.2-vsphere.yaml => kubelet-v1.30.0-vsphere.yaml} (99%) delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-aws.yaml rename pkg/userdata/rockylinux/testdata/{kubelet-v1.29.2-aws-external.yaml => kubelet-v1.30.0-aws-external.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.29-aws.yaml => kubelet-v1.30.0-aws.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.29.2-nutanix.yaml => kubelet-v1.30.0-nutanix.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.29.2-vsphere-mirrors.yaml => kubelet-v1.30.0-vsphere-mirrors.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.29.2-vsphere-proxy.yaml => kubelet-v1.30.0-vsphere-proxy.yaml} (99%) rename pkg/userdata/rockylinux/testdata/{kubelet-v1.29.2-vsphere.yaml => kubelet-v1.30.0-vsphere.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.27.9.yaml => version-1.27.0.yaml} (99%) rename pkg/userdata/ubuntu/testdata/{version-1.28.5.yaml => version-1.28.0.yaml} (99%) delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.28.7.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.29.2.yaml rename pkg/userdata/ubuntu/testdata/{version-1.27.11.yaml => version-1.30.0.yaml} (99%) diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 9022cc691..df65767a6 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -82,7 +82,8 @@ presubmits: memory: 7Gi - name: pull-machine-controller-e2e-aws-arm - run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" + # In-tree CCM is not supported for AWS starting from k8s 1.27. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. + # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -113,7 +114,8 @@ presubmits: memory: 7Gi - name: pull-machine-controller-e2e-aws-ebs-encryption-enabled - run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" + # In-tree CCM is not supported for AWS starting from k8s 1.27. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. + # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: diff --git a/README.md b/README.md index f0e41d67f..dc87a214b 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ machine-controller tries to follow the Kubernetes version Currently supported K8S versions are: +- 1.30 - 1.29 - 1.28 - 1.27 diff --git a/examples/alibaba-machinedeployment.yaml b/examples/alibaba-machinedeployment.yaml index 154b6196c..c34377b9b 100644 --- a/examples/alibaba-machinedeployment.yaml +++ b/examples/alibaba-machinedeployment.yaml @@ -62,4 +62,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index 7e9745903..3affb6ab9 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -54,4 +54,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/aws-machinedeployment.yaml b/examples/aws-machinedeployment.yaml index b82c2b505..76443a95a 100644 --- a/examples/aws-machinedeployment.yaml +++ b/examples/aws-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/azure-machinedeployment.yaml b/examples/azure-machinedeployment.yaml index 755a1e5a7..ddfb72bbe 100644 --- a/examples/azure-machinedeployment.yaml +++ b/examples/azure-machinedeployment.yaml @@ -91,4 +91,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/digitalocean-machinedeployment.yaml b/examples/digitalocean-machinedeployment.yaml index 804c8c3b4..02a2810a4 100644 --- a/examples/digitalocean-machinedeployment.yaml +++ b/examples/digitalocean-machinedeployment.yaml @@ -57,4 +57,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/equinixmetal-machinedeployment.yaml b/examples/equinixmetal-machinedeployment.yaml index dbce45cce..94ec553ad 100644 --- a/examples/equinixmetal-machinedeployment.yaml +++ b/examples/equinixmetal-machinedeployment.yaml @@ -51,4 +51,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index 7c598af34..aa220f166 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -90,4 +90,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/hetzner-machinedeployment.yaml b/examples/hetzner-machinedeployment.yaml index 1d3adb228..3384e5dbb 100644 --- a/examples/hetzner-machinedeployment.yaml +++ b/examples/hetzner-machinedeployment.yaml @@ -67,4 +67,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index c96d47350..a58fabb2b 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -69,4 +69,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/linode-machinedeployment.yaml b/examples/linode-machinedeployment.yaml index f00928ac4..3757be954 100644 --- a/examples/linode-machinedeployment.yaml +++ b/examples/linode-machinedeployment.yaml @@ -53,4 +53,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/nutanix-machinedeployment.yaml b/examples/nutanix-machinedeployment.yaml index bb7e863a6..bc20126c1 100644 --- a/examples/nutanix-machinedeployment.yaml +++ b/examples/nutanix-machinedeployment.yaml @@ -82,4 +82,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/opennebula-machinedeployment.yaml b/examples/opennebula-machinedeployment.yaml index ab4309da7..b706270ce 100644 --- a/examples/opennebula-machinedeployment.yaml +++ b/examples/opennebula-machinedeployment.yaml @@ -67,4 +67,4 @@ spec: # use cloud-init for flatcar as ignition doesn't know anything about OpenNebula yet provisioningUtility: "cloud-init" versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index ddfa3c030..1de28e02c 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -166,4 +166,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/scaleway-machinedeployment.yaml b/examples/scaleway-machinedeployment.yaml index b996f3f0c..7858f9763 100644 --- a/examples/scaleway-machinedeployment.yaml +++ b/examples/scaleway-machinedeployment.yaml @@ -61,4 +61,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/vmware-cloud-director-machinedeployment.yaml b/examples/vmware-cloud-director-machinedeployment.yaml index 40d076a8e..ae7983ecd 100644 --- a/examples/vmware-cloud-director-machinedeployment.yaml +++ b/examples/vmware-cloud-director-machinedeployment.yaml @@ -89,4 +89,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml index 1bae0b162..f2e7df996 100644 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ b/examples/vsphere-datastore-cluster-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index 4e1bb6cef..0387105cf 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/examples/vultr-machinedeployment.yaml b/examples/vultr-machinedeployment.yaml index 03e2b7f07..6b6635a44 100644 --- a/examples/vultr-machinedeployment.yaml +++ b/examples/vultr-machinedeployment.yaml @@ -70,4 +70,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.28.5 + kubelet: 1.29.4 diff --git a/go.mod b/go.mod index f14ee922f..972fbec4c 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/kubermatic/machine-controller -go 1.21 +go 1.22.0 -toolchain go1.21.5 +toolchain go1.22.2 require ( cloud.google.com/go/logging v1.9.0 @@ -55,17 +55,19 @@ require ( google.golang.org/grpc v1.62.1 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.29.3 - k8s.io/apiextensions-apiserver v0.29.3 - k8s.io/apimachinery v0.29.3 - k8s.io/client-go v0.29.3 - k8s.io/cloud-provider v0.29.3 + k8s.io/api v0.30.0 + k8s.io/apiextensions-apiserver v0.30.0 + k8s.io/apimachinery v0.30.0 + k8s.io/client-go v0.30.0 + k8s.io/cloud-provider v0.30.0 k8s.io/klog v1.0.0 - k8s.io/kubelet v0.29.3 + k8s.io/kubelet v0.30.0 k8s.io/utils v0.0.0-20240310230437-4693a0247e57 kubevirt.io/api v1.2.0 kubevirt.io/containerized-data-importer-api v1.58.1 - sigs.k8s.io/controller-runtime v0.17.2 + // Pinned due to a breaking change in k8s.io/client-go/tools/leaderelection in v0.30.0 + // TODO: Update to the latest semver version when https://github.com/kubernetes-sigs/controller-runtime/pull/2693 is released + sigs.k8s.io/controller-runtime v0.17.1-0.20240418082203-04706074d2f1 sigs.k8s.io/yaml v1.4.0 ) @@ -106,7 +108,7 @@ require ( github.com/docker/distribution v2.8.3+incompatible // indirect github.com/emicklei/go-restful/v3 v3.11.1 // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.8.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/flatcar/ignition v0.36.2 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect @@ -147,8 +149,8 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/ginkgo/v2 v2.16.0 // indirect - github.com/onsi/gomega v1.31.1 // indirect + github.com/onsi/ginkgo/v2 v2.17.1 // indirect + github.com/onsi/gomega v1.32.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect @@ -173,13 +175,13 @@ require ( go.uber.org/multierr v1.11.0 // indirect go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect - golang.org/x/net v0.22.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/sync v0.6.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.17.0 // indirect + golang.org/x/tools v0.18.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect @@ -190,9 +192,9 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/component-base v0.29.3 // indirect - k8s.io/klog/v2 v2.110.1 // indirect - k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910 // indirect + k8s.io/component-base v0.30.0 // indirect + k8s.io/klog/v2 v2.120.1 // indirect + k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index d489d67d3..28edabc29 100644 --- a/go.sum +++ b/go.sum @@ -236,8 +236,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= -github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flatcar/container-linux-config-transpiler v0.9.4 h1:yXQ0NB8PeNrKJPrZvbv5/DV63PNhTqt8vaf8YxmX/RA= @@ -278,7 +278,6 @@ github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -583,16 +582,16 @@ github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042 github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.16.0 h1:7q1w9frJDzninhXxjZd+Y/x54XNjG/UlRLIYPZafsPM= -github.com/onsi/ginkgo/v2 v2.16.0/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= +github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= -github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f h1:3BMVfQpz1xe8MmJprp1+NL8hrpl9I04JVP9EczdCOqE= @@ -925,8 +924,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1131,8 +1130,8 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1311,23 +1310,23 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= -k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= -k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= -k8s.io/apiextensions-apiserver v0.29.3 h1:9HF+EtZaVpFjStakF4yVufnXGPRppWFEQ87qnO91YeI= -k8s.io/apiextensions-apiserver v0.29.3/go.mod h1:po0XiY5scnpJfFizNGo6puNU6Fq6D70UJY2Cb2KwAVc= +k8s.io/api v0.30.0 h1:siWhRq7cNjy2iHssOB9SCGNCl2spiF1dO3dABqZ8niA= +k8s.io/api v0.30.0/go.mod h1:OPlaYhoHs8EQ1ql0R/TsUgaRPhpKNxIMrKQfWUp8QSE= +k8s.io/apiextensions-apiserver v0.30.0 h1:jcZFKMqnICJfRxTgnC4E+Hpcq8UEhT8B2lhBcQ+6uAs= +k8s.io/apiextensions-apiserver v0.30.0/go.mod h1:N9ogQFGcrbWqAY9p2mUAL5mGxsLqwgtUce127VtRX5Y= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= -k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= +k8s.io/apimachinery v0.30.0 h1:qxVPsyDM5XS96NIh9Oj6LavoVFYff/Pon9cZeDIkHHA= +k8s.io/apimachinery v0.30.0/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= -k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= -k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= -k8s.io/cloud-provider v0.29.3 h1:y39hNq0lrPD1qmqQ2ykwMJGeWF9LsepVkR2a4wskwLc= -k8s.io/cloud-provider v0.29.3/go.mod h1:daDV1WkAO6pTrdsn7v8TpN/q9n75ExUC4RJDl7vlPKk= +k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ= +k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY= +k8s.io/cloud-provider v0.30.0 h1:hz1MXkFjsyO167sRZVchXEi2YYMQ6kolBi79nuICjzw= +k8s.io/cloud-provider v0.30.0/go.mod h1:iyVcGvDfmZ7m5cliI9TTHj0VTjYDNpc/K71Gp6hukjU= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.29.3 h1:Oq9/nddUxlnrCuuR2K/jp6aflVvc0uDvxMzAWxnGzAo= -k8s.io/component-base v0.29.3/go.mod h1:Yuj33XXjuOk2BAaHsIGHhCKZQAgYKhqIxIjIr2UXYio= +k8s.io/component-base v0.30.0 h1:cj6bp38g0ainlfYtaOQuRELh5KSYjhKxM+io7AUIk4o= +k8s.io/component-base v0.30.0/go.mod h1:V9x/0ePFNaKeKYA3bOvIbrNoluTSG+fSJKjLdjOoeXQ= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= @@ -1338,15 +1337,15 @@ k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= -k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= -k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910 h1:1Rp/XEKP5uxPs6QrsngEHAxBjaAR78iJRiJq5Fi7LSU= -k8s.io/kube-openapi v0.0.0-20240105020646-a37d4de58910/go.mod h1:Pa1PvrP7ACSkuX6I7KYomY6cmMA0Tx86waBhDUgoKPw= -k8s.io/kubelet v0.29.3 h1:X9h0ZHzc+eUeNTaksbN0ItHyvGhQ7Z0HPjnQD2oHdwU= -k8s.io/kubelet v0.29.3/go.mod h1:jDiGuTkFOUynyBKzOoC1xRSWlgAZ9UPcTYeFyjr6vas= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= +k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/kubelet v0.30.0 h1:/pqHVR2Rn8ExCpn211wL3pMtqRFpcBcJPl4+1INbIMk= +k8s.io/kubelet v0.30.0/go.mod h1:WukdKqbQxnj+csn3K8XOKeX7Sh60J/da25IILjvvB5s= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -1363,8 +1362,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeGOUvw0= -sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= +sigs.k8s.io/controller-runtime v0.17.1-0.20240418082203-04706074d2f1 h1:W15Y5zHVUsH1YJvstRqy6lG0KquU7kS2ooGC5poLnrU= +sigs.k8s.io/controller-runtime v0.17.1-0.20240418082203-04706074d2f1/go.mod h1:umEFUKWCSYpq2U4tNN7riBXU6iiulk7bdF0XZq9LzvU= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index 6d20a7b22..bd740e2ed 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -102,40 +102,40 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.29.2-aws", + name: "kubelet-v1.30.0-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, }, { - name: "kubelet-v1.29.2-aws-external", + name: "kubelet-v1.30.0-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.29.2-vsphere", + name: "kubelet-v1.30.0-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.29.2-vsphere-proxy", + name: "kubelet-v1.30.0-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, cloudProviderName: stringPtr("vsphere"), @@ -145,11 +145,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.29.2-vsphere-mirrors", + name: "kubelet-v1.30.0-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, cloudProviderName: stringPtr("vsphere"), @@ -163,7 +163,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.5", + Kubelet: "1.28.0", }, }, }, diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml index 1f2c60b4b..074abdf66 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml @@ -333,6 +333,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml index c489d477a..f0a23d9b3 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml @@ -143,7 +143,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -333,6 +333,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.29-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29-aws.yaml index 61dd8a9bd..92ba01380 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.29-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.29-aws.yaml @@ -223,8 +223,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ @@ -333,6 +331,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-aws-external.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws-external.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.30.0-aws-external.yaml index f88e1edd9..3b2791812 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-aws-external.yaml @@ -143,7 +143,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -333,6 +333,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-aws.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.30.0-aws.yaml index 0cc636fb6..4cfc6026e 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-aws.yaml @@ -143,7 +143,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -223,8 +223,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ @@ -333,6 +331,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml index 9ac00af2a..f5f678623 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml @@ -156,7 +156,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -238,8 +238,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ @@ -350,6 +348,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-proxy.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere-proxy.yaml index 3ddf865fa..d63a72dc2 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere-proxy.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere-proxy.yaml @@ -156,7 +156,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -238,8 +238,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ @@ -350,6 +348,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere.yaml similarity index 99% rename from pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere.yaml index cc75ec35b..8cf7a2a31 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.29.2-vsphere.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere.yaml @@ -148,7 +148,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -230,8 +230,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ @@ -341,6 +339,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go index d93b87a2f..4b46652c9 100644 --- a/pkg/userdata/centos/provider_test.go +++ b/pkg/userdata/centos/provider_test.go @@ -102,50 +102,50 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.29.2-aws", + name: "kubelet-v1.30.0-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, }, { - name: "kubelet-v1.29.2-nutanix", + name: "kubelet-v1.30.0-nutanix", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, cloudProviderName: stringPtr("nutanix"), }, { - name: "kubelet-v1.29.2-aws-external", + name: "kubelet-v1.30.0-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.29.2-vsphere", + name: "kubelet-v1.30.0-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.29.2-vsphere-proxy", + name: "kubelet-v1.30.0-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, cloudProviderName: stringPtr("vsphere"), @@ -155,11 +155,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.29.2-vsphere-mirrors", + name: "kubelet-v1.30.0-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, cloudProviderName: stringPtr("vsphere"), @@ -173,7 +173,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.5", + Kubelet: "1.28.0", }, }, }, @@ -182,7 +182,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.5", + Kubelet: "1.29.0", }, }, }, diff --git a/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml index 5cd009103..89a34c232 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml @@ -339,6 +339,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml index c043c9718..3791281fe 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -339,6 +339,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml index e64a46ccb..be17408c0 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -229,8 +229,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ @@ -339,6 +337,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.30.0-aws-external.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.29.2-aws-external.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.30.0-aws-external.yaml index 83e3124a4..cc06362a7 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.30.0-aws-external.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -339,6 +339,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.30.0-aws.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.29.2-aws.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.30.0-aws.yaml index 67f9242a4..8b7d461a9 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.29.2-aws.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.30.0-aws.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -229,8 +229,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ @@ -339,6 +337,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29.2-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.30.0-nutanix.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.29.2-nutanix.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.30.0-nutanix.yaml index f5f9f18cd..8b533f0aa 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.29.2-nutanix.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.30.0-nutanix.yaml @@ -156,7 +156,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -236,8 +236,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=nutanix \ - --cloud-config=/etc/kubernetes/cloud-config \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ @@ -347,6 +345,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml index b5401e2c4..5c696b39b 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml @@ -162,7 +162,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -244,8 +244,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ @@ -356,6 +354,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-proxy.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere-proxy.yaml index 54ba3c434..f814bb806 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere-proxy.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere-proxy.yaml @@ -162,7 +162,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -244,8 +244,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ @@ -356,6 +354,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere.yaml similarity index 99% rename from pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere.yaml index 5164e1e12..32dbe0b3a 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.29.2-vsphere.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere.yaml @@ -154,7 +154,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -236,8 +236,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ @@ -347,6 +345,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/flatcar/provider_test.go b/pkg/userdata/flatcar/provider_test.go index faaa60a92..a5ece494a 100644 --- a/pkg/userdata/flatcar/provider_test.go +++ b/pkg/userdata/flatcar/provider_test.go @@ -155,7 +155,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.28.5", + name: "ignition_v1.28.0", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -171,7 +171,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.5", + Kubelet: "1.28.0", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -186,7 +186,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.29.2", + name: "ignition_v1.30.0", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -202,7 +202,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -217,7 +217,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.29.2", + name: "ignition_v1.30.0", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -233,7 +233,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -248,7 +248,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "ignition_v1.29.2", + name: "ignition_v1.30.0", providerSpec: &providerconfigtypes.Config{ CloudProvider: "vsphere", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -264,7 +264,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -310,7 +310,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.28.5", + name: "cloud-init_v1.28.0", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -326,7 +326,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.5", + Kubelet: "1.28.0", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -341,7 +341,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.29.2", + name: "cloud-init_v1.30.0", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -357,7 +357,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -372,7 +372,7 @@ func TestUserDataGeneration(t *testing.T) { }, }, { - name: "cloud-init_v1.29.2", + name: "cloud-init_v1.30.0", providerSpec: &providerconfigtypes.Config{ CloudProvider: "anexia", SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"}, @@ -388,7 +388,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, ccProvider: &fakeCloudConfigProvider{ @@ -415,7 +415,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.2", + Kubelet: "1.30.0", }, }, ccProvider: &fakeCloudConfigProvider{}, diff --git a/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml b/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml index 027f2b4a8..3262adb19 100644 --- a/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml +++ b/pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml @@ -1,509 +1,523 @@ #cloud-config users: - - name: core - ssh_authorized_keys: - - ssh-rsa AAABBB - - ssh-rsa CCCDDD +- name: core + ssh_authorized_keys: + - ssh-rsa AAABBB + - ssh-rsa CCCDDD + coreos: units: - - name: static-nic.network - content: | - [Match] - # Because of difficulty predicting specific NIC names on different cloud providers, - # we only support static addressing on VSphere. There should be a single NIC attached - # that we will match by name prefix 'en' which denotes ethernet devices. - Name=en* - - [Network] - DHCP=no - Address=192.168.81.4/24 - Gateway=192.168.81.1 - DNS=8.8.8.8 - - - name: update-engine.service - command: stop - mask: true - - name: locksmithd.service - command: stop - mask: true - - name: download-script.service - enable: true - command: start - content: | - [Unit] - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/download.sh - [Install] - WantedBy=multi-user.target - - - name: kubelet-healthcheck.service - enable: true - command: start - drop-ins: - - name: 40-download.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service + - name: static-nic.network + content: | + [Match] + # Because of difficulty predicting specific NIC names on different cloud providers, + # we only support static addressing on VSphere. There should be a single NIC attached + # that we will match by name prefix 'en' which denotes ethernet devices. + Name=en* + + [Network] + DHCP=no + Address=192.168.81.4/24 + Gateway=192.168.81.1 + DNS=8.8.8.8 + + - name: update-engine.service + command: stop + mask: true + - name: locksmithd.service + command: stop + mask: true + - name: download-script.service + enable: true + command: start + content: | + [Unit] + Requires=network-online.target + After=network-online.target + [Service] + Type=oneshot + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/download.sh + [Install] + WantedBy=multi-user.target + + - name: kubelet-healthcheck.service + enable: true + command: start + drop-ins: + - name: 40-download.conf content: | [Unit] - Requires=kubelet.service - After=kubelet.service + Requires=download-script.service + After=download-script.service + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet - [Install] - WantedBy=multi-user.target + [Install] + WantedBy=multi-user.target - - name: nodeip.service - enable: true - command: start - content: | - [Unit] - Description=Setup Kubelet Node IP Env - Requires=network-online.target - After=network-online.target + - name: nodeip.service + enable: true + command: start + content: | + [Unit] + Description=Setup Kubelet Node IP Env + Requires=network-online.target + After=network-online.target + + [Service] + ExecStart=/opt/bin/setup_net_env.sh + RemainAfterExit=yes + Type=oneshot + [Install] + WantedBy=multi-user.target + + - name: kubelet.service + enable: true + command: start + drop-ins: + - name: 10-nodeip.conf + content: | [Service] - ExecStart=/opt/bin/setup_net_env.sh - RemainAfterExit=yes - Type=oneshot - [Install] - WantedBy=multi-user.target - - - name: kubelet.service - enable: true - command: start - drop-ins: - - name: 10-nodeip.conf - content: | - [Service] - EnvironmentFile=/etc/kubernetes/nodeip.conf - - name: resolv.conf - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - - name: 40-download.conf - content: | - [Unit] - Requires=download-script.service - After=download-script.service - - name: 50-rpc-statd.conf - content: | - [Unit] - Wants=rpc-statd.service + EnvironmentFile=/etc/kubernetes/nodeip.conf + - name: resolv.conf content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=anexia \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - - - name: apply-sysctl-settings.service - enable: true - command: start + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" + - name: 40-download.conf content: | [Unit] - Requires=network-online.target - After=network-online.target - [Service] - Type=oneshot - ExecStart=/opt/bin/apply_sysctl_settings.sh - [Install] - WantedBy=multi-user.target - -write_files: - - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - permissions: "0644" - content: | - [Journal] - SystemMaxUse=5G - - - path: "/etc/kubernetes/kubelet.conf" - permissions: "0644" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - - path: /opt/load-kernel-modules.sh - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - - path: /etc/sysctl.d/k8s.conf - permissions: "0644" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - - path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - - path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" - permissions: "0755" + Requires=download-script.service + After=download-script.service + - name: 50-rpc-statd.conf + content: | + [Unit] + Wants=rpc-statd.service content: | - # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar - # so we enable IPv6 Router Advertisement here. - # See for details https://github.com/flatcar-linux/Flatcar/issues/384 - [Network] - IPv6AcceptRA=true + [Unit] + After=containerd.service + Requires=containerd.service - - path: /etc/kubernetes/bootstrap-kubelet.conf - permissions: "0400" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - - path: /etc/kubernetes/cloud-config - permissions: "0400" - content: | - {anexia-config:true} + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ - - path: /etc/kubernetes/pki/ca.crt - permissions: "0644" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - - path: /etc/hostname - permissions: "0600" - content: "node1" - - - path: /etc/ssh/sshd_config - permissions: "0600" - user: root - content: | - # Use most defaults for sshd configuration. - Subsystem sftp internal-sftp - ClientAliveInterval 180 - UseDNS no - UsePAM yes - PrintLastLog no # handled by PAM - PrintMotd no # handled by PAM - PasswordAuthentication no - ChallengeResponseAuthentication no - - - path: /opt/bin/download.sh - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.2.0}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") - cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat < /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: /etc/sysctl.d/k8s.conf + permissions: "0644" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" + permissions: "0755" + content: | + # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar + # so we enable IPv6 Router Advertisement here. + # See for details https://github.com/flatcar-linux/Flatcar/issues/384 + [Network] + IPv6AcceptRA=true + +- path: /etc/kubernetes/bootstrap-kubelet.conf + permissions: "0400" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: /etc/kubernetes/cloud-config + permissions: "0400" + content: | + {anexia-config:true} + +- path: /etc/kubernetes/pki/ca.crt + permissions: "0644" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + + +- path: /etc/hostname + permissions: "0600" + content: 'node1' + +- path: /etc/ssh/sshd_config + permissions: "0600" + user: root + content: | + # Use most defaults for sshd configuration. + Subsystem sftp internal-sftp + ClientAliveInterval 180 + UseDNS no + UsePAM yes + PrintLastLog no # handled by PAM + PrintMotd no # handled by PAM + PasswordAuthentication no + ChallengeResponseAuthentication no + +- path: /opt/bin/download.sh + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + opt_bin=/opt/bin + usr_local_bin=/usr/local/bin + cni_bin_dir=/opt/cni/bin + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + arch=${HOST_ARCH-} + if [ -z "$arch" ] + then + case $(uname -m) in + x86_64) + arch="amd64" + ;; + aarch64) + arch="arm64" + ;; + *) + echo "unsupported CPU architecture, exiting" + exit 1 + ;; + esac + fi + CNI_VERSION="${CNI_VERSION:-v1.2.0}" + cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" + cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" + curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" + cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") + cd "$cni_bin_dir" + sha256sum -c <<<"$cni_sum" + tar xvf "$cni_filename" + rm -f "$cni_filename" + cd - + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" + cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" + cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" + curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" + cd "$opt_bin" + sha256sum -c <<<"$cri_tools_sum" + tar xvf "$cri_tools_filename" + rm -f "$cri_tools_filename" + ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" + cd - + KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" + kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" + kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" + kube_sum_file="$kube_dir/sha256" + mkdir -p "$kube_dir" + : >"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + + mkdir -p /etc/systemd/system/containerd.service.d + + cat < /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: /etc/sysctl.d/k8s.conf - permissions: "0644" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" - permissions: "0755" - content: | - # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar - # so we enable IPv6 Router Advertisement here. - # See for details https://github.com/flatcar-linux/Flatcar/issues/384 - [Network] - IPv6AcceptRA=true - -- path: /etc/kubernetes/bootstrap-kubelet.conf - permissions: "0400" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: /etc/kubernetes/cloud-config - permissions: "0400" - content: | - {anexia-config:true} - -- path: /etc/kubernetes/pki/ca.crt - permissions: "0644" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - -- path: /etc/hostname - permissions: "0600" - content: 'node1' - -- path: /etc/ssh/sshd_config - permissions: "0600" - user: root - content: | - # Use most defaults for sshd configuration. - Subsystem sftp internal-sftp - ClientAliveInterval 180 - UseDNS no - UsePAM yes - PrintLastLog no # handled by PAM - PrintMotd no # handled by PAM - PasswordAuthentication no - ChallengeResponseAuthentication no - -- path: /opt/bin/download.sh - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.2.0}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") - cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat < /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-aws-external.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-aws-external.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-aws-external.yaml index 9d84e778d..89cfc07d6 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-aws-external.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -346,6 +346,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.29-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-aws.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.29-aws.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-aws.yaml index cb322562c..a8fb80046 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.29-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-aws.yaml @@ -144,7 +144,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -236,8 +236,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ @@ -346,6 +344,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-nutanix.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-nutanix.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-nutanix.yaml index 36101c601..81478aaf1 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-nutanix.yaml @@ -151,7 +151,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -243,8 +243,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=nutanix \ - --cloud-config=/etc/kubernetes/cloud-config \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ @@ -354,6 +352,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml index 0f70ac398..2a3fd2368 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-mirrors.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml @@ -157,7 +157,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -251,8 +251,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ @@ -363,6 +361,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere-proxy.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-proxy.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere-proxy.yaml index 4a6d9a1ba..9875e6e11 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere-proxy.yaml @@ -157,7 +157,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -251,8 +251,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ @@ -363,6 +361,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere.yaml similarity index 99% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere.yaml index a3c2c5a45..edf34abf3 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.29.2-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere.yaml @@ -149,7 +149,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.2}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -243,8 +243,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ @@ -354,6 +352,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 492c9fcd4..eef43a2b3 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -94,7 +94,7 @@ kPe6XoSbiLm/kxk32T0= ) const ( - defaultVersion = "1.27.6" + defaultVersion = "1.29.0" ) type fakeCloudConfigProvider struct { @@ -129,9 +129,10 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.27.11"), - semver.MustParse("v1.28.7"), - semver.MustParse("v1.29.2"), + semver.MustParse("v1.27.0"), + semver.MustParse("v1.28.0"), + semver.MustParse("v1.29.0"), + semver.MustParse("v1.30.0"), } var tests []userDataTestCase diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index cb8e939bd..f22f8e9b3 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -152,7 +152,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -422,6 +422,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml index baa7cbba2..0efa6733d 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -423,6 +423,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml index b0bbae3c0..a219cc83f 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -423,6 +423,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml index ae4df986c..4ecb2c10c 100644 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml @@ -152,7 +152,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -412,6 +412,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml index 834cdb583..afc0ca39e 100644 --- a/pkg/userdata/ubuntu/testdata/docker.yaml +++ b/pkg/userdata/ubuntu/testdata/docker.yaml @@ -152,7 +152,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -417,6 +417,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml index a95022455..64bdb9fa0 100644 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml @@ -410,6 +410,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml index 9be9387bd..d6bcc6537 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -412,6 +412,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml index 23911aad4..193977e44 100644 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml @@ -412,6 +412,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml index cea7dc61f..6dd5c6aa0 100644 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ b/pkg/userdata/ubuntu/testdata/nutanix.yaml @@ -229,8 +229,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=nutanix \ - --cloud-config=/etc/kubernetes/cloud-config \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ @@ -421,6 +419,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml index aed39b9ab..7c24e7fc0 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -423,6 +423,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml index 28d09a5f2..a45955b95 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -423,6 +423,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml index fa7016c7d..b21555d8d 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml @@ -418,6 +418,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml index 18241af54..08decba36 100644 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.27.6}" + KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -226,8 +226,6 @@ write_files: --kubeconfig=/var/lib/kubelet/kubeconfig \ --config=/etc/kubernetes/kubelet.conf \ --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=openstack \ - --cloud-config=/etc/kubernetes/cloud-config \ --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ @@ -418,6 +416,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/version-1.27.9.yaml b/pkg/userdata/ubuntu/testdata/version-1.27.0.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.27.9.yaml rename to pkg/userdata/ubuntu/testdata/version-1.27.0.yaml index 41ae7b59f..64bdb9fa0 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.27.9.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.27.0.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.27.9}" + KUBE_VERSION="${KUBE_VERSION:-v1.27.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -410,6 +410,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/version-1.28.5.yaml b/pkg/userdata/ubuntu/testdata/version-1.28.0.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.28.5.yaml rename to pkg/userdata/ubuntu/testdata/version-1.28.0.yaml index 1ce969630..2b496a23b 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.28.5.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.28.0.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.5}" + KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -410,6 +410,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/version-1.28.7.yaml b/pkg/userdata/ubuntu/testdata/version-1.28.7.yaml deleted file mode 100644 index 2b99ca5a9..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.28.7.yaml +++ /dev/null @@ -1,459 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.29.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.29.0.yaml index 0d62ccc34..5c3f6533b 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.29.0.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.29.0.yaml @@ -410,6 +410,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/version-1.29.2.yaml b/pkg/userdata/ubuntu/testdata/version-1.29.2.yaml deleted file mode 100644 index 3831bb0e0..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.29.2.yaml +++ /dev/null @@ -1,459 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.27.11.yaml b/pkg/userdata/ubuntu/testdata/version-1.30.0.yaml similarity index 99% rename from pkg/userdata/ubuntu/testdata/version-1.27.11.yaml rename to pkg/userdata/ubuntu/testdata/version-1.30.0.yaml index 8cf0a7d5c..e450b0951 100644 --- a/pkg/userdata/ubuntu/testdata/version-1.27.11.yaml +++ b/pkg/userdata/ubuntu/testdata/version-1.30.0.yaml @@ -150,7 +150,7 @@ write_files: rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.27.11}" + KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -410,6 +410,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml index bc78ed1e5..ff6e90790 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml @@ -429,6 +429,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml index 5b1c95607..275591c13 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml @@ -436,6 +436,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml index d418274a6..5c858de6d 100644 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ b/pkg/userdata/ubuntu/testdata/vsphere.yaml @@ -419,6 +419,8 @@ write_files: options: json: infoBufferSize: "0" + text: + infoBufferSize: "0" verbosity: 0 memorySwap: {} nodeStatusReportFrequency: 0s diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index b949a7e53..9b2b40d94 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -84,7 +84,7 @@ const ( ) const ( - defaultKubernetesVersion = "1.28.7" + defaultKubernetesVersion = "1.29.4" awsDefaultKubernetesVersion = "1.26.12" defaultContainerRuntime = "containerd" ) @@ -347,7 +347,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.26. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.27.11", "1.28.7", "1.29.2"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.27.13", "1.28.9", "1.29.4", "1.30.0"))) runScenarios(context.Background(), t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -427,7 +427,7 @@ func TestAWSProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := Not(VersionSelector("1.27.11", "1.28.7", "1.29.2")) + selector := Not(VersionSelector("1.27.13", "1.28.9", "1.29.4", "1.30.0")) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -481,7 +481,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.11", "1.28.7", "1.29.2"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.13", "1.28.9", "1.29.4", "1.30.0"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -503,7 +503,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.11", "1.28.7", "1.29.2"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.13", "1.28.9", "1.29.4", "1.30.0"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -598,7 +598,8 @@ func TestAzureProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") } - selector := Not(OsSelector("amzn2")) + // In-tree cloud provider is not supported from Kubernetes v1.30. + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.30.0"))) // act params := []string{ @@ -626,7 +627,8 @@ func TestAzureCustomImageReferenceProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") } - selector := OsSelector("ubuntu") + // In-tree cloud provider is not supported from Kubernetes v1.30. + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.0"))) // act params := []string{ fmt.Sprintf("<< AZURE_TENANT_ID >>=%s", azureTenantID), @@ -688,7 +690,7 @@ func TestGCEProvisioningE2E(t *testing.T) { } // Act. GCE does not support CentOS. - selector := And(OsSelector("ubuntu", "flatcar"), Not(VersionSelector("1.29.2"))) + selector := OsSelector("ubuntu", "flatcar") params := []string{ fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>=%s", safeBase64Encoding(googleServiceAccount)), } @@ -841,7 +843,8 @@ func getVSphereTestParams(t *testing.T) []string { func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() - selector := Not(OsSelector("amzn2", "centos")) + // In-tree cloud provider is not supported from Kubernetes v1.30. + selector := And(Not(OsSelector("amzn2", "centos")), Not(VersionSelector("1.30.0"))) params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) @@ -852,7 +855,8 @@ func TestVsphereProvisioningE2E(t *testing.T) { func TestVsphereMultipleNICProvisioningE2E(t *testing.T) { t.Parallel() - selector := OsSelector("ubuntu") + // In-tree cloud provider is not supported from Kubernetes v1.30. + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.0"))) params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereMultipleNICManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) @@ -863,7 +867,8 @@ func TestVsphereMultipleNICProvisioningE2E(t *testing.T) { func TestVsphereDatastoreClusterProvisioningE2E(t *testing.T) { t.Parallel() - selector := OsSelector("ubuntu", "centos", "rhel", "flatcar") + // In-tree cloud provider is not supported from Kubernetes v1.30. + selector := And(OsSelector("ubuntu", "centos", "rhel", "flatcar"), Not(VersionSelector("1.30.0"))) params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereDSCManifest, fmt.Sprintf("vs-dsc-%s", *testRunIdentifier)) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index a0f2f0eb0..3389f8bf4 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -34,9 +34,10 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.27.11"), - semver.MustParse("v1.28.7"), - semver.MustParse("v1.29.2"), + semver.MustParse("v1.27.13"), + semver.MustParse("v1.28.9"), + semver.MustParse("v1.29.4"), + semver.MustParse("v1.30.0"), } operatingSystems = []providerconfigtypes.OperatingSystem{ diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml index d92a6a034..71485b5f2 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml @@ -55,4 +55,4 @@ spec: rhelOrganizationName: "" rhelActivationKey: "" versions: - kubelet: 1.28.5 + kubelet: 1.29.4 From eaccda7855b6cd493f082243e0a57027158d7bd0 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 23 Apr 2024 21:44:37 +0500 Subject: [PATCH 384/489] Fix vSphere anti-affinity (#1779) Signed-off-by: Waleed Malik --- .prow/provider-vsphere.yaml | 32 ++++++ pkg/cloudprovider/provider/vsphere/helper.go | 5 - .../provider/vsphere/provider.go | 18 ++-- pkg/cloudprovider/provider/vsphere/rule.go | 98 +++++++++---------- test/e2e/provisioning/all_e2e_test.go | 18 ++++ ...chinedeployment-vsphere-anti-affinity.yaml | 54 ++++++++++ 6 files changed, 156 insertions(+), 69 deletions(-) create mode 100644 test/e2e/provisioning/testdata/machinedeployment-vsphere-anti-affinity.yaml diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 3cb5b6d1d..c5866d1cf 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -140,3 +140,35 @@ presubmits: cpu: 2 limits: memory: 7Gi + + - name: pull-machine-controller-e2e-vsphere-anti-affinity + always_run: false + decorate: true + clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + labels: + preset-hetzner: "true" + preset-e2e-ssh: "true" + preset-vsphere: "true" + preset-rhel: "true" + preset-goproxy: "true" + preset-kind-volume-mounts: "true" + preset-docker-mirror: "true" + preset-kubeconfig-ci: "true" + spec: + containers: + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + command: + - "./hack/ci/run-e2e-tests.sh" + args: + - "TestVsphereAntiAffinityProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: vsphere + securityContext: + privileged: true + resources: + requests: + memory: 7Gi + cpu: 2 + limits: + memory: 7Gi diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go index a700f744e..f6b6000ab 100644 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ b/pkg/cloudprovider/provider/vsphere/helper.go @@ -103,12 +103,10 @@ func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, if err := clonedVMTask.WaitEx(ctx); err != nil { return nil, fmt.Errorf("error when waiting for result of clone task: %w", err) } - virtualMachine, err := session.Finder.VirtualMachine(ctx, vmName) if err != nil { return nil, fmt.Errorf("failed to get virtual machine object after cloning: %w", err) } - vmDevices, err := virtualMachine.Device(ctx) if err != nil { return nil, fmt.Errorf("failed to list devices of template VM: %w", err) @@ -138,7 +136,6 @@ func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, guestInfoUserData = "guestinfo.ignition.config.data" guestInfoUserDataEncoding = "guestinfo.ignition.config.data.encoding" - for _, item := range mvm.Config.VAppConfig.GetVmConfigInfo().Property { switch item.Id { case guestInfoUserData: @@ -170,7 +167,6 @@ func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, } diskUUIDEnabled := true - var deviceSpecs []types.BaseVirtualDeviceConfigSpec if config.DiskSizeGB != nil { disks, err := getDisksFromVM(ctx, virtualMachine) @@ -221,7 +217,6 @@ func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, if err := removeFloppyDevice(ctx, virtualMachine); err != nil { return nil, fmt.Errorf("failed to remove floppy device: %w", err) } - return virtualMachine, nil } diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index c3452bde0..c8d828276 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -23,7 +23,6 @@ import ( "net/url" "os" "strings" - "sync" "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/object" @@ -49,7 +48,6 @@ import ( type provider struct { configVarResolver *providerconfig.ConfigVarResolver - mutex sync.Mutex } // New returns a VSphere provider. @@ -384,8 +382,7 @@ func (p *provider) create(ctx context.Context, log *zap.SugaredLogger, machine * } if config.VMAntiAffinity { - machineSetName := machine.Name[:strings.LastIndex(machine.Name, "-")] - if err := p.createOrUpdateVMAntiAffinityRule(ctx, session, machineSetName, config); err != nil { + if err := p.createOrUpdateVMAntiAffinityRule(ctx, log, session, machine, config); err != nil { return nil, fmt.Errorf("failed to add VM to anti affinity rule: %w", err) } } @@ -452,6 +449,12 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine return false, fmt.Errorf("failed to delete tags: %w", err) } + if config.VMAntiAffinity { + if err := p.createOrUpdateVMAntiAffinityRule(ctx, log, session, machine, config); err != nil { + return false, fmt.Errorf("failed to update VMs in anti-affinity rule: %w", err) + } + } + powerState, err := virtualMachine.PowerState(ctx) if err != nil { return false, fmt.Errorf("failed to get virtual machine power state: %w", err) @@ -507,13 +510,6 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine return false, fmt.Errorf("failed to destroy vm %s: %w", virtualMachine.Name(), err) } - if config.VMAntiAffinity { - machineSetName := machine.Name[:strings.LastIndex(machine.Name, "-")] - if err := p.createOrUpdateVMAntiAffinityRule(ctx, session, machineSetName, config); err != nil { - return false, fmt.Errorf("failed to add VM to anti affinity rule: %w", err) - } - } - if pc.OperatingSystem != providerconfigtypes.OperatingSystemFlatcar { filemanager := datastore.NewFileManager(session.Datacenter, false) diff --git a/pkg/cloudprovider/provider/vsphere/rule.go b/pkg/cloudprovider/provider/vsphere/rule.go index 8df7f3811..98e713ba5 100644 --- a/pkg/cloudprovider/provider/vsphere/rule.go +++ b/pkg/cloudprovider/provider/vsphere/rule.go @@ -20,64 +20,75 @@ import ( "context" "errors" "fmt" - "reflect" "strings" - "time" + "sync" - "github.com/aws/smithy-go/ptr" "github.com/vmware/govmomi/find" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/mo" "github.com/vmware/govmomi/vim25/types" + "go.uber.org/zap" + + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + + "k8s.io/utils/ptr" ) +var lock sync.Mutex + // createOrUpdateVMAntiAffinityRule creates or updates an anti affinity rule with the name in the given cluster. // VMs are attached to the rule based on their folder path and name prefix in vsphere. // A minimum of two VMs is required. -func (p *provider) createOrUpdateVMAntiAffinityRule(ctx context.Context, session *Session, name string, config *Config) error { - p.mutex.Lock() - defer p.mutex.Unlock() - +func (p *provider) createOrUpdateVMAntiAffinityRule(ctx context.Context, log *zap.SugaredLogger, session *Session, machine *clusterv1alpha1.Machine, config *Config) error { + lock.Lock() + defer lock.Unlock() cluster, err := session.Finder.ClusterComputeResource(ctx, config.Cluster) if err != nil { return err } + machineSetName := machine.Name[:strings.LastIndex(machine.Name, "-")] vmsInFolder, err := session.Finder.VirtualMachineList(ctx, strings.Join([]string{config.Folder, "*"}, "/")) if err != nil { if errors.Is(err, &find.NotFoundError{}) { - return removeVMAntiAffinityRule(ctx, session, config.Cluster, name) + return removeVMAntiAffinityRule(ctx, session, config.Cluster, machineSetName) } return err } var ruleVMRef []types.ManagedObjectReference for _, vm := range vmsInFolder { - if strings.HasPrefix(vm.Name(), name) { + // Only add VMs with the same machineSetName to the rule and exclude the machine itself if it is being deleted + if strings.HasPrefix(vm.Name(), machineSetName) && !(vm.Name() == machine.Name && machine.DeletionTimestamp != nil) { ruleVMRef = append(ruleVMRef, vm.Reference()) } } - // minimum of two vms required - if len(ruleVMRef) < 2 { - return removeVMAntiAffinityRule(ctx, session, config.Cluster, name) + if len(ruleVMRef) == 0 { + log.Debugf("No VMs in folder %s with name prefix %s found", config.Folder, machineSetName) + return removeVMAntiAffinityRule(ctx, session, config.Cluster, machineSetName) + } else if len(ruleVMRef) < 2 { + // DRS rule must have at least two virtual machine members + log.Debugf("Not enough VMs in folder %s to create anti-affinity rule", config.Folder) + return nil } - info, err := findClusterAntiAffinityRuleByName(ctx, cluster, name) + info, err := findClusterAntiAffinityRuleByName(ctx, cluster, machineSetName) if err != nil { return err } + log.Debugf("Creating or updating anti-affinity rule for VMs %v in cluster %s", ruleVMRef, config.Cluster) operation := types.ArrayUpdateOperationEdit //create new rule if info == nil { info = &types.ClusterAntiAffinityRuleSpec{ ClusterRuleInfo: types.ClusterRuleInfo{ - Enabled: ptr.Bool(true), - Mandatory: ptr.Bool(false), - Name: name, - UserCreated: ptr.Bool(true), + Enabled: ptr.To(true), + Mandatory: ptr.To(false), + Name: machineSetName, + UserCreated: ptr.To(true), }, } operation = types.ArrayUpdateOperationAdd @@ -95,49 +106,22 @@ func (p *provider) createOrUpdateVMAntiAffinityRule(ctx context.Context, session }, } + log.Debugf("Performing %q for anti-affinity rule for VMs %v in cluster %s", operation, ruleVMRef, config.Cluster) task, err := cluster.Reconfigure(ctx, spec, true) if err != nil { return err } - err = task.WaitEx(ctx) + taskResult, err := task.WaitForResultEx(ctx) if err != nil { - return err + return fmt.Errorf("error waiting for cluster %v reconfiguration to complete", cluster.Name()) } - - return waitForRule(ctx, cluster, info) -} - -// waitForRule checks periodically the vsphere api for the ClusterAntiAffinityRule and returns error if the rule was not found after a timeout. -func waitForRule(ctx context.Context, cluster *object.ClusterComputeResource, rule *types.ClusterAntiAffinityRuleSpec) error { - timeout := time.NewTimer(10 * time.Second) - ticker := time.NewTicker(500 * time.Millisecond) - defer timeout.Stop() - defer ticker.Stop() - - for { - select { - case <-timeout.C: - - info, err := findClusterAntiAffinityRuleByName(ctx, cluster, rule.Name) - if err != nil { - return err - } - - if !reflect.DeepEqual(rule, info) { - return fmt.Errorf("expected anti affinity changes not found in vsphere") - } - case <-ticker.C: - info, err := findClusterAntiAffinityRuleByName(ctx, cluster, rule.Name) - if err != nil { - return err - } - - if reflect.DeepEqual(rule, info) { - return nil - } - } + if taskResult.State != types.TaskInfoStateSuccess { + return fmt.Errorf("cluster %v reconfiguration task was not successful", cluster.Name()) } + log.Debugf("Successfully created/updated anti-affinity rule for machineset %v against machine %v", machineSetName, machine.Name) + + return nil } // removeVMAntiAffinityRule removes an anti affinity rule with the name in the given cluster. @@ -172,7 +156,15 @@ func removeVMAntiAffinityRule(ctx context.Context, session *Session, clusterPath if err != nil { return err } - return task.WaitEx(ctx) + + taskResult, err := task.WaitForResultEx(ctx) + if err != nil { + return fmt.Errorf("error waiting for cluster %v reconfiguration to complete", cluster.Name()) + } + if taskResult.State != types.TaskInfoStateSuccess { + return fmt.Errorf("cluster %v reconfiguration task was not successful", cluster.Name()) + } + return nil } func findClusterAntiAffinityRuleByName(ctx context.Context, cluster *object.ClusterComputeResource, name string) (*types.ClusterAntiAffinityRuleSpec, error) { diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 9b2b40d94..d41bc76ae 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -66,6 +66,7 @@ const ( LinodeManifest = "./testdata/machinedeployment-linode.yaml" VMwareCloudDirectorManifest = "./testdata/machinedeployment-vmware-cloud-director.yaml" VSPhereManifest = "./testdata/machinedeployment-vsphere.yaml" + VSPhereAntiAffinityManifest = "./testdata/machinedeployment-vsphere-anti-affinity.yaml" VSPhereMultipleNICManifest = "./testdata/machinedeployment-vsphere-multiple-nic.yaml" VSPhereDSCManifest = "./testdata/machinedeployment-vsphere-datastore-cluster.yaml" VSPhereResourcePoolManifest = "./testdata/machinedeployment-vsphere-resource-pool.yaml" @@ -862,6 +863,23 @@ func TestVsphereMultipleNICProvisioningE2E(t *testing.T) { runScenarios(context.Background(), t, selector, params, VSPhereMultipleNICManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) } +// TestVsphereAntiAffinityProvisioningE2E - is the same as the TestVsphereProvisioning suit but has anti-affinity rules applied to the VMs. +func TestVsphereAntiAffinityProvisioningE2E(t *testing.T) { + t.Parallel() + + params := getVSphereTestParams(t) + + scenario := scenario{ + name: "VSphere Anti-Affinity provisioning", + osName: "ubuntu", + containerRuntime: defaultContainerRuntime, + kubernetesVersion: defaultKubernetesVersion, + executor: verifyCreateAndDelete, + } + + testScenario(context.Background(), t, scenario, *testRunIdentifier, params, VSPhereAntiAffinityManifest, false) +} + // TestVsphereDatastoreClusterProvisioning - is the same as the TestVsphereProvisioning suite but specifies a DatastoreCluster // instead of the Datastore in the provider specs. func TestVsphereDatastoreClusterProvisioningE2E(t *testing.T) { diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-anti-affinity.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-anti-affinity.yaml new file mode 100644 index 000000000..8f74c8465 --- /dev/null +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-anti-affinity.yaml @@ -0,0 +1,54 @@ +apiVersion: "cluster.k8s.io/v1alpha1" +kind: MachineDeployment +metadata: + name: << MACHINE_NAME >> + namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> +spec: + replicas: 3 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + name: << MACHINE_NAME >> + template: + metadata: + labels: + name: << MACHINE_NAME >> + spec: + providerSpec: + value: + sshPublicKeys: + - "<< YOUR_PUBLIC_KEY >>" + cloudProvider: "vsphere" + cloudProviderSpec: + templateVMName: '<< OS_Image_Template >>' + username: '<< VSPHERE_USERNAME >>' + vsphereURL: '<< VSPHERE_ADDRESS >>' + datacenter: 'Hamburg' + folder: '/Hamburg/vm/Kubermatic-ci' + password: << VSPHERE_PASSWORD >> + # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically + cluster: Kubermatic + vmAntiAffinity: true + datastore: vsan + cpus: 2 + MemoryMB: 4096 + diskSizeGB: << DISK_SIZE >> + allowInsecure: true + operatingSystem: "<< OS_NAME >>" + operatingSystemSpec: + distUpgradeOnBoot: false + disableAutoUpdate: true + attachSubscription: false + # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` + rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" + # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` + rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" + rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" + versions: + kubelet: "<< KUBERNETES_VERSION >>" From 0231d551ec36d073bd71bb564461a6d5c2319dd7 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 2 May 2024 14:45:45 +0500 Subject: [PATCH 385/489] Remove user-data plugins from machine-controller (#1789) * Remove userdata plugins Signed-off-by: Waleed Malik * Code cleanup Signed-off-by: Waleed Malik * Update scripts Signed-off-by: Waleed Malik * Remove pipelines for testing userdata Signed-off-by: Waleed Malik * deprecate use-external-bootstrap Signed-off-by: Waleed Malik * Update Dockerfile Signed-off-by: Waleed Malik * Cleanup for pipelines Signed-off-by: Waleed Malik * More cleanup Signed-off-by: Waleed Malik * More cleanup Signed-off-by: Waleed Malik * Update README Signed-off-by: Waleed Malik * Fix formatting for scripts Signed-off-by: Waleed Malik * Make use-external-bootstrap no-op Signed-off-by: Waleed Malik * More cleanup Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- .prow/provider-aws.yaml | 35 - .prow/provider-gcp.yaml | 34 - Dockerfile | 1 - Makefile | 12 +- README.md | 2 +- cmd/machine-controller/main.go | 7 +- cmd/userdata/amzn2/main.go | 58 -- cmd/userdata/centos/main.go | 58 -- cmd/userdata/flatcar/main.go | 58 -- cmd/userdata/rhel/main.go | 58 -- cmd/userdata/rockylinux/main.go | 58 -- cmd/userdata/ubuntu/main.go | 58 -- cmd/webhook/main.go | 24 +- examples/machine-controller.yaml | 2 - hack/ci/download-gocache.sh | 2 +- hack/ci/run-e2e-tests.sh | 8 +- hack/ci/setup-machine-controller-in-kind.sh | 58 +- hack/lib.sh | 2 +- hack/run-machine-controller.sh | 5 +- hack/verify-boilerplate.sh | 3 +- image-builder/build.sh | 302 +++---- image-builder/download_kubernetes.sh | 70 +- pkg/admission/admission.go | 43 +- pkg/admission/machines.go | 12 +- pkg/apis/cluster/common/consts.go | 45 - pkg/apis/plugin/plugin.go | 69 -- pkg/controller/machine/controller.go | 146 +-- pkg/controller/machine/kubeconfig.go | 256 ------ pkg/controller/machine/kubeconfig_test.go | 100 --- pkg/controller/util/machine.go | 3 - pkg/providerconfig/types.go | 4 +- pkg/providerconfig/types/types.go | 9 + pkg/providerconfig/types_test.go | 2 +- pkg/userdata/amzn2/provider.go | 330 ------- pkg/userdata/amzn2/provider_test.go | 278 ------ .../amzn2/testdata/kubelet-v1.27-aws.yaml | 456 ---------- .../amzn2/testdata/kubelet-v1.28-aws.yaml | 456 ---------- .../amzn2/testdata/kubelet-v1.29-aws.yaml | 454 ---------- .../kubelet-v1.30.0-aws-external.yaml | 456 ---------- .../amzn2/testdata/kubelet-v1.30.0-aws.yaml | 454 ---------- .../kubelet-v1.30.0-vsphere-mirrors.yaml | 471 ---------- .../kubelet-v1.30.0-vsphere-proxy.yaml | 478 ---------- .../testdata/kubelet-v1.30.0-vsphere.yaml | 462 ---------- pkg/userdata/centos/provider.go | 384 -------- pkg/userdata/centos/provider_test.go | 288 ------ .../centos/testdata/kubelet-v1.27-aws.yaml | 462 ---------- .../centos/testdata/kubelet-v1.28-aws.yaml | 462 ---------- .../centos/testdata/kubelet-v1.29-aws.yaml | 460 ---------- .../kubelet-v1.30.0-aws-external.yaml | 462 ---------- .../centos/testdata/kubelet-v1.30.0-aws.yaml | 460 ---------- .../testdata/kubelet-v1.30.0-nutanix.yaml | 468 ---------- .../kubelet-v1.30.0-vsphere-mirrors.yaml | 477 ---------- .../kubelet-v1.30.0-vsphere-proxy.yaml | 484 ---------- .../testdata/kubelet-v1.30.0-vsphere.yaml | 468 ---------- pkg/userdata/convert/ignition-converter.go | 44 - pkg/userdata/flatcar/flatcar.go | 13 - pkg/userdata/flatcar/provider.go | 834 ------------------ pkg/userdata/flatcar/provider_test.go | 497 ----------- .../flatcar/testdata/cloud-init_v1.28.0.yaml | 523 ----------- .../flatcar/testdata/cloud-init_v1.29.0.yaml | 521 ----------- .../flatcar/testdata/cloud-init_v1.30.0.yaml | 521 ----------- pkg/userdata/flatcar/testdata/containerd.yaml | 502 ----------- .../flatcar/testdata/ignition_v1.28.0.json | 1 - .../flatcar/testdata/ignition_v1.29.0.json | 1 - .../flatcar/testdata/ignition_v1.30.0.json | 1 - pkg/userdata/helper/common_test.go | 34 - .../helper/download_binaries_script.go | 220 ----- .../helper/download_binaries_script_test.go | 54 -- pkg/userdata/helper/helper.go | 216 ----- pkg/userdata/helper/kubelet.go | 379 -------- pkg/userdata/helper/kubelet_test.go | 145 --- pkg/userdata/helper/template_functions.go | 81 -- .../testdata/download_binaries_v1.27.0.golden | 17 - .../testdata/download_binaries_v1.28.0.golden | 17 - .../testdata/download_binaries_v1.29.0.golden | 17 - .../testdata/download_binaries_v1.30.0.golden | 17 - ...let_systemd_unit_cloud-provider-set.golden | 34 - ...t_systemd_unit_multiple-dns-servers.golden | 35 - ...kublet_systemd_unit_pause-image-set.golden | 35 - .../kublet_systemd_unit_taints-set.golden | 35 - ...stemd_unit_version-v1.27.0-external.golden | 36 - ...kublet_systemd_unit_version-v1.27.0.golden | 35 - ...stemd_unit_version-v1.28.0-external.golden | 36 - ...kublet_systemd_unit_version-v1.28.0.golden | 35 - ...stemd_unit_version-v1.29.0-external.golden | 36 - ...kublet_systemd_unit_version-v1.29.0.golden | 35 - ...stemd_unit_version-v1.30.0-external.golden | 36 - ...kublet_systemd_unit_version-v1.30.0.golden | 35 - .../safe_download_binaries_v1.30.0.golden | 65 -- pkg/userdata/manager/manager.go | 98 -- pkg/userdata/manager/plugin.go | 144 --- pkg/userdata/plugin/plugin.go | 90 -- pkg/userdata/rhel/provider.go | 432 --------- pkg/userdata/rhel/provider_test.go | 298 ------- .../rhel/testdata/kubelet-v1.28-aws.yaml | 503 ----------- .../rhel/testdata/kubelet-v1.28-nutanix.yaml | 512 ----------- .../rhel/testdata/kubelet-v1.29-aws.yaml | 501 ----------- .../rhel/testdata/kubelet-v1.29-nutanix.yaml | 510 ----------- .../kubelet-v1.30.0-aws-external.yaml | 503 ----------- .../rhel/testdata/kubelet-v1.30.0-aws.yaml | 501 ----------- .../kubelet-v1.30.0-vsphere-mirrors.yaml | 519 ----------- .../kubelet-v1.30.0-vsphere-proxy.yaml | 526 ----------- .../testdata/kubelet-v1.30.0-vsphere.yaml | 510 ----------- .../rhel/testdata/pod-cidr-azure-rhel.yaml | 507 ----------- pkg/userdata/rockylinux/provider.go | 352 -------- pkg/userdata/rockylinux/provider_test.go | 270 ------ .../testdata/kubelet-v1.28-aws.yaml | 469 ---------- .../kubelet-v1.30.0-aws-external.yaml | 469 ---------- .../testdata/kubelet-v1.30.0-aws.yaml | 467 ---------- .../testdata/kubelet-v1.30.0-nutanix.yaml | 475 ---------- .../kubelet-v1.30.0-vsphere-mirrors.yaml | 484 ---------- .../kubelet-v1.30.0-vsphere-proxy.yaml | 491 ----------- .../testdata/kubelet-v1.30.0-vsphere.yaml | 475 ---------- pkg/userdata/scripts/health-monitor.sh | 117 --- pkg/userdata/ubuntu/provider.go | 380 -------- pkg/userdata/ubuntu/provider_test.go | 693 --------------- pkg/userdata/ubuntu/testdata/containerd.yaml | 473 ---------- .../digitalocean-dualstack-IPv6+IPv4.yaml | 474 ---------- .../testdata/digitalocean-dualstack.yaml | 474 ---------- .../ubuntu/testdata/dist-upgrade-on-boot.yaml | 463 ---------- pkg/userdata/ubuntu/testdata/docker.yaml | 468 ---------- .../kubelet-version-without-v-prefix.yaml | 461 ---------- .../ubuntu/testdata/multiple-dns-servers.yaml | 463 ---------- .../ubuntu/testdata/multiple-ssh-keys.yaml | 463 ---------- pkg/userdata/ubuntu/testdata/nutanix.yaml | 470 ---------- .../openstack-dualstack-IPv6+IPv4.yaml | 474 ---------- .../ubuntu/testdata/openstack-dualstack.yaml | 474 ---------- .../openstack-overwrite-cloud-config.yaml | 469 ---------- pkg/userdata/ubuntu/testdata/openstack.yaml | 467 ---------- .../ubuntu/testdata/version-1.27.0.yaml | 461 ---------- .../ubuntu/testdata/version-1.28.0.yaml | 461 ---------- .../ubuntu/testdata/version-1.29.0.yaml | 461 ---------- .../ubuntu/testdata/version-1.30.0.yaml | 461 ---------- .../ubuntu/testdata/vsphere-mirrors.yaml | 480 ---------- .../ubuntu/testdata/vsphere-proxy.yaml | 487 ---------- pkg/userdata/ubuntu/testdata/vsphere.yaml | 470 ---------- test/e2e/provisioning/all_e2e_test.go | 4 - 137 files changed, 283 insertions(+), 36308 deletions(-) delete mode 100644 cmd/userdata/amzn2/main.go delete mode 100644 cmd/userdata/centos/main.go delete mode 100644 cmd/userdata/flatcar/main.go delete mode 100644 cmd/userdata/rhel/main.go delete mode 100644 cmd/userdata/rockylinux/main.go delete mode 100644 cmd/userdata/ubuntu/main.go delete mode 100644 pkg/apis/plugin/plugin.go delete mode 100644 pkg/controller/machine/kubeconfig.go delete mode 100644 pkg/controller/machine/kubeconfig_test.go delete mode 100644 pkg/userdata/amzn2/provider.go delete mode 100644 pkg/userdata/amzn2/provider_test.go delete mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml delete mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml delete mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.29-aws.yaml delete mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.30.0-aws-external.yaml delete mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.30.0-aws.yaml delete mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml delete mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere-proxy.yaml delete mode 100644 pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere.yaml delete mode 100644 pkg/userdata/centos/provider.go delete mode 100644 pkg/userdata/centos/provider_test.go delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.30.0-aws-external.yaml delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.30.0-aws.yaml delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.30.0-nutanix.yaml delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere-proxy.yaml delete mode 100644 pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere.yaml delete mode 100644 pkg/userdata/convert/ignition-converter.go delete mode 100644 pkg/userdata/flatcar/provider.go delete mode 100644 pkg/userdata/flatcar/provider_test.go delete mode 100644 pkg/userdata/flatcar/testdata/cloud-init_v1.28.0.yaml delete mode 100644 pkg/userdata/flatcar/testdata/cloud-init_v1.29.0.yaml delete mode 100644 pkg/userdata/flatcar/testdata/cloud-init_v1.30.0.yaml delete mode 100644 pkg/userdata/flatcar/testdata/containerd.yaml delete mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.28.0.json delete mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.29.0.json delete mode 100644 pkg/userdata/flatcar/testdata/ignition_v1.30.0.json delete mode 100644 pkg/userdata/helper/common_test.go delete mode 100644 pkg/userdata/helper/download_binaries_script.go delete mode 100644 pkg/userdata/helper/download_binaries_script_test.go delete mode 100644 pkg/userdata/helper/helper.go delete mode 100644 pkg/userdata/helper/kubelet.go delete mode 100644 pkg/userdata/helper/kubelet_test.go delete mode 100644 pkg/userdata/helper/template_functions.go delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.27.0.golden delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.28.0.golden delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.29.0.golden delete mode 100644 pkg/userdata/helper/testdata/download_binaries_v1.30.0.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_pause-image-set.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_taints-set.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.0-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.0.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.0-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.0.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.30.0-external.golden delete mode 100644 pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.30.0.golden delete mode 100644 pkg/userdata/helper/testdata/safe_download_binaries_v1.30.0.golden delete mode 100644 pkg/userdata/manager/manager.go delete mode 100644 pkg/userdata/manager/plugin.go delete mode 100644 pkg/userdata/plugin/plugin.go delete mode 100644 pkg/userdata/rhel/provider.go delete mode 100644 pkg/userdata/rhel/provider_test.go delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.28-aws.yaml delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.29-aws.yaml delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.29-nutanix.yaml delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.30.0-aws-external.yaml delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.30.0-aws.yaml delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.30.0-vsphere-proxy.yaml delete mode 100644 pkg/userdata/rhel/testdata/kubelet-v1.30.0-vsphere.yaml delete mode 100644 pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml delete mode 100644 pkg/userdata/rockylinux/provider.go delete mode 100644 pkg/userdata/rockylinux/provider_test.go delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-aws-external.yaml delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-aws.yaml delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-nutanix.yaml delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere-proxy.yaml delete mode 100644 pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere.yaml delete mode 100644 pkg/userdata/scripts/health-monitor.sh delete mode 100644 pkg/userdata/ubuntu/provider.go delete mode 100644 pkg/userdata/ubuntu/provider_test.go delete mode 100644 pkg/userdata/ubuntu/testdata/containerd.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/docker.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/nutanix.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/openstack.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.27.0.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.28.0.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.29.0.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/version-1.30.0.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml delete mode 100644 pkg/userdata/ubuntu/testdata/vsphere.yaml diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index df65767a6..1811fc824 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -46,41 +46,6 @@ presubmits: limits: memory: 7Gi - - name: pull-machine-controller-e2e-aws-spot-instance-legacy-userdata - # In-tree CCM is not supported for AWS starting from k8s 1.27. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. - # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 - env: - - name: OPERATING_SYSTEM_MANAGER - value: "false" - - name: CLOUD_PROVIDER - value: aws - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAWSSpotInstanceProvisioningE2E" - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - name: pull-machine-controller-e2e-aws-arm # In-tree CCM is not supported for AWS starting from k8s 1.27. Please see https://github.com/kubermatic/machine-controller/issues/1626 for updates. # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 27862120d..3b34bb8a1 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -44,37 +44,3 @@ presubmits: cpu: 2 limits: memory: 7Gi - - - name: pull-machine-controller-e2e-gce-legacy-userdata - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-gce: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-rhel: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestGCEProvisioningE2E" - env: - - name: OPERATING_SYSTEM_MANAGER - value: "false" - - name: CLOUD_PROVIDER - value: gce - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi diff --git a/Dockerfile b/Dockerfile index 0ce3d2494..6c87ca442 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,6 @@ RUN apk add --no-cache ca-certificates cdrkit COPY --from=builder \ /go/src/github.com/kubermatic/machine-controller/machine-controller \ - /go/src/github.com/kubermatic/machine-controller/machine-controller-userdata-* \ /go/src/github.com/kubermatic/machine-controller/webhook \ /usr/local/bin/ USER nobody diff --git a/Makefile b/Makefile index 88d409470..f5c592bea 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,6 @@ IMAGE_TAG = \ IMAGE_NAME ?= $(REGISTRY)/$(REGISTRY_NAMESPACE)/machine-controller:$(IMAGE_TAG) OS = amzn2 centos ubuntu rhel flatcar rockylinux -USERDATA_BIN = $(patsubst %, machine-controller-userdata-%, $(OS)) BASE64_ENC = \ $(shell if base64 -w0 <(echo "") &> /dev/null; then echo "base64 -w0"; else echo "base64 -b0"; fi) @@ -43,13 +42,7 @@ BASE64_ENC = \ all: build-machine-controller webhook .PHONY: build-machine-controller -build-machine-controller: machine-controller $(USERDATA_BIN) - -machine-controller-userdata-%: cmd/userdata/% $(shell find cmd/userdata/$* pkg -name '*.go') - GOOS=$(GOOS) go build -v \ - $(LDFLAGS) \ - -o $@ \ - github.com/kubermatic/machine-controller/cmd/userdata/$* +build-machine-controller: machine-controller %: cmd/% $(shell find cmd/$* pkg -name '*.go') GOOS=$(GOOS) go build -v \ @@ -60,8 +53,7 @@ machine-controller-userdata-%: cmd/userdata/% $(shell find cmd/userdata/$* pkg - .PHONY: clean clean: rm -f machine-controller \ - webhook \ - $(USERDATA_BIN) + webhook .PHONY: lint lint: diff --git a/README.md b/README.md index dc87a214b..0803f7d4d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Kubermatic machine-controller -**Important Note: User data plugins for machine-controller are deprecated and will soon be removed. [Operating System Manager](https://github.com/kubermatic/operating-system-manager) is the successor of user data plugins. It's responsible for creating and managing the required configurations for worker nodes in a Kubernetes cluster with better modularity and extensibility. Please refer to [Operating System Manager][8] for more details.** +**Important Note: User data plugins for machine-controller have been removed. [Operating System Manager](https://github.com/kubermatic/operating-system-manager) is the successor of user data plugins. It's responsible for creating and managing the required configurations for worker nodes in a Kubernetes cluster with better modularity and extensibility. Please refer to [Operating System Manager][8] for more details.** ## Table of Contents diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 0931c26f8..cf15f855a 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -136,9 +136,6 @@ type controllerRunOptions struct { node machinecontroller.NodeSettings - // Enable external bootstrap management by consuming secrets that are used to configure an instance's user-data. - useExternalBootstrap bool - // A port range to reserve for services with NodePort visibility. nodePortRange string @@ -188,7 +185,7 @@ func main() { flag.StringVar(&nodePortRange, "node-port-range", "30000-32767", "A port range to reserve for services with NodePort visibility") flag.StringVar(&nodeRegistryCredentialsSecret, "node-registry-credentials-secret", "", "A Secret object reference, that contains auth info for image registry in namespace/secret-name form, example: kube-system/registry-credentials. See doc at https://github.com/kubermaric/machine-controller/blob/main/docs/registry-authentication.md") flag.BoolVar(&useOSM, "use-osm", false, "DEPRECATED: use osm controller for node bootstrap [use use-external-bootstrap instead]") - flag.BoolVar(&useExternalBootstrap, "use-external-bootstrap", false, "use an external bootstrap provider for instance user-data (e.g. operating-system-manager, also known as OSM)") + flag.BoolVar(&useExternalBootstrap, "use-external-bootstrap", true, "DEPRECATED: This flag is no-op and will have no effect since machine-controller only supports external bootstrap mechanism. This flag is only kept for backwards compatibility and will be removed in the future") flag.StringVar(&overrideBootstrapKubeletAPIServer, "override-bootstrap-kubelet-apiserver", "", "Override for the API server address used in worker nodes bootstrap-kubelet.conf") flag.Parse() @@ -296,7 +293,6 @@ func main() { RegistryCredentialsSecretRef: nodeRegistryCredentialsSecret, ContainerRuntime: containerRuntimeConfig, }, - useExternalBootstrap: useExternalBootstrap || useOSM, nodePortRange: nodePortRange, overrideBootstrapKubeletAPIServer: overrideBootstrapKubeletAPIServer, } @@ -437,7 +433,6 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { bs.opt.bootstrapTokenServiceAccountName, bs.opt.skipEvictionAfter, bs.opt.node, - bs.opt.useExternalBootstrap, bs.opt.nodePortRange, bs.opt.overrideBootstrapKubeletAPIServer, ); err != nil { diff --git a/cmd/userdata/amzn2/main.go b/cmd/userdata/amzn2/main.go deleted file mode 100644 index 2c317143b..000000000 --- a/cmd/userdata/amzn2/main.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2021 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for Amazon Linux 2. -// - -package main - -import ( - "flag" - "log" - - "go.uber.org/zap" - - machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" - "github.com/kubermatic/machine-controller/pkg/userdata/amzn2" - userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" -) - -func main() { - // Parse flags. - var debug bool - flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging") - - logFlags := machinecontrollerlog.NewDefaultOptions() - logFlags.AddFlags(flag.CommandLine) - - flag.Parse() - - if err := logFlags.Validate(); err != nil { - log.Fatalf("Invalid options: %v", err) - } - - rawLog := machinecontrollerlog.New(logFlags.Debug, logFlags.Format) - log := rawLog.Sugar() - - // Instantiate provider and start plugin. - var provider = &amzn2.Provider{} - var p = userdataplugin.New(provider, debug) - - if err := p.Run(log); err != nil { - log.Fatalw("Failed to run Amazon Linux 2 plugin", zap.Error(err)) - } -} diff --git a/cmd/userdata/centos/main.go b/cmd/userdata/centos/main.go deleted file mode 100644 index 3369abaac..000000000 --- a/cmd/userdata/centos/main.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for CentOS. -// - -package main - -import ( - "flag" - "log" - - "go.uber.org/zap" - - machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" - "github.com/kubermatic/machine-controller/pkg/userdata/centos" - userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" -) - -func main() { - // Parse flags. - var debug bool - flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging") - - logFlags := machinecontrollerlog.NewDefaultOptions() - logFlags.AddFlags(flag.CommandLine) - - flag.Parse() - - if err := logFlags.Validate(); err != nil { - log.Fatalf("Invalid options: %v", err) - } - - rawLog := machinecontrollerlog.New(logFlags.Debug, logFlags.Format) - log := rawLog.Sugar() - - // Instantiate provider and start plugin. - var provider = ¢os.Provider{} - var p = userdataplugin.New(provider, debug) - - if err := p.Run(log); err != nil { - log.Fatalw("Failed to run CentOS plugin", zap.Error(err)) - } -} diff --git a/cmd/userdata/flatcar/main.go b/cmd/userdata/flatcar/main.go deleted file mode 100644 index e1bb145a6..000000000 --- a/cmd/userdata/flatcar/main.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for flatcar. -// - -package main - -import ( - "flag" - "log" - - "go.uber.org/zap" - - machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" - "github.com/kubermatic/machine-controller/pkg/userdata/flatcar" - userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" -) - -func main() { - // Parse flags. - var debug bool - flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging") - - logFlags := machinecontrollerlog.NewDefaultOptions() - logFlags.AddFlags(flag.CommandLine) - - flag.Parse() - - if err := logFlags.Validate(); err != nil { - log.Fatalf("Invalid options: %v", err) - } - - rawLog := machinecontrollerlog.New(logFlags.Debug, logFlags.Format) - log := rawLog.Sugar() - - // Instantiate provider and start plugin. - var provider = &flatcar.Provider{} - var p = userdataplugin.New(provider, debug) - - if err := p.Run(log); err != nil { - log.Fatalw("Failed to run Flatcar plugin", zap.Error(err)) - } -} diff --git a/cmd/userdata/rhel/main.go b/cmd/userdata/rhel/main.go deleted file mode 100644 index aaae79699..000000000 --- a/cmd/userdata/rhel/main.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for RHEL. -// - -package main - -import ( - "flag" - "log" - - "go.uber.org/zap" - - machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" - userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" - "github.com/kubermatic/machine-controller/pkg/userdata/rhel" -) - -func main() { - // Parse flags. - var debug bool - flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging") - - logFlags := machinecontrollerlog.NewDefaultOptions() - logFlags.AddFlags(flag.CommandLine) - - flag.Parse() - - if err := logFlags.Validate(); err != nil { - log.Fatalf("Invalid options: %v", err) - } - - rawLog := machinecontrollerlog.New(logFlags.Debug, logFlags.Format) - log := rawLog.Sugar() - - // Instantiate provider and start plugin. - var provider = &rhel.Provider{} - var p = userdataplugin.New(provider, debug) - - if err := p.Run(log); err != nil { - log.Fatalw("Failed to run RHEL plugin", zap.Error(err)) - } -} diff --git a/cmd/userdata/rockylinux/main.go b/cmd/userdata/rockylinux/main.go deleted file mode 100644 index 5a81bea0e..000000000 --- a/cmd/userdata/rockylinux/main.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for RockyLinux. -// - -package main - -import ( - "flag" - "log" - - "go.uber.org/zap" - - machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" - userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" - "github.com/kubermatic/machine-controller/pkg/userdata/rockylinux" -) - -func main() { - // Parse flags. - var debug bool - flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging") - - logFlags := machinecontrollerlog.NewDefaultOptions() - logFlags.AddFlags(flag.CommandLine) - - flag.Parse() - - if err := logFlags.Validate(); err != nil { - log.Fatalf("Invalid options: %v", err) - } - - rawLog := machinecontrollerlog.New(logFlags.Debug, logFlags.Format) - log := rawLog.Sugar() - - // Instantiate provider and start plugin. - var provider = &rockylinux.Provider{} - var p = userdataplugin.New(provider, debug) - - if err := p.Run(log); err != nil { - log.Fatalw("Failed to run RockyLinux plugin", zap.Error(err)) - } -} diff --git a/cmd/userdata/ubuntu/main.go b/cmd/userdata/ubuntu/main.go deleted file mode 100644 index 67f1b4af0..000000000 --- a/cmd/userdata/ubuntu/main.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for Ubuntu. -// - -package main - -import ( - "flag" - "log" - - "go.uber.org/zap" - - machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" - userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" - "github.com/kubermatic/machine-controller/pkg/userdata/ubuntu" -) - -func main() { - // Parse flags. - var debug bool - flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging") - - logFlags := machinecontrollerlog.NewDefaultOptions() - logFlags.AddFlags(flag.CommandLine) - - flag.Parse() - - if err := logFlags.Validate(); err != nil { - log.Fatalf("Invalid options: %v", err) - } - - rawLog := machinecontrollerlog.New(logFlags.Debug, logFlags.Format) - log := rawLog.Sugar() - - // Instantiate provider and start plugin. - var provider = &ubuntu.Provider{} - var p = userdataplugin.New(provider, debug) - - if err := p.Run(log); err != nil { - log.Fatalw("Failed to run Ubuntu plugin", zap.Error(err)) - } -} diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 606e2664c..cc48082ae 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -28,7 +28,6 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" "github.com/kubermatic/machine-controller/pkg/node" - userdatamanager "github.com/kubermatic/machine-controller/pkg/userdata/manager" "k8s.io/client-go/tools/clientcmd" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -73,7 +72,7 @@ func main() { // OSM specific flags flag.BoolVar(&opt.useOSM, "use-osm", false, "DEPRECATED: osm controller is enabled for node bootstrap [use use-external-bootstrap instead]") - flag.BoolVar(&opt.useExternalBootstrap, "use-external-bootstrap", false, "user-data is provided by external bootstrap mechanism (e.g. operating-system-manager, also known as OSM)") + flag.BoolVar(&opt.useExternalBootstrap, "use-external-bootstrap", true, "DEPRECATED: This flag is no-op and will have no effect since machine-controller only supports external bootstrap mechanism. This flag is only kept for backwards compatibility and will be removed in the future") flag.Parse() @@ -129,21 +128,14 @@ func main() { } } - um, err := userdatamanager.New(log) - if err != nil { - log.Fatalw("Failed to initialise userdata plugins", zap.Error(err)) - } - srv, err := admission.Builder{ - ListenAddress: opt.admissionListenAddress, - Log: log, - Client: client, - WorkerClient: workerClient, - UserdataManager: um, - UseExternalBootstrap: opt.useExternalBootstrap || opt.useOSM, - NodeFlags: nodeFlags, - Namespace: opt.namespace, - VersionConstraints: constraint, + ListenAddress: opt.admissionListenAddress, + Log: log, + Client: client, + WorkerClient: workerClient, + NodeFlags: nodeFlags, + Namespace: opt.namespace, + VersionConstraints: constraint, // we could change this to get the CertDir from the configured CertName // and KeyName, but doing so does not bring us any benefits but would diff --git a/examples/machine-controller.yaml b/examples/machine-controller.yaml index 4ccb06b56..995dd1ac1 100644 --- a/examples/machine-controller.yaml +++ b/examples/machine-controller.yaml @@ -240,7 +240,6 @@ spec: - -cluster-dns=10.10.10.10 - -metrics-address=0.0.0.0:8080 - -health-probe-address=0.0.0.0:8085 - - -use-external-bootstrap=true # Machines that fail to join the cluster within this timeout and # are owned by a MachineSet will get deleted so the MachineSet # controller re-creates them @@ -284,7 +283,6 @@ spec: # on debug level, full Machine objects with inline credentials might be logged, beware! - -log-debug=false - -log-format=json # json or console - - -use-external-bootstrap=true - -namespace=kube-system - -listen-address=0.0.0.0:9876 volumeMounts: diff --git a/hack/ci/download-gocache.sh b/hack/ci/download-gocache.sh index d9a94d119..8b1244349 100755 --- a/hack/ci/download-gocache.sh +++ b/hack/ci/download-gocache.sh @@ -50,7 +50,7 @@ GIT_BRANCH="${PULL_BASE_REF:-}" CACHE_VERSION="${PULL_BASE_SHA:-}" # Periodics just use their head ref -if [[ -z "${CACHE_VERSION}" ]]; then +if [[ -z ${CACHE_VERSION} ]]; then CACHE_VERSION="$(git rev-parse HEAD)" GIT_BRANCH="main" fi diff --git a/hack/ci/run-e2e-tests.sh b/hack/ci/run-e2e-tests.sh index 2b235c06b..7c592da63 100755 --- a/hack/ci/run-e2e-tests.sh +++ b/hack/ci/run-e2e-tests.sh @@ -37,7 +37,6 @@ trap cleanup EXIT export GIT_HEAD_HASH="$(git rev-parse HEAD)" export MC_VERSION="${GIT_HEAD_HASH}" -export OPERATING_SYSTEM_MANAGER="${OPERATING_SYSTEM_MANAGER:-true}" TEST_NAME="Pre-warm Go build cache" echodate "Attempting to pre-warm Go build cache" @@ -51,13 +50,8 @@ echodate "Building machine-controller and webhook..." make all pushElapsed binary_build_duration_milliseconds $beforeBuild -# Copy userdata plugins. -echodate "Copying machine-controller plugins..." -cp machine-controller-userdata-* /usr/local/bin -ls -l /usr/local/bin - # Install genisoimage, this is required for generating user-data for vSphere -if [[ "${JOB_NAME:-}" = *"pull-machine-controller-e2e-vsphere"* ]]; then +if [[ ${JOB_NAME:-} == *"pull-machine-controller-e2e-vsphere"* ]]; then echo "Installing genisoimage..." apt install -y genisoimage fi diff --git a/hack/ci/setup-machine-controller-in-kind.sh b/hack/ci/setup-machine-controller-in-kind.sh index b1aea2db2..6aa09c2bf 100755 --- a/hack/ci/setup-machine-controller-in-kind.sh +++ b/hack/ci/setup-machine-controller-in-kind.sh @@ -22,7 +22,6 @@ if [ -z "${KIND_CLUSTER_NAME:-}" ]; then fi export MC_VERSION="${MC_VERSION:-$(git rev-parse HEAD)}" -export OPERATING_SYSTEM_MANAGER="${OPERATING_SYSTEM_MANAGER:-true}" OSM_REPO_URL="${OSM_REPO_URL:-https://github.com/kubermatic/operating-system-manager.git}" OSM_REPO_TAG="${OSM_REPO_TAG:-main}" @@ -51,7 +50,7 @@ echodate "Successfully built and loaded machine-controller image" if [ ! -f machine-controller-deployed ]; then # The 10 minute window given by default for the node to appear is too short # when we upgrade the instance during the upgrade test - if [[ ${LC_JOB_NAME:-} = "pull-machine-controller-e2e-ubuntu-upgrade" ]]; then + if [[ ${LC_JOB_NAME:-} == "pull-machine-controller-e2e-ubuntu-upgrade" ]]; then sed -i '/.*join-cluster-timeout=.*/d' examples/machine-controller.yaml fi sed -i -e 's/-worker-count=5/-worker-count=50/g' examples/machine-controller.yaml @@ -59,11 +58,6 @@ if [ ! -f machine-controller-deployed ]; then url="-override-bootstrap-kubelet-apiserver=$MASTER_URL" sed -i "s;-node-csr-approver=true;$url;g" examples/machine-controller.yaml - # Ensure that we update `use-external-bootstrap` flag if OSM is disabled - if [[ "$OPERATING_SYSTEM_MANAGER" == "false" ]]; then - sed -i "s;-use-external-bootstrap=true;-use-external-bootstrap=false;g" examples/machine-controller.yaml - fi - # e2e tests logs are primarily read by humans, if ever sed -i 's/log-format=json/log-format=console/g' examples/machine-controller.yaml @@ -73,32 +67,30 @@ if [ ! -f machine-controller-deployed ]; then protokol --kubeconfig "$KUBECONFIG" --flat --output "$ARTIFACTS/logs" --namespace kube-system 'machine-controller-*' > /dev/null 2>&1 & fi -if [[ "$OPERATING_SYSTEM_MANAGER" == "true" ]]; then - OSM_TMP_DIR=/tmp/osm - echodate "Clone OSM respository" - ( - # Clone OSM repo - mkdir -p $OSM_TMP_DIR - echodate "Cloning cluster exposer" - git clone --depth 1 --branch "${OSM_REPO_TAG}" "${OSM_REPO_URL}" $OSM_TMP_DIR - ) - - ( - OSM_TAG="$(git -C $OSM_TMP_DIR rev-parse HEAD)" - echodate "Installing operating-system-manager with image: $OSM_TAG" - - # In release branches we'll have this pinned to a specific semver instead of latest. - sed -i "s;:latest;:$OSM_TAG;g" examples/operating-system-manager.yaml - - # This is required for running e2e tests in KIND - url="-override-bootstrap-kubelet-apiserver=$MASTER_URL" - sed -i "s;-container-runtime=containerd;$url;g" examples/operating-system-manager.yaml - sed -i -e 's/-worker-count=5/-worker-count=50/g' examples/operating-system-manager.yaml - kubectl apply -f examples/operating-system-manager.yaml - ) - - protokol --kubeconfig "$KUBECONFIG" --flat --output "$ARTIFACTS/logs" --namespace kube-system 'operating-system-manager-*' > /dev/null 2>&1 & -fi +OSM_TMP_DIR=/tmp/osm +echodate "Clone OSM respository" +( + # Clone OSM repo + mkdir -p $OSM_TMP_DIR + echodate "Cloning cluster exposer" + git clone --depth 1 --branch "${OSM_REPO_TAG}" "${OSM_REPO_URL}" $OSM_TMP_DIR +) + +( + OSM_TAG="$(git -C $OSM_TMP_DIR rev-parse HEAD)" + echodate "Installing operating-system-manager with image: $OSM_TAG" + + # In release branches we'll have this pinned to a specific semver instead of latest. + sed -i "s;:latest;:$OSM_TAG;g" examples/operating-system-manager.yaml + + # This is required for running e2e tests in KIND + url="-override-bootstrap-kubelet-apiserver=$MASTER_URL" + sed -i "s;-container-runtime=containerd;$url;g" examples/operating-system-manager.yaml + sed -i -e 's/-worker-count=5/-worker-count=50/g' examples/operating-system-manager.yaml + kubectl apply -f examples/operating-system-manager.yaml +) + +protokol --kubeconfig "$KUBECONFIG" --flat --output "$ARTIFACTS/logs" --namespace kube-system 'operating-system-manager-*' > /dev/null 2>&1 & sleep 10 retry 10 check_all_deployments_ready kube-system diff --git a/hack/lib.sh b/hack/lib.sh index 5fc9131c4..1d9930c18 100644 --- a/hack/lib.sh +++ b/hack/lib.sh @@ -140,7 +140,7 @@ retry() { actual_retry $@ rc=$? set -e - elapsed_time=$(($(date +%s) - $start_time)) + elapsed_time=$(($(date +%s) - start_time)) write_junit "$rc" "$elapsed_time" return $rc } diff --git a/hack/run-machine-controller.sh b/hack/run-machine-controller.sh index fd8bb620c..a1d2f68e8 100755 --- a/hack/run-machine-controller.sh +++ b/hack/run-machine-controller.sh @@ -21,8 +21,6 @@ set -e MC_KUBECONFIG=${MC_KUBECONFIG:-$(dirname $0)/../.kubeconfig} # If you want to use the default kubeconfig `export MC_KUBECONFIG=$KUBECONFIG` -# `-use-external-bootstrap` flag can be removed to use legacy userdata that is generated by machine-controller. - make -C $(dirname $0)/.. build-machine-controller $(dirname $0)/../machine-controller \ -kubeconfig=$MC_KUBECONFIG \ @@ -31,5 +29,4 @@ $(dirname $0)/../machine-controller \ -cluster-dns=169.254.20.10 \ -enable-profiling \ -metrics-address=0.0.0.0:8080 \ - -health-probe-address=0.0.0.0:8085 \ - -use-external-bootstrap=true + -health-probe-address=0.0.0.0:8085 diff --git a/hack/verify-boilerplate.sh b/hack/verify-boilerplate.sh index 101dbdc31..f7d2ed3b0 100755 --- a/hack/verify-boilerplate.sh +++ b/hack/verify-boilerplate.sh @@ -21,5 +21,4 @@ cd $(dirname $0)/.. boilerplate \ -boilerplates hack/boilerplate \ -exclude pkg/machines/v1alpha1 \ - -exclude pkg/signals \ - -exclude pkg/userdata/scripts + -exclude pkg/signals diff --git a/image-builder/build.sh b/image-builder/build.sh index 40eb535e2..c8784a55c 100755 --- a/image-builder/build.sh +++ b/image-builder/build.sh @@ -22,56 +22,56 @@ K8S_RELEASE="" TARGET_OS="" usage() { - echo -e "usage:" - echo -e "\t$0 --target-os centos7|debian9|ubuntu-xenial|ubuntu-bionic [--release K8S-RELEASE]" + echo -e "usage:" + echo -e "\t$0 --target-os centos7|debian9|ubuntu-xenial|ubuntu-bionic [--release K8S-RELEASE]" } while [ $# -gt 0 ]; do - case "$1" in - --release) - K8S_RELEASE="$2" - shift - ;; - --target-os) - if [[ -z "$2" ]]; then - echo "You must specify target OS. Currently 'centos7' is supported." - exit 1 - fi - TARGET_OS="$2" - shift - ;; - *) - echo "Unknown parameter \"$1\"" - usage - exit 1 - ;; - esac - shift + case "$1" in + --release) + K8S_RELEASE="$2" + shift + ;; + --target-os) + if [[ -z $2 ]]; then + echo "You must specify target OS. Currently 'centos7' is supported." + exit 1 + fi + TARGET_OS="$2" + shift + ;; + *) + echo "Unknown parameter \"$1\"" + usage + exit 1 + ;; + esac + shift done -if [[ -z "$TARGET_OS" ]]; then - usage - exit 1 +if [[ -z $TARGET_OS ]]; then + usage + exit 1 fi if ! which guestmount &>/dev/null; then - echo "guestmount is not available. On Ubuntu, you need to install libguestfs-tools" - exit 1 + echo "guestmount is not available. On Ubuntu, you need to install libguestfs-tools" + exit 1 fi if ! which qemu-img &>/dev/null; then - echo "qemu-img is not available. On Ubuntu, you need to install qemu-utils" - exit 1 + echo "qemu-img is not available. On Ubuntu, you need to install qemu-utils" + exit 1 fi if ! which gpg2 &>/dev/null; then - echo "gpg2 is not available. On Ubuntu, you need to install gnupg2" - exit 1 + echo "gpg2 is not available. On Ubuntu, you need to install gnupg2" + exit 1 fi # if no K8S version has was specified on the command line, get the latest stable -if [[ -z "$K8S_RELEASE" ]]; then - K8S_RELEASE="$(curl -sSL https://dl.k8s.io/release/stable.txt)" +if [[ -z $K8S_RELEASE ]]; then + K8S_RELEASE="$(curl -sSL https://dl.k8s.io/release/stable.txt)" fi TEMPDIR="$(mktemp -d)" @@ -81,131 +81,133 @@ mkdir -p "$TARGETFS" "$SCRIPT_DIR/downloads" trap "sudo mountpoint --quiet $TARGETFS && sudo umount --recursive $TARGETFS; rm -rf $TEMPDIR" EXIT SIGINT get_centos7_image() { - CENTOS7_BUILD="1802" - echo " * Downloading vanilla CentOS image." - wget "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-$CENTOS7_BUILD.qcow2.xz" -P "$TEMPDIR" - - echo " * Verifying GPG signature" - wget --quiet "https://cloud.centos.org/centos/7/images/sha256sum.txt.asc" -O "$TEMPDIR/centos7-sha256sum.txt.asc" - gpg2 --quiet --import "$SCRIPT_DIR/RPM-GPG-KEY-CentOS-7" - gpg2 "$TEMPDIR/centos7-sha256sum.txt.asc" - - echo " * Verifying SHA256 digest" - EXPECTED_SHA256="$(grep "CentOS-7-x86_64-GenericCloud-$CENTOS7_BUILD.qcow2.xz$" < "$TEMPDIR/centos7-sha256sum.txt" | cut -f1 -d ' ')" - CALCULATED_SHA256="$(sha256sum "$TEMPDIR/CentOS-7-x86_64-GenericCloud-$CENTOS7_BUILD.qcow2.xz" | cut -f1 -d ' ')" - if [[ "$CALCULATED_SHA256" != "$EXPECTED_SHA256" ]]; then - echo " * SHA256 digest verification failed. '$CALCULATED_SHA256' != '$EXPECTED_SHA256'" - exit 1 - fi - - echo " * Decompressing" - unxz --keep "$TEMPDIR/CentOS-7-x86_64-GenericCloud-$CENTOS7_BUILD.qcow2.xz" - mv "$TEMPDIR/CentOS-7-x86_64-GenericCloud-$CENTOS7_BUILD.qcow2" "$SCRIPT_DIR/downloads/CentOS-7-x86_64-GenericCloud.qcow2" + CENTOS7_BUILD="1802" + echo " * Downloading vanilla CentOS image." + wget "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-$CENTOS7_BUILD.qcow2.xz" -P "$TEMPDIR" + + echo " * Verifying GPG signature" + wget --quiet "https://cloud.centos.org/centos/7/images/sha256sum.txt.asc" -O "$TEMPDIR/centos7-sha256sum.txt.asc" + gpg2 --quiet --import "$SCRIPT_DIR/RPM-GPG-KEY-CentOS-7" + gpg2 "$TEMPDIR/centos7-sha256sum.txt.asc" + + echo " * Verifying SHA256 digest" + EXPECTED_SHA256="$(grep "CentOS-7-x86_64-GenericCloud-$CENTOS7_BUILD.qcow2.xz$" <"$TEMPDIR/centos7-sha256sum.txt" | cut -f1 -d ' ')" + CALCULATED_SHA256="$(sha256sum "$TEMPDIR/CentOS-7-x86_64-GenericCloud-$CENTOS7_BUILD.qcow2.xz" | cut -f1 -d ' ')" + if [[ $CALCULATED_SHA256 != "$EXPECTED_SHA256" ]]; then + echo " * SHA256 digest verification failed. '$CALCULATED_SHA256' != '$EXPECTED_SHA256'" + exit 1 + fi + + echo " * Decompressing" + unxz --keep "$TEMPDIR/CentOS-7-x86_64-GenericCloud-$CENTOS7_BUILD.qcow2.xz" + mv "$TEMPDIR/CentOS-7-x86_64-GenericCloud-$CENTOS7_BUILD.qcow2" "$SCRIPT_DIR/downloads/CentOS-7-x86_64-GenericCloud.qcow2" } get_debian9_image() { - DEBIAN_CD_SIGNING_KEY_FINGERPRINT="DF9B9C49EAA9298432589D76DA87E80D6294BE9B" - - echo " * Downloading vanilla Debian image." - wget "https://cdimage.debian.org/cdimage/openstack/current-9/debian-9-openstack-amd64.qcow2" -P "$TEMPDIR" - - echo " * Verifying GPG signature" - wget --quiet "https://cdimage.debian.org/cdimage/openstack/current-9/SHA512SUMS" -O "$TEMPDIR/Debian-SHA512SUMS" - wget --quiet "https://cdimage.debian.org/cdimage/openstack/current-9/SHA512SUMS.sign" -O "$TEMPDIR/Debian-SHA512SUMS.sign" - gpg2 --quiet --recv-keys "$DEBIAN_CD_SIGNING_KEY_FINGERPRINT" - gpg2 --quiet --verify "$TEMPDIR/Debian-SHA512SUMS.sign" - - echo " * Verifying SHA512 digest" - EXPECTED_SHA512="$(grep 'debian-9-openstack-amd64.qcow2$' < "$TEMPDIR/Debian-SHA512SUMS" | cut -f1 -d ' ')" - CALCULATED_SHA512="$(sha512sum "$TEMPDIR/debian-9-openstack-amd64.qcow2" | cut -f1 -d ' ')" - if [[ "$CALCULATED_SHA512" != "$EXPECTED_SHA512" ]]; then - echo " * SHA512 digest verification failed. '$CALCULATED_SHA512' != '$EXPECTED_SHA512'" - exit 1 - fi - - echo " * Finalizing" - mv "$TEMPDIR/debian-9-openstack-amd64.qcow2" "$SCRIPT_DIR/downloads/debian-9-openstack-amd64.qcow2" + DEBIAN_CD_SIGNING_KEY_FINGERPRINT="DF9B9C49EAA9298432589D76DA87E80D6294BE9B" + + echo " * Downloading vanilla Debian image." + wget "https://cdimage.debian.org/cdimage/openstack/current-9/debian-9-openstack-amd64.qcow2" -P "$TEMPDIR" + + echo " * Verifying GPG signature" + wget --quiet "https://cdimage.debian.org/cdimage/openstack/current-9/SHA512SUMS" -O "$TEMPDIR/Debian-SHA512SUMS" + wget --quiet "https://cdimage.debian.org/cdimage/openstack/current-9/SHA512SUMS.sign" -O "$TEMPDIR/Debian-SHA512SUMS.sign" + gpg2 --quiet --recv-keys "$DEBIAN_CD_SIGNING_KEY_FINGERPRINT" + gpg2 --quiet --verify "$TEMPDIR/Debian-SHA512SUMS.sign" + + echo " * Verifying SHA512 digest" + EXPECTED_SHA512="$(grep 'debian-9-openstack-amd64.qcow2$' <"$TEMPDIR/Debian-SHA512SUMS" | cut -f1 -d ' ')" + CALCULATED_SHA512="$(sha512sum "$TEMPDIR/debian-9-openstack-amd64.qcow2" | cut -f1 -d ' ')" + if [[ $CALCULATED_SHA512 != "$EXPECTED_SHA512" ]]; then + echo " * SHA512 digest verification failed. '$CALCULATED_SHA512' != '$EXPECTED_SHA512'" + exit 1 + fi + + echo " * Finalizing" + mv "$TEMPDIR/debian-9-openstack-amd64.qcow2" "$SCRIPT_DIR/downloads/debian-9-openstack-amd64.qcow2" } get_ubuntu_image() { - local UBUNTU_CLOUD_IMAGE_SIGNING_KEY_FINGERPRINT="D2EB44626FDDC30B513D5BB71A5D6C4C7DB87C81" - local RELEASE="$1" - local IMG_NAME - if [[ $RELEASE == "xenial" ]]; then - IMG_NAME="$RELEASE-server-cloudimg-amd64-disk1.vmdk" - else - IMG_NAME="$RELEASE-server-cloudimg-amd64.vmdk" - fi - - echo " * Downloading vanilla Ubuntu image." - wget "https://cloud-images.ubuntu.com/$RELEASE/current/$IMG_NAME" -P "$TEMPDIR" - - echo " * Verifying GPG signature" - wget --quiet "https://cloud-images.ubuntu.com/$RELEASE/current/SHA256SUMS" -O "$TEMPDIR/Ubuntu-SHA256SUMS" - wget --quiet "https://cloud-images.ubuntu.com/$RELEASE/current/SHA256SUMS.gpg" -O "$TEMPDIR/Ubuntu-SHA256SUMS.gpg" - gpg2 --quiet --recv-keys $UBUNTU_CLOUD_IMAGE_SIGNING_KEY_FINGERPRINT - gpg2 --quiet --verify "$TEMPDIR/Ubuntu-SHA256SUMS.gpg" "$TEMPDIR/Ubuntu-SHA256SUMS" - - echo " * Verifying SHA256 digest" - EXPECTED_SHA256="$(grep "$IMG_NAME\$" < "$TEMPDIR/Ubuntu-SHA256SUMS" | cut -f1 -d ' ')" - CALCULATED_SHA256="$(sha256sum "$TEMPDIR/$IMG_NAME" | cut -f1 -d ' ')" - if [[ "$CALCULATED_SHA256" != "$EXPECTED_SHA256" ]]; then - echo " * SHA256 digest verification failed. '$CALCULATED_SHA256' != '$EXPECTED_SHA256'" - exit 1 - fi - - # This is needed because Ubuntu cloud images come in a Read-Only format - # that can only be used for linked-base VMs. - echo " * Converting to a read-write enabled image" - qemu-img convert -O vmdk "$TEMPDIR/$IMG_NAME" "$TEMPDIR/$IMG_NAME-rw" - - echo " * Finalizing" - mv "$TEMPDIR/$IMG_NAME-rw" "$SCRIPT_DIR/downloads/$RELEASE-server-cloudimg-amd64.vmdk" + local UBUNTU_CLOUD_IMAGE_SIGNING_KEY_FINGERPRINT="D2EB44626FDDC30B513D5BB71A5D6C4C7DB87C81" + local RELEASE="$1" + local IMG_NAME + if [[ $RELEASE == "xenial" ]]; then + IMG_NAME="$RELEASE-server-cloudimg-amd64-disk1.vmdk" + else + IMG_NAME="$RELEASE-server-cloudimg-amd64.vmdk" + fi + + echo " * Downloading vanilla Ubuntu image." + wget "https://cloud-images.ubuntu.com/$RELEASE/current/$IMG_NAME" -P "$TEMPDIR" + + echo " * Verifying GPG signature" + wget --quiet "https://cloud-images.ubuntu.com/$RELEASE/current/SHA256SUMS" -O "$TEMPDIR/Ubuntu-SHA256SUMS" + wget --quiet "https://cloud-images.ubuntu.com/$RELEASE/current/SHA256SUMS.gpg" -O "$TEMPDIR/Ubuntu-SHA256SUMS.gpg" + gpg2 --quiet --recv-keys $UBUNTU_CLOUD_IMAGE_SIGNING_KEY_FINGERPRINT + gpg2 --quiet --verify "$TEMPDIR/Ubuntu-SHA256SUMS.gpg" "$TEMPDIR/Ubuntu-SHA256SUMS" + + echo " * Verifying SHA256 digest" + EXPECTED_SHA256="$(grep "$IMG_NAME\$" <"$TEMPDIR/Ubuntu-SHA256SUMS" | cut -f1 -d ' ')" + CALCULATED_SHA256="$(sha256sum "$TEMPDIR/$IMG_NAME" | cut -f1 -d ' ')" + if [[ $CALCULATED_SHA256 != "$EXPECTED_SHA256" ]]; then + echo " * SHA256 digest verification failed. '$CALCULATED_SHA256' != '$EXPECTED_SHA256'" + exit 1 + fi + + # This is needed because Ubuntu cloud images come in a Read-Only format + # that can only be used for linked-base VMs. + echo " * Converting to a read-write enabled image" + qemu-img convert -O vmdk "$TEMPDIR/$IMG_NAME" "$TEMPDIR/$IMG_NAME-rw" + + echo " * Finalizing" + mv "$TEMPDIR/$IMG_NAME-rw" "$SCRIPT_DIR/downloads/$RELEASE-server-cloudimg-amd64.vmdk" } mount_rootfs() { - local IMAGE="$1" - local FOLDER="$2" - case $TARGET_OS in - debian9|centos7|ubuntu-*) - echo " * /" - sudo guestmount -a "$IMAGE" -m "/dev/sda1" "$TARGETFS" - ;; - *) - echo "mount_rootfs(): unknown OS \"$TARGET_OS\"" - usage - exit 1 - esac + local IMAGE="$1" + local FOLDER="$2" + case $TARGET_OS in + debian9 | centos7 | ubuntu-*) + echo " * /" + sudo guestmount -a "$IMAGE" -m "/dev/sda1" "$TARGETFS" + ;; + *) + echo "mount_rootfs(): unknown OS \"$TARGET_OS\"" + usage + exit 1 + ;; + esac } case $TARGET_OS in - centos7) - CLEAN_IMAGE="$SCRIPT_DIR/downloads/CentOS-7-x86_64-GenericCloud.qcow2" - if [[ ! -f "$CLEAN_IMAGE" ]]; then - get_centos7_image - fi - ;; - debian9) - CLEAN_IMAGE="$SCRIPT_DIR/downloads/debian-9-openstack-amd64.qcow2" - if [[ ! -f "$CLEAN_IMAGE" ]]; then - get_debian9_image - fi - ;; - ubuntu-xenial) - CLEAN_IMAGE="$SCRIPT_DIR/downloads/xenial-server-cloudimg-amd64.vmdk" - if [[ ! -f "$CLEAN_IMAGE" ]]; then - get_ubuntu_image xenial - fi - ;; - ubuntu-bionic) - CLEAN_IMAGE="$SCRIPT_DIR/downloads/bionic-server-cloudimg-amd64.vmdk" - if [[ ! -f "$CLEAN_IMAGE" ]]; then - get_ubuntu_image bionic - fi - ;; - *) - usage - exit 1 +centos7) + CLEAN_IMAGE="$SCRIPT_DIR/downloads/CentOS-7-x86_64-GenericCloud.qcow2" + if [[ ! -f $CLEAN_IMAGE ]]; then + get_centos7_image + fi + ;; +debian9) + CLEAN_IMAGE="$SCRIPT_DIR/downloads/debian-9-openstack-amd64.qcow2" + if [[ ! -f $CLEAN_IMAGE ]]; then + get_debian9_image + fi + ;; +ubuntu-xenial) + CLEAN_IMAGE="$SCRIPT_DIR/downloads/xenial-server-cloudimg-amd64.vmdk" + if [[ ! -f $CLEAN_IMAGE ]]; then + get_ubuntu_image xenial + fi + ;; +ubuntu-bionic) + CLEAN_IMAGE="$SCRIPT_DIR/downloads/bionic-server-cloudimg-amd64.vmdk" + if [[ ! -f $CLEAN_IMAGE ]]; then + get_ubuntu_image bionic + fi + ;; +*) + usage + exit 1 + ;; esac echo " * Verifying/Downloading kubernetes" @@ -224,11 +226,11 @@ sudo cp "$SCRIPT_DIR/downloads/kubelet-$K8S_RELEASE" "$TARGETFS/opt/bin/kubelet" echo " * Finalizing" sudo umount --recursive "$TARGETFS" EXTENSION="${CLEAN_IMAGE##*.}" -if [[ "$EXTENSION" == "vmdk" ]]; then - cp "$TEMPDIR/work-in-progress-image" "$SCRIPT_DIR/$TARGET_OS-output.vmdk" +if [[ $EXTENSION == "vmdk" ]]; then + cp "$TEMPDIR/work-in-progress-image" "$SCRIPT_DIR/$TARGET_OS-output.vmdk" else - echo " * Converting to VMDK" - qemu-img convert -O vmdk "$TEMPDIR/work-in-progress-image" "$SCRIPT_DIR/$TARGET_OS-output.vmdk" + echo " * Converting to VMDK" + qemu-img convert -O vmdk "$TEMPDIR/work-in-progress-image" "$SCRIPT_DIR/$TARGET_OS-output.vmdk" fi echo "$(realpath "$SCRIPT_DIR/$TARGET_OS-output.vmdk") ready." diff --git a/image-builder/download_kubernetes.sh b/image-builder/download_kubernetes.sh index 0b5cdd042..ddcd277ad 100755 --- a/image-builder/download_kubernetes.sh +++ b/image-builder/download_kubernetes.sh @@ -26,49 +26,49 @@ mkdir -p "$SCRIPT_DIR/downloads" K8S_RELEASE="" while [ $# -gt 0 ]; do - case "$1" in - --release) - K8S_RELEASE="$2" - shift - ;; - *) - echo "Unknown parameter \"$1\"" - exit 1 - ;; - esac - shift + case "$1" in + --release) + K8S_RELEASE="$2" + shift + ;; + *) + echo "Unknown parameter \"$1\"" + exit 1 + ;; + esac + shift done -if [[ -z "$K8S_RELEASE" ]]; then - K8S_RELEASE="$(curl -sSL https://dl.k8s.io/release/stable.txt)" - echo " * Latest stable version is $K8S_RELEASE" +if [[ -z $K8S_RELEASE ]]; then + K8S_RELEASE="$(curl -sSL https://dl.k8s.io/release/stable.txt)" + echo " * Latest stable version is $K8S_RELEASE" else - echo " * Using version $K8S_RELEASE" + echo " * Using version $K8S_RELEASE" fi wget --quiet https://dl.k8s.io/$K8S_RELEASE/bin/linux/amd64/{kubeadm,kubelet,kubectl}.sha1 -P "$TEMPDIR" for util in kubeadm kubelet kubectl; do - echo " * $util" - if [[ -x "$SCRIPT_DIR/downloads/$util-$K8S_RELEASE" ]]; then - CALCULATED_SHA1="$(sha1sum "$SCRIPT_DIR/downloads/$util-$K8S_RELEASE" | cut -f1 -d ' ')" - EXPECTED_SHA1="$(<"$TEMPDIR/$util.sha1")" - if [[ "$CALCULATED_SHA1" != "$EXPECTED_SHA1" ]]; then - echo " * SHA1 digest verification failed. $CALCULATED_SHA1 != $EXPECTED_SHA1" - echo " * The downloaded $util is either corrupted or out of date. Check your downloads and remove manually to continue." - exit 1 - fi - else - wget "https://dl.k8s.io/$K8S_RELEASE/bin/linux/amd64/$util" -P "$TEMPDIR" + echo " * $util" + if [[ -x "$SCRIPT_DIR/downloads/$util-$K8S_RELEASE" ]]; then + CALCULATED_SHA1="$(sha1sum "$SCRIPT_DIR/downloads/$util-$K8S_RELEASE" | cut -f1 -d ' ')" + EXPECTED_SHA1="$(<"$TEMPDIR/$util.sha1")" + if [[ $CALCULATED_SHA1 != "$EXPECTED_SHA1" ]]; then + echo " * SHA1 digest verification failed. $CALCULATED_SHA1 != $EXPECTED_SHA1" + echo " * The downloaded $util is either corrupted or out of date. Check your downloads and remove manually to continue." + exit 1 + fi + else + wget "https://dl.k8s.io/$K8S_RELEASE/bin/linux/amd64/$util" -P "$TEMPDIR" - CALCULATED_SHA1="$(sha1sum "$TEMPDIR/$util" | cut -f1 -d ' ')" - EXPECTED_SHA1="$(<"$TEMPDIR/$util.sha1")" - if [[ "$CALCULATED_SHA1" != "$EXPECTED_SHA1" ]]; then - echo " * SHA1 digest verification failed. $CALCULATED_SHA1 != $EXPECTED_SHA1. Download failed." - exit 1 - fi + CALCULATED_SHA1="$(sha1sum "$TEMPDIR/$util" | cut -f1 -d ' ')" + EXPECTED_SHA1="$(<"$TEMPDIR/$util.sha1")" + if [[ $CALCULATED_SHA1 != "$EXPECTED_SHA1" ]]; then + echo " * SHA1 digest verification failed. $CALCULATED_SHA1 != $EXPECTED_SHA1. Download failed." + exit 1 + fi - mv "$TEMPDIR/$util" "$SCRIPT_DIR/downloads/$util-$K8S_RELEASE" - chmod +x "$SCRIPT_DIR/downloads/$util-$K8S_RELEASE" - fi + mv "$TEMPDIR/$util" "$SCRIPT_DIR/downloads/$util-$K8S_RELEASE" + chmod +x "$SCRIPT_DIR/downloads/$util-$K8S_RELEASE" + fi done diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go index b10d76871..0203c2a96 100644 --- a/pkg/admission/admission.go +++ b/pkg/admission/admission.go @@ -33,7 +33,6 @@ import ( machinecontroller "github.com/kubermatic/machine-controller/pkg/controller/machine" "github.com/kubermatic/machine-controller/pkg/node" - userdatamanager "github.com/kubermatic/machine-controller/pkg/userdata/manager" admissionv1 "k8s.io/api/admission/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -45,28 +44,24 @@ import ( ) type admissionData struct { - log *zap.SugaredLogger - client ctrlruntimeclient.Client - workerClient ctrlruntimeclient.Client - userDataManager *userdatamanager.Manager - nodeSettings machinecontroller.NodeSettings - useExternalBootstrap bool - namespace string - constraints *semver.Constraints + log *zap.SugaredLogger + client ctrlruntimeclient.Client + workerClient ctrlruntimeclient.Client + nodeSettings machinecontroller.NodeSettings + namespace string + constraints *semver.Constraints } var jsonPatch = admissionv1.PatchTypeJSONPatch type Builder struct { - ListenAddress string - Log *zap.SugaredLogger - Client ctrlruntimeclient.Client - WorkerClient ctrlruntimeclient.Client - UserdataManager *userdatamanager.Manager - UseExternalBootstrap bool - NodeFlags *node.Flags - Namespace string - VersionConstraints *semver.Constraints + ListenAddress string + Log *zap.SugaredLogger + Client ctrlruntimeclient.Client + WorkerClient ctrlruntimeclient.Client + NodeFlags *node.Flags + Namespace string + VersionConstraints *semver.Constraints CertDir string CertName string @@ -75,13 +70,11 @@ type Builder struct { func (build Builder) Build() (webhook.Server, error) { ad := &admissionData{ - log: build.Log, - client: build.Client, - workerClient: build.WorkerClient, - userDataManager: build.UserdataManager, - useExternalBootstrap: build.UseExternalBootstrap, - namespace: build.Namespace, - constraints: build.VersionConstraints, + log: build.Log, + client: build.Client, + workerClient: build.WorkerClient, + namespace: build.Namespace, + constraints: build.VersionConstraints, } if err := build.NodeFlags.UpdateNodeSettings(&ad.nodeSettings); err != nil { diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index 2ee451bcb..379fe0a6e 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -27,7 +27,6 @@ import ( "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider" - controllerutil "github.com/kubermatic/machine-controller/pkg/controller/util" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -113,13 +112,6 @@ func (ad *admissionData) mutateMachines(ctx context.Context, ar admissionv1.Admi machine.Labels = make(map[string]string) } - // Set LegacyMachineControllerUserDataLabel to false if external bootstrapping is expected for managing the machine configuration. - if ad.useExternalBootstrap { - machine.Labels[controllerutil.LegacyMachineControllerUserDataLabel] = "false" - } else { - machine.Labels[controllerutil.LegacyMachineControllerUserDataLabel] = "true" - } - return createAdmissionResponse(log, machineOriginal, &machine) } @@ -144,7 +136,7 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec } // Verify operating system. - if _, err := ad.userDataManager.ForOS(providerConfig.OperatingSystem); err != nil { + if err := providerConfig.OperatingSystem.Validate(); err != nil { return fmt.Errorf("failed to get OS '%s': %w", providerConfig.OperatingSystem, err) } @@ -174,9 +166,7 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec defaultedOperatingSystemSpec, err := providerconfig.DefaultOperatingSystemSpec( providerConfig.OperatingSystem, - providerConfig.CloudProvider, providerConfig.OperatingSystemSpec, - ad.useExternalBootstrap, ) if err != nil { return err diff --git a/pkg/apis/cluster/common/consts.go b/pkg/apis/cluster/common/consts.go index ca58aafca..34eeacacf 100644 --- a/pkg/apis/cluster/common/consts.go +++ b/pkg/apis/cluster/common/consts.go @@ -18,8 +18,6 @@ package common import ( "fmt" - "strconv" - "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -175,49 +173,6 @@ func SetKubeletFlags(metaobj metav1.Object, flags map[KubeletFlags]string) { metaobj.SetAnnotations(annts) } -func GetKubeletConfigs(annotations map[string]string) map[string]string { - configs := map[string]string{} - for name, value := range annotations { - if strings.HasPrefix(name, KubeletConfigAnnotationPrefixV1) { - nameConfigValue := strings.SplitN(name, "/", 2) - if len(nameConfigValue) != 2 { - continue - } - configs[nameConfigValue[1]] = value - } - } - return configs -} - -func GetKubeletFeatureGates(annotations map[string]string) map[string]bool { - result := map[string]bool{} - for name, value := range annotations { - if strings.HasPrefix(name, KubeletFeatureGatesAnnotationPrefixV1) { - nameGateValue := strings.SplitN(name, "/", 2) - if len(nameGateValue) != 2 { - continue - } - realBool, _ := strconv.ParseBool(value) - result[nameGateValue[1]] = realBool - } - } - return result -} - -func GetKubeletFlags(annotations map[string]string) map[KubeletFlags]string { - result := map[KubeletFlags]string{} - for name, value := range annotations { - if strings.HasPrefix(name, KubeletFlagsGroupAnnotationPrefixV1) { - nameFlagValue := strings.SplitN(name, "/", 2) - if len(nameFlagValue) != 2 { - continue - } - result[KubeletFlags(nameFlagValue[1])] = value - } - } - return result -} - const OperatingSystemLabelV1 = "v1.machine-controller.kubermatic.io/operating-system" func SetOSLabel(metaobj metav1.Object, osName string) { diff --git a/pkg/apis/plugin/plugin.go b/pkg/apis/plugin/plugin.go deleted file mode 100644 index 2ed8ea5fb..000000000 --- a/pkg/apis/plugin/plugin.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// Environment and serialisation types for UserData plugins. -// - -package plugin - -import ( - "net" - - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/containerruntime" - - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" -) - -const ( - // EnvUserDataRequest names the environment variable containing - // the user data request. - EnvUserDataRequest = "MACHINE_CONTROLLER_USER_DATA_REQUEST" - - // EnvPluginDir names the environment variable containing - // a user defined location of the plugins. - EnvPluginDir = "MACHINE_CONTROLLER_USERDATA_PLUGIN_DIR" -) - -// UserDataRequest requests user data with the given arguments. -type UserDataRequest struct { - MachineSpec clusterv1alpha1.MachineSpec - Kubeconfig *clientcmdapi.Config - CloudProviderName string - CloudConfig string - DNSIPs []net.IP - ExternalCloudProvider bool - HTTPProxy string - NoProxy string - PauseImage string - KubeletCloudProviderName string - KubeletFeatureGates map[string]bool - KubeletConfigs map[string]string - ContainerRuntime containerruntime.Config - NodePortRange string -} - -// UserDataResponse contains the responded user data. -type UserDataResponse struct { - UserData string - Err string -} - -// ErrorResponse contains a single responded error. -type ErrorResponse struct { - Err string -} diff --git a/pkg/controller/machine/controller.go b/pkg/controller/machine/controller.go index 0bf4b2013..ad85ace9e 100644 --- a/pkg/controller/machine/controller.go +++ b/pkg/controller/machine/controller.go @@ -22,7 +22,6 @@ import ( "fmt" "net" "sort" - "strconv" "strings" "time" @@ -34,7 +33,6 @@ import ( "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/apis/plugin" "github.com/kubermatic/machine-controller/pkg/bootstrap" "github.com/kubermatic/machine-controller/pkg/cloudprovider" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" @@ -49,8 +47,6 @@ import ( "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" "github.com/kubermatic/machine-controller/pkg/rhsm" - userdatamanager "github.com/kubermatic/machine-controller/pkg/userdata/manager" - userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" "github.com/kubermatic/machine-controller/pkg/userdata/rhel" corev1 "k8s.io/api/core/v1" @@ -114,7 +110,6 @@ type Reconciler struct { metrics *MetricsCollection kubeconfigProvider KubeconfigProvider providerData *cloudprovidertypes.ProviderData - userDataManager *userdatamanager.Manager joinClusterTimeout *time.Duration name string bootstrapTokenServiceAccountName *types.NamespacedName @@ -123,7 +118,6 @@ type Reconciler struct { redhatSubscriptionManager rhsm.RedHatSubscriptionManager satelliteSubscriptionManager rhsm.SatelliteSubscriptionManager - useExternalBootstrap bool nodePortRange string overrideBootstrapKubeletAPIServer string } @@ -189,7 +183,6 @@ func Add( bootstrapTokenServiceAccountName *types.NamespacedName, skipEvictionAfter time.Duration, nodeSettings NodeSettings, - useExternalBootstrap bool, nodePortRange string, overrideBootstrapKubeletAPIServer string, ) error { @@ -209,16 +202,9 @@ func Add( redhatSubscriptionManager: rhsm.NewRedHatSubscriptionManager(log), satelliteSubscriptionManager: rhsm.NewSatelliteSubscriptionManager(log), - useExternalBootstrap: useExternalBootstrap, nodePortRange: nodePortRange, overrideBootstrapKubeletAPIServer: overrideBootstrapKubeletAPIServer, } - m, err := userdatamanager.New(log) - if err != nil { - return fmt.Errorf("failed to create userdatamanager: %w", err) - } - reconciler.userDataManager = m - utilruntime.ErrorHandlers = append(utilruntime.ErrorHandlers, func(error) { reconciler.metrics.Errors.Add(1) }) @@ -446,15 +432,9 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, mach return r.deleteMachine(ctx, log, prov, providerConfig.CloudProvider, machine, skipEviction) } - // Step 3: Essentially creates an instance for the given machine. - userdataPlugin, err := r.userDataManager.ForOS(providerConfig.OperatingSystem) - if err != nil { - return nil, fmt.Errorf("failed to userdata provider for '%s': %w", providerConfig.OperatingSystem, err) - } - - // case 3.2: creates an instance if there is no node associated with the given machine + // case 3.1: creates an instance if there is no node associated with the given machine if machine.Status.NodeRef == nil { - return r.ensureInstanceExistsForMachine(ctx, log, prov, machine, userdataPlugin, providerConfig) + return r.ensureInstanceExistsForMachine(ctx, log, prov, machine, providerConfig) } node, err := r.getNodeByNodeRef(ctx, machine.Status.NodeRef) @@ -481,10 +461,10 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, mach if r.nodeSettings.ExternalCloudProvider { return r.handleNodeFailuresWithExternalCCM(ctx, log, prov, providerConfig, node, machine) } - return r.ensureInstanceExistsForMachine(ctx, log, prov, machine, userdataPlugin, providerConfig) + return r.ensureInstanceExistsForMachine(ctx, log, prov, machine, providerConfig) } - // case 3.3: if the node exists and both external and internal CCM are not available. Then set the provider-id for the node. + // case 3.2: if the node exists and both external and internal CCM are not available. Then set the provider-id for the node. inTree, err := providerconfigtypes.IntreeCloudProviderImplementationSupported(providerConfig.CloudProvider, machine.Spec.Versions.Kubelet) if err != nil { return nil, fmt.Errorf("failed to check if cloud provider %q has in-tree implementation: %w", providerConfig.CloudProvider, err) @@ -501,7 +481,7 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, mach r.recorder.Event(machine, corev1.EventTypeNormal, "ProviderIDUpdated", "Successfully updated providerID on node") nodeLog.Info("Added ProviderID to the node") } - // case 3.4: if the node exists make sure if it has labels and taints attached to it. + // case 3.3: if the node exists make sure if it has labels and taints attached to it. return nil, r.ensureNodeLabelsAnnotationsAndTaints(ctx, nodeLog, node, machine) } @@ -814,7 +794,6 @@ func (r *Reconciler) ensureInstanceExistsForMachine( log *zap.SugaredLogger, prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine, - userdataPlugin userdataplugin.Provider, providerConfig *providerconfigtypes.Config, ) (*reconcile.Result, error) { log.Debug("Requesting instance for machine from cloudprovider because no associated node with status ready found...") @@ -827,109 +806,32 @@ func (r *Reconciler) ensureInstanceExistsForMachine( if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { log.Debug("Validated machine spec") - var kubeconfig *clientcmdapi.Config - - // an external provider will take care of the bootstrap kubeconfig and token by itself. - if !r.useExternalBootstrap { - kubeconfig, err = r.createBootstrapKubeconfig(ctx, log, machine.Name) - if err != nil { - return nil, fmt.Errorf("failed to create bootstrap kubeconfig: %w", err) - } - } - - // grab kubelet featureGates from the annotations - kubeletFeatureGates := common.GetKubeletFeatureGates(machine.GetAnnotations()) - if len(kubeletFeatureGates) == 0 { - // fallback to command-line input - kubeletFeatureGates = r.nodeSettings.KubeletFeatureGates - } - - // grab kubelet general options from the annotations - kubeletFlags := common.GetKubeletFlags(machine.GetAnnotations()) - kubeletConfigs := common.GetKubeletConfigs(machine.GetAnnotations()) - - // look up for ExternalCloudProvider feature, with fallback to command-line input - externalCloudProvider := r.nodeSettings.ExternalCloudProvider - if val, ok := kubeletFlags[common.ExternalCloudProviderKubeletFlag]; ok { - externalCloudProvider, _ = strconv.ParseBool(val) - } - - cloudConfig, kubeletCloudProviderName, err := prov.GetCloudConfig(machine.Spec) - if err != nil { - return nil, fmt.Errorf("failed to render cloud config: %w", err) - } - - if providerConfig.CloudProvider == providerconfigtypes.CloudProviderVsphere && externalCloudProvider { - cloudConfig = "" - } - - registryCredentials, err := containerruntime.GetContainerdAuthConfig(ctx, r.client, r.nodeSettings.RegistryCredentialsSecretRef) + // Here we do stuff! + var userdata string + referencedMachineDeployment, machineDeploymentRevision, err := controllerutil.GetMachineDeploymentNameAndRevisionForMachine(ctx, machine, r.client) if err != nil { - return nil, fmt.Errorf("failed to get containerd auth config: %w", err) + return nil, fmt.Errorf("failed to find machine's MachineDployment: %w", err) } - crRuntime := r.nodeSettings.ContainerRuntime - crRuntime.RegistryCredentials = registryCredentials - - if val, ok := kubeletConfigs[common.ContainerLogMaxSizeKubeletConfig]; ok { - crRuntime.ContainerLogMaxSize = val + bootstrapSecretName := fmt.Sprintf(bootstrap.CloudConfigSecretNamePattern, + referencedMachineDeployment, + machine.Namespace, + bootstrap.BootstrapCloudConfig) + + bootstrapSecret := &corev1.Secret{} + if err := r.client.Get(ctx, + types.NamespacedName{Name: bootstrapSecretName, Namespace: util.CloudInitNamespace}, + bootstrapSecret); err != nil { + log.Errorw("cloud-init configuration: cloud config is not ready yet", "secret", bootstrap.BootstrapCloudConfig) + return &reconcile.Result{RequeueAfter: 3 * time.Second}, nil } - if val, ok := kubeletConfigs[common.ContainerLogMaxFilesKubeletConfig]; ok { - crRuntime.ContainerLogMaxFiles = val + bootstrapSecretRevision := bootstrapSecret.Annotations[bootstrap.MachineDeploymentRevision] + if bootstrapSecretRevision != machineDeploymentRevision { + return nil, fmt.Errorf("cloud-init configuration: cloud config %q is not ready yet", bootstrap.BootstrapCloudConfig) } - // Here we do stuff! - var userdata string - - if r.useExternalBootstrap { - referencedMachineDeployment, machineDeploymentRevision, err := controllerutil.GetMachineDeploymentNameAndRevisionForMachine(ctx, machine, r.client) - if err != nil { - return nil, fmt.Errorf("failed to find machine's MachineDployment: %w", err) - } - - bootstrapSecretName := fmt.Sprintf(bootstrap.CloudConfigSecretNamePattern, - referencedMachineDeployment, - machine.Namespace, - bootstrap.BootstrapCloudConfig) - - bootstrapSecret := &corev1.Secret{} - if err := r.client.Get(ctx, - types.NamespacedName{Name: bootstrapSecretName, Namespace: util.CloudInitNamespace}, - bootstrapSecret); err != nil { - log.Errorw("cloud-init configuration: cloud config is not ready yet", "secret", bootstrap.BootstrapCloudConfig) - return &reconcile.Result{RequeueAfter: 3 * time.Second}, nil - } - - bootstrapSecretRevision := bootstrapSecret.Annotations[bootstrap.MachineDeploymentRevision] - if bootstrapSecretRevision != machineDeploymentRevision { - return nil, fmt.Errorf("cloud-init configuration: cloud config %q is not ready yet", bootstrap.BootstrapCloudConfig) - } - - userdata = getOSMBootstrapUserdata(machine.Spec.Name, *bootstrapSecret) - } else { - req := plugin.UserDataRequest{ - MachineSpec: machine.Spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: string(providerConfig.CloudProvider), - ExternalCloudProvider: externalCloudProvider, - DNSIPs: r.nodeSettings.ClusterDNSIPs, - PauseImage: r.nodeSettings.PauseImage, - KubeletCloudProviderName: kubeletCloudProviderName, - KubeletFeatureGates: kubeletFeatureGates, - KubeletConfigs: kubeletConfigs, - NoProxy: r.nodeSettings.NoProxy, - HTTPProxy: r.nodeSettings.HTTPProxy, - ContainerRuntime: crRuntime, - NodePortRange: r.nodePortRange, - } - - userdata, err = userdataPlugin.UserData(log, req) - if err != nil { - return nil, fmt.Errorf("failed get userdata: %w", err) - } - } + userdata = getOSMBootstrapUserdata(machine.Spec.Name, *bootstrapSecret) // Create the instance if _, err = r.createProviderInstance(ctx, log, prov, machine, userdata); err != nil { diff --git a/pkg/controller/machine/kubeconfig.go b/pkg/controller/machine/kubeconfig.go deleted file mode 100644 index 1ae436607..000000000 --- a/pkg/controller/machine/kubeconfig.go +++ /dev/null @@ -1,256 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "time" - - "go.uber.org/zap" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/selection" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/rand" - "k8s.io/client-go/kubernetes/scheme" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - secretTypeBootstrapToken corev1.SecretType = "bootstrap.kubernetes.io/token" - machineNameLabelKey string = "machine.k8s.io/machine.name" - tokenIDKey string = "token-id" - tokenSecretKey string = "token-secret" - expirationKey string = "expiration" - tokenFormatter string = "%s.%s" - // Keep this short, userdata is limited. - contextIdentifier string = "c" -) - -func (r *Reconciler) createBootstrapKubeconfig(ctx context.Context, log *zap.SugaredLogger, name string) (*clientcmdapi.Config, error) { - var token string - var err error - - if r.bootstrapTokenServiceAccountName != nil { - token, err = r.getTokenFromServiceAccount(ctx, *r.bootstrapTokenServiceAccountName) - if err != nil { - return nil, fmt.Errorf("failed to get token from ServiceAccount %s/%s: %w", r.bootstrapTokenServiceAccountName.Namespace, r.bootstrapTokenServiceAccountName.Name, err) - } - } else { - token, err = r.createBootstrapToken(ctx, name) - if err != nil { - return nil, fmt.Errorf("failed to create bootstrap token: %w", err) - } - } - - infoKubeconfig, err := r.kubeconfigProvider.GetKubeconfig(ctx, log) - if err != nil { - return nil, err - } - - outConfig := infoKubeconfig.DeepCopy() - - // Some consumers expect a valid `Contexts` map and the serialization - // for the Context ignores empty string fields, hence we must make sure - // both the Cluster and the User have a non-empty key. - clusterContextName := "" - // This is supposed to have a length of 1. We have code further down the - // line that extracts the CA cert and errors out if that is not the case, - // so we can simply iterate over it here. - for key := range infoKubeconfig.Clusters { - clusterContextName = key - } - cluster := outConfig.Clusters[clusterContextName].DeepCopy() - delete(outConfig.Clusters, clusterContextName) - outConfig.Clusters[contextIdentifier] = cluster - - outConfig.AuthInfos = map[string]*clientcmdapi.AuthInfo{ - contextIdentifier: { - Token: token, - }, - } - - // This is supposed to have a length of 1. We have code further down the - // line that extracts the CA cert and errors out if that is not the case. - // - // This handles a very special case in which we want to override the API server - // address that will be used in the `bootstrap-kubelet.conf` in the worker nodes for - // our E2E tests that run in KIND clusters. - if r.overrideBootstrapKubeletAPIServer != "" { - for key := range outConfig.Clusters { - outConfig.Clusters[key].Server = r.overrideBootstrapKubeletAPIServer - } - } - - outConfig.Contexts = map[string]*clientcmdapi.Context{contextIdentifier: {Cluster: contextIdentifier, AuthInfo: contextIdentifier}} - outConfig.CurrentContext = contextIdentifier - - return outConfig, nil -} - -func (r *Reconciler) getTokenFromServiceAccount(ctx context.Context, name types.NamespacedName) (string, error) { - sa := &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: name.Name, Namespace: name.Namespace}} - raw, err := r.getAsUnstructured(ctx, sa) - if err != nil { - return "", fmt.Errorf("failed to get serviceAccount %q: %w", name.String(), err) - } - sa = raw.(*corev1.ServiceAccount) - for _, serviceAccountSecretName := range sa.Secrets { - serviceAccountSecret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: sa.Namespace, Name: serviceAccountSecretName.Name}} - raw, err = r.getAsUnstructured(ctx, serviceAccountSecret) - if err != nil { - return "", fmt.Errorf("failed to get serviceAccountSecret: %w", err) - } - serviceAccountSecret = raw.(*corev1.Secret) - if serviceAccountSecret.Type != corev1.SecretTypeServiceAccountToken { - continue - } - return string(serviceAccountSecret.Data[corev1.ServiceAccountTokenKey]), nil - } - return "", errors.New("no serviceAccountSecret found") -} - -func (r *Reconciler) createBootstrapToken(ctx context.Context, name string) (string, error) { - existingSecret, err := r.getSecretIfExists(ctx, name) - if err != nil { - return "", err - } - if existingSecret != nil { - return r.updateSecretExpirationAndGetToken(ctx, existingSecret) - } - - tokenID := rand.String(6) - tokenSecret := rand.String(16) - - secret := corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("bootstrap-token-%s", tokenID), - Namespace: metav1.NamespaceSystem, - Labels: map[string]string{machineNameLabelKey: name}, - }, - Type: secretTypeBootstrapToken, - Data: map[string][]byte{ - "description": []byte("bootstrap token for " + name), - tokenIDKey: []byte(tokenID), - tokenSecretKey: []byte(tokenSecret), - expirationKey: []byte(metav1.Now().Add(1 * time.Hour).Format(time.RFC3339)), - "usage-bootstrap-authentication": []byte("true"), - "usage-bootstrap-signing": []byte("true"), - "auth-extra-groups": []byte("system:bootstrappers:machine-controller:default-node-token"), - }, - } - - if err := r.client.Create(ctx, &secret); err != nil { - return "", fmt.Errorf("failed to create bootstrap token secret: %w", err) - } - - return fmt.Sprintf(tokenFormatter, tokenID, tokenSecret), nil -} - -func (r *Reconciler) updateSecretExpirationAndGetToken(ctx context.Context, secret *corev1.Secret) (string, error) { - if secret.Data == nil { - secret.Data = map[string][]byte{} - } - tokenID := string(secret.Data[tokenIDKey]) - tokenSecret := string(secret.Data[tokenSecretKey]) - token := fmt.Sprintf(tokenFormatter, tokenID, tokenSecret) - - expirationTime, err := time.Parse(time.RFC3339, string(secret.Data[expirationKey])) - if err != nil { - return "", err - } - - // If the token is close to expire, reset it's expiration time - if time.Until(expirationTime).Minutes() < 30 { - secret.Data[expirationKey] = []byte(metav1.Now().Add(1 * time.Hour).Format(time.RFC3339)) - } else { - return token, nil - } - - if err := r.client.Update(ctx, secret); err != nil { - return "", fmt.Errorf("failed to update secret: %w", err) - } - return token, nil -} - -func (r *Reconciler) getSecretIfExists(ctx context.Context, name string) (*corev1.Secret, error) { - req, err := labels.NewRequirement(machineNameLabelKey, selection.Equals, []string{name}) - if err != nil { - return nil, err - } - selector := labels.NewSelector().Add(*req) - secrets := &corev1.SecretList{} - if err := r.client.List(ctx, secrets, - &ctrlruntimeclient.ListOptions{ - Namespace: metav1.NamespaceSystem, - LabelSelector: selector}); err != nil { - return nil, err - } - - if len(secrets.Items) == 0 { - return nil, nil - } - if len(secrets.Items) > 1 { - return nil, fmt.Errorf("expected to find exactly one secret for the given machine name =%s but found %d", name, len(secrets.Items)) - } - return &secrets.Items[0], nil -} - -// getAsUnstructured is a helper to get an object as unstrucuted.Unstructered from the client. -// The purpose of this is to avoid establishing a lister, which the cache-backed client automatically -// does. The object passed in must have name and namespace set. The returned object will -// be the same as the passed in one, if there was no error. -func (r *Reconciler) getAsUnstructured(ctx context.Context, obj runtime.Object) (runtime.Object, error) { - metaObj, ok := obj.(metav1.Object) - if !ok { - return nil, errors.New("can not assert object as metav1.Object") - } - kinds, _, err := scheme.Scheme.ObjectKinds(obj) - if err != nil { - return nil, fmt.Errorf("failed to get kinds for object: %w", err) - } - if len(kinds) == 0 { - return nil, fmt.Errorf("found no kind for object %t", obj) - } - apiVersion, kind := kinds[0].ToAPIVersionAndKind() - - target := &unstructured.Unstructured{} - target.SetAPIVersion(apiVersion) - target.SetKind(kind) - name := types.NamespacedName{Name: metaObj.GetName(), Namespace: metaObj.GetNamespace()} - - if err := r.client.Get(ctx, name, target); err != nil { - return nil, fmt.Errorf("failed to get object: %w", err) - } - - rawJSON, err := target.MarshalJSON() - if err != nil { - return nil, fmt.Errorf("failed to marshal unstructured.Unstructured: %w", err) - } - if err := json.Unmarshal(rawJSON, obj); err != nil { - return nil, fmt.Errorf("failed to marshal unstructured.Unstructued into %T: %w", obj, err) - } - return obj, nil -} diff --git a/pkg/controller/machine/kubeconfig_test.go b/pkg/controller/machine/kubeconfig_test.go deleted file mode 100644 index a98b702a1..000000000 --- a/pkg/controller/machine/kubeconfig_test.go +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controller - -import ( - "bytes" - "context" - "testing" - "time" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" - ctrlruntimefake "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func TestUpdateSecretExpirationAndGetToken(t *testing.T) { - tests := []struct { - initialExpirationTime time.Time - shouldRenew bool - }{ - { - initialExpirationTime: time.Now().Add(1 * time.Hour), - shouldRenew: false, - }, - { - initialExpirationTime: time.Now().Add(25 * time.Minute), - shouldRenew: true, - }, - { - initialExpirationTime: time.Now().Add(-25 * time.Minute), - shouldRenew: true, - }, - } - - reconciler := Reconciler{} - - for _, testCase := range tests { - ctx := context.Background() - secret := &corev1.Secret{} - secret.Name = "secret" - secret.Namespace = metav1.NamespaceSystem - data := map[string][]byte{} - data[tokenSecretKey] = []byte("tokenSecret") - data[tokenIDKey] = []byte("tokenID") - data[expirationKey] = []byte(testCase.initialExpirationTime.Format(time.RFC3339)) - secret.Data = data - reconciler.client = ctrlruntimefake. - NewClientBuilder(). - WithScheme(scheme.Scheme). - WithObjects(secret). - Build() - - if _, err := reconciler.updateSecretExpirationAndGetToken(ctx, secret); err != nil { - t.Fatalf("Unexpected error running updateSecretExpirationAndGetToken: %v", err) - } - - updatedSecret := &corev1.Secret{} - if err := reconciler.client.Get(ctx, types.NamespacedName{ - Namespace: metav1.NamespaceSystem, - Name: "secret", - }, updatedSecret); err != nil { - t.Fatalf("Unsexpected error getting secret: %v", err) - } - - if testCase.shouldRenew && - bytes.Equal(updatedSecret.Data[expirationKey], []byte(testCase.initialExpirationTime.Format(time.RFC3339))) { - t.Errorf("Error, token secret did not update but was expected to!") - } - - if !testCase.shouldRenew && - !bytes.Equal(updatedSecret.Data[expirationKey], []byte(testCase.initialExpirationTime.Format(time.RFC3339))) { - t.Errorf("Error, token secret was expected to get updated, but did not happen!") - } - - expirationTimeParsed, err := time.Parse(time.RFC3339, string(secret.Data[expirationKey])) - if err != nil { - t.Fatalf("Failed to parse timestamp from secret: %v", err) - } - - if time.Until(expirationTimeParsed).Minutes() < 0 { - t.Errorf("Error, secret expiration is in the past!") - } - } -} diff --git a/pkg/controller/util/machine.go b/pkg/controller/util/machine.go index 06afef450..a8e9b090a 100644 --- a/pkg/controller/util/machine.go +++ b/pkg/controller/util/machine.go @@ -26,9 +26,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// LegacyMachineControllerUserDataLabel is set to true when machine-controller is used for managing machine configuration. -const LegacyMachineControllerUserDataLabel = "machine.clusters.k8s.io/legacy-machine-controller-user-data" - func GetMachineDeploymentNameAndRevisionForMachine(ctx context.Context, machine *clusterv1alpha1.Machine, c client.Client) (string, string, error) { var ( machineSetName string diff --git a/pkg/providerconfig/types.go b/pkg/providerconfig/types.go index cf1520071..511773c81 100644 --- a/pkg/providerconfig/types.go +++ b/pkg/providerconfig/types.go @@ -181,9 +181,7 @@ func NewConfigVarResolver(ctx context.Context, client ctrlruntimeclient.Client) func DefaultOperatingSystemSpec( osys providerconfigtypes.OperatingSystem, - cloudProvider providerconfigtypes.CloudProvider, operatingSystemSpec runtime.RawExtension, - externalBootstrapEnabled bool, ) (runtime.RawExtension, error) { switch osys { case providerconfigtypes.OperatingSystemAmazonLinux2: @@ -191,7 +189,7 @@ func DefaultOperatingSystemSpec( case providerconfigtypes.OperatingSystemCentOS: return centos.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemFlatcar: - return flatcar.DefaultConfigForCloud(operatingSystemSpec, cloudProvider, externalBootstrapEnabled), nil + return flatcar.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemRHEL: return rhel.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemUbuntu: diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 6fb9e967d..72583bd86 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -45,6 +45,15 @@ const ( OperatingSystemRockyLinux OperatingSystem = "rockylinux" ) +func (os OperatingSystem) Validate() error { + for _, supportedOS := range AllOperatingSystems { + if os == supportedOS { + return nil + } + } + return ErrOSNotSupported +} + type CloudProvider string const ( diff --git a/pkg/providerconfig/types_test.go b/pkg/providerconfig/types_test.go index 13deb05ae..b9abc31ce 100644 --- a/pkg/providerconfig/types_test.go +++ b/pkg/providerconfig/types_test.go @@ -30,7 +30,7 @@ func TestDefaultOperatingSystemSpec(t *testing.T) { for _, osys := range providerconfigtypes.AllOperatingSystems { osys := osys t.Run(string(osys), func(t *testing.T) { - operatingSystemSpec, err := DefaultOperatingSystemSpec(osys, "", runtime.RawExtension{}, true) + operatingSystemSpec, err := DefaultOperatingSystemSpec(osys, runtime.RawExtension{}) if err != nil { t.Error("no error expected") diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go deleted file mode 100644 index e2b838328..000000000 --- a/pkg/userdata/amzn2/provider.go +++ /dev/null @@ -1,330 +0,0 @@ -/* -Copyright 2021 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for Amazon Linux 2. -// - -package amzn2 - -import ( - "errors" - "fmt" - "strings" - "text/template" - - "github.com/Masterminds/semver/v3" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - userdatahelper "github.com/kubermatic/machine-controller/pkg/userdata/helper" -) - -// Provider is a pkg/userdata/plugin.Provider implementation. -type Provider struct{} - -// UserData renders user-data template to string. -func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) (string, error) { - tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap(log)).Parse(userDataTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse user-data template: %w", err) - } - - kubeletVersion, err := semver.NewVersion(req.MachineSpec.Versions.Kubelet) - if err != nil { - return "", fmt.Errorf("invalid kubelet version: %w", err) - } - - pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) - if err != nil { - return "", fmt.Errorf("failed to get provider config: %w", err) - } - - if pconfig.OverwriteCloudConfig != nil { - req.CloudConfig = *pconfig.OverwriteCloudConfig - } - - if pconfig.Network.IsStaticIPConfig() { - return "", errors.New("static IP config is not supported with Amazon Linux 2") - } - - amznConfig, err := LoadConfig(pconfig.OperatingSystemSpec) - if err != nil { - return "", fmt.Errorf("failed to parse OperatingSystemSpec: %w", err) - } - - kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) - if err != nil { - return "", err - } - - kubernetesCACert, err := userdatahelper.GetCACert(req.Kubeconfig) - if err != nil { - return "", fmt.Errorf("error extracting cacert: %w", err) - } - - crEngine := req.ContainerRuntime.Engine() - crScript, err := crEngine.ScriptFor(providerconfigtypes.OperatingSystemAmazonLinux2) - if err != nil { - return "", fmt.Errorf("failed to generate container runtime install script: %w", err) - } - - crConfig, err := crEngine.Config() - if err != nil { - return "", fmt.Errorf("failed to generate container runtime config: %w", err) - } - - crAuthConfig, err := crEngine.AuthConfig() - if err != nil { - return "", fmt.Errorf("failed to generate container runtime auth config: %w", err) - } - - data := struct { - plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - OSConfig *Config - KubeletVersion string - Kubeconfig string - KubernetesCACert string - NodeIPScript string - ExtraKubeletFlags []string - ContainerRuntimeScript string - ContainerRuntimeConfigFileName string - ContainerRuntimeConfig string - ContainerRuntimeAuthConfigFileName string - ContainerRuntimeAuthConfig string - ContainerRuntimeName string - }{ - UserDataRequest: req, - ProviderSpec: pconfig, - OSConfig: amznConfig, - KubeletVersion: kubeletVersion.String(), - Kubeconfig: kubeconfigString, - KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(pconfig.Network.GetIPFamily()), - ExtraKubeletFlags: crEngine.KubeletFlags(), - ContainerRuntimeScript: crScript, - ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), - ContainerRuntimeConfig: crConfig, - ContainerRuntimeAuthConfigFileName: crEngine.AuthConfigFileName(), - ContainerRuntimeAuthConfig: crAuthConfig, - ContainerRuntimeName: crEngine.String(), - } - - buf := strings.Builder{} - if err = tmpl.Execute(&buf, data); err != nil { - return "", fmt.Errorf("failed to execute user-data template: %w", err) - } - - return userdatahelper.CleanupTemplateOutput(buf.String()) -} - -// UserData template. -const userDataTemplate = `#cloud-config -{{ if ne .CloudProviderName "aws" }} -hostname: {{ .MachineSpec.Name }} -{{- /* Never set the hostname on AWS nodes. Kubernetes(kube-proxy) requires the hostname to be the private dns name */}} -{{ end }} - -{{- if .OSConfig.DistUpgradeOnBoot }} -package_upgrade: true -package_reboot_if_required: true -{{- end }} - -ssh_pwauth: false - -{{- if ne (len .ProviderSpec.SSHPublicKeys) 0 }} -ssh_authorized_keys: -{{- range .ProviderSpec.SSHPublicKeys }} - - "{{ . }}" -{{- end }} -{{- end }} - -write_files: -{{- if .HTTPProxy }} -- path: "/etc/environment" - content: | -{{ proxyEnvironment .HTTPProxy .NoProxy | indent 4 }} -{{- end }} - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | -{{ journalDConfig | indent 4 }} - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | -{{ kernelModulesScript | indent 4 }} - -- path: "/etc/sysctl.d/k8s.conf" - content: | -{{ kernelSettings | indent 4 }} - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - -{{- /* As we added some modules and don't want to reboot, restart the service */}} - systemctl restart systemd-modules-load.service - sysctl --system - {{ if ne .CloudProviderName "aws" }} -{{- /* The normal way of setting it via cloud-init is broken, see */}} -{{- /* https://bugs.launchpad.net/cloud-init/+bug/1662542 */}} - hostnamectl set-hostname {{ .MachineSpec.Name }} - {{ end }} - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - {{- if or (eq .CloudProviderName "vsphere") (eq .CloudProviderName "vmware-cloud-director") }} - open-vm-tools \ - {{- end }} - ipvsadm - -{{ .ContainerRuntimeScript | indent 4 }} - -{{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - {{ if eq .CloudProviderName "vsphere" }} - systemctl enable --now vmtoolsd.service - {{ end -}} - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - -{{- if ne (len .CloudConfig) 0 }} -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | -{{ .CloudConfig | indent 4 }} -{{- end }} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | -{{ .NodeIPScript | indent 4 }} - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | -{{ .Kubeconfig | indent 4 }} - -- path: "/etc/kubernetes/kubelet.conf" - content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - -- path: "/etc/kubernetes/pki/ca.crt" - content: | -{{ .KubernetesCACert | indent 4 }} - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: {{ .ContainerRuntimeConfigFileName }} - permissions: "0644" - content: | -{{ .ContainerRuntimeConfig | indent 4 }} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | -{{ kubeletHealthCheckSystemdUnit | indent 4 }} - -{{- with .ProviderSpec.CAPublicKey }} - -- path: "/etc/ssh/trusted-user-ca-keys.pem" - content: | -{{ . | indent 4 }} - -- path: "/etc/ssh/sshd_config" - content: | -{{ sshConfigAddendum | indent 4 }} - append: true -{{- end }} - -runcmd: -- systemctl enable --now setup.service -` diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go deleted file mode 100644 index bd740e2ed..000000000 --- a/pkg/userdata/amzn2/provider_test.go +++ /dev/null @@ -1,278 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for Amazon Linux 2. -// - -package amzn2 - -import ( - "flag" - "net" - "testing" - - "go.uber.org/zap" - - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - "github.com/kubermatic/machine-controller/pkg/containerruntime" - testhelper "github.com/kubermatic/machine-controller/pkg/test" - "github.com/kubermatic/machine-controller/pkg/userdata/convert" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" -) - -var ( - update = flag.Bool("update", false, "update testdata files") - - pemCertificate = `-----BEGIN CERTIFICATE----- -MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV -BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG -A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 -DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 -NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG -cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv -c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B -AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS -R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT -ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk -JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 -mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW -caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G -A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt -hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB -MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES -MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv -bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h -U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao -eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 -UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD -58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n -sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF -kPe6XoSbiLm/kxk32T0= ------END CERTIFICATE-----` -) - -// fakeCloudConfigProvider simulates cloud config provider for test. -type fakeCloudConfigProvider struct { - config string - name string - err error -} - -func (p *fakeCloudConfigProvider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return p.config, p.name, p.err -} - -// userDataTestCase contains the data for a table-driven test. -type userDataTestCase struct { - name string - spec clusterv1alpha1.MachineSpec - clusterDNSIPs []net.IP - cloudProviderName *string - externalCloudProvider bool - httpProxy string - noProxy string - insecureRegistries string - registryMirrors string - pauseImage string - containerruntime string -} - -// TestUserDataGeneration runs the data generation for different -// environments. -func TestUserDataGeneration(t *testing.T) { - t.Parallel() - - tests := []userDataTestCase{ - { - name: "kubelet-v1.30.0-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - }, - { - name: "kubelet-v1.30.0-aws-external", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - externalCloudProvider: true, - }, - { - name: "kubelet-v1.30.0-vsphere", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - cloudProviderName: stringPtr("vsphere"), - }, - { - name: "kubelet-v1.30.0-vsphere-proxy", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128/", - noProxy: "192.168.1.0", - insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "kubelet-v1.30.0-vsphere-mirrors", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128/", - noProxy: "192.168.1.0", - registryMirrors: "https://registry.docker-cn.com/", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "kubelet-v1.28-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", - }, - }, - }, - { - name: "kubelet-v1.29-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.0", - }, - }, - }, - { - name: "kubelet-v1.27-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.27.0", - }, - }, - }, - } - - defaultCloudProvider := &fakeCloudConfigProvider{ - name: "aws", - config: "{aws-config:true}", - err: nil, - } - kubeconfig := &clientcmdapi.Config{ - Clusters: map[string]*clientcmdapi.Cluster{ - "": { - Server: "https://server/", - CertificateAuthorityData: []byte(pemCertificate), - }, - }, - AuthInfos: map[string]*clientcmdapi.AuthInfo{ - "": { - Token: "my-token", - }, - }, - } - provider := Provider{} - - kubeletFeatureGates := map[string]bool{ - "RotateKubeletServerCertificate": true, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - emptyProviderSpec := clusterv1alpha1.ProviderSpec{ - Value: &runtime.RawExtension{}, - } - test.spec.ProviderSpec = emptyProviderSpec - var cloudProvider *fakeCloudConfigProvider - if test.cloudProviderName != nil { - cloudProvider = &fakeCloudConfigProvider{ - name: *test.cloudProviderName, - config: "{config:true}", - err: nil, - } - } else { - cloudProvider = defaultCloudProvider - } - cloudConfig, cloudProviderName, err := cloudProvider.GetCloudConfig(test.spec) - if err != nil { - t.Fatalf("failed to get cloud config: %v", err) - } - - containerRuntimeOpts := containerruntime.Opts{ - ContainerRuntime: test.containerruntime, - InsecureRegistries: test.insecureRegistries, - RegistryMirrors: test.registryMirrors, - } - containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) - if err != nil { - t.Fatalf("failed to generate container runtime config: %v", err) - } - - req := plugin.UserDataRequest{ - MachineSpec: test.spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: cloudProviderName, - KubeletCloudProviderName: cloudProviderName, - DNSIPs: test.clusterDNSIPs, - ExternalCloudProvider: test.externalCloudProvider, - HTTPProxy: test.httpProxy, - NoProxy: test.noProxy, - PauseImage: test.pauseImage, - KubeletFeatureGates: kubeletFeatureGates, - ContainerRuntime: containerRuntimeConfig, - } - - s, err := provider.UserData(zap.NewNop().Sugar(), req) - if err != nil { - t.Errorf("error getting userdata: '%v'", err) - } - - // Check if we can gzip it. - if _, err := convert.GzipString(s); err != nil { - t.Fatal(err) - } - goldenName := test.name + ".yaml" - testhelper.CompareOutput(t, goldenName, s, *update) - }) - } -} - -// stringPtr returns pointer to given string. -func stringPtr(a string) *string { - return &a -} diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml deleted file mode 100644 index 074abdf66..000000000 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.27-aws.yaml +++ /dev/null @@ -1,456 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml deleted file mode 100644 index f0a23d9b3..000000000 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.28-aws.yaml +++ /dev/null @@ -1,456 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.29-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.29-aws.yaml deleted file mode 100644 index 92ba01380..000000000 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.29-aws.yaml +++ /dev/null @@ -1,454 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-aws-external.yaml deleted file mode 100644 index 3b2791812..000000000 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-aws-external.yaml +++ /dev/null @@ -1,456 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=${KUBELET_HOSTNAME} \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-aws.yaml deleted file mode 100644 index 4cfc6026e..000000000 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-aws.yaml +++ /dev/null @@ -1,454 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml deleted file mode 100644 index f5f678623..000000000 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml +++ /dev/null @@ -1,471 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false - -write_files: -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry.docker-cn.com/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere-proxy.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere-proxy.yaml deleted file mode 100644 index d63a72dc2..000000000 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere-proxy.yaml +++ /dev/null @@ -1,478 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false - -write_files: -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - [plugins."io.containerd.grpc.v1.cri".registry.configs] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000".tls] - insecure_skip_verify = true - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000".tls] - insecure_skip_verify = true - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere.yaml deleted file mode 100644 index 8cf7a2a31..000000000 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.30.0-vsphere.yaml +++ /dev/null @@ -1,462 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/provider.go b/pkg/userdata/centos/provider.go deleted file mode 100644 index 0f24ee2c8..000000000 --- a/pkg/userdata/centos/provider.go +++ /dev/null @@ -1,384 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for CentOS. -// - -package centos - -import ( - "errors" - "fmt" - "strings" - "text/template" - - "github.com/Masterminds/semver/v3" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - userdatahelper "github.com/kubermatic/machine-controller/pkg/userdata/helper" -) - -// Provider is a pkg/userdata/plugin.Provider implementation. -type Provider struct{} - -// UserData renders user-data template to string. -func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) (string, error) { - tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap(log)).Parse(userDataTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse user-data template: %w", err) - } - - kubeletVersion, err := semver.NewVersion(req.MachineSpec.Versions.Kubelet) - if err != nil { - return "", fmt.Errorf("invalid kubelet version: %w", err) - } - - pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) - if err != nil { - return "", fmt.Errorf("failed to get provider config: %w", err) - } - - if pconfig.OverwriteCloudConfig != nil { - req.CloudConfig = *pconfig.OverwriteCloudConfig - } - - if pconfig.Network.IsStaticIPConfig() { - return "", errors.New("static IP config is not supported with CentOS") - } - - centosConfig, err := LoadConfig(pconfig.OperatingSystemSpec) - if err != nil { - return "", fmt.Errorf("failed to parse OperatingSystemSpec: %w", err) - } - - kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) - if err != nil { - return "", err - } - - kubernetesCACert, err := userdatahelper.GetCACert(req.Kubeconfig) - if err != nil { - return "", fmt.Errorf("error extracting cacert: %w", err) - } - - crEngine := req.ContainerRuntime.Engine() - crScript, err := crEngine.ScriptFor(providerconfigtypes.OperatingSystemCentOS) - if err != nil { - return "", fmt.Errorf("failed to generate container runtime install script: %w", err) - } - - crConfig, err := crEngine.Config() - if err != nil { - return "", fmt.Errorf("failed to generate container runtime config: %w", err) - } - - crAuthConfig, err := crEngine.AuthConfig() - if err != nil { - return "", fmt.Errorf("failed to generate container runtime auth config: %w", err) - } - - data := struct { - plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - OSConfig *Config - KubeletVersion string - Kubeconfig string - KubernetesCACert string - NodeIPScript string - ExtraKubeletFlags []string - ContainerRuntimeScript string - ContainerRuntimeConfigFileName string - ContainerRuntimeConfig string - ContainerRuntimeAuthConfigFileName string - ContainerRuntimeAuthConfig string - ContainerRuntimeName string - }{ - UserDataRequest: req, - ProviderSpec: pconfig, - OSConfig: centosConfig, - KubeletVersion: kubeletVersion.String(), - Kubeconfig: kubeconfigString, - KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(pconfig.Network.GetIPFamily()), - ExtraKubeletFlags: crEngine.KubeletFlags(), - ContainerRuntimeScript: crScript, - ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), - ContainerRuntimeConfig: crConfig, - ContainerRuntimeAuthConfigFileName: crEngine.AuthConfigFileName(), - ContainerRuntimeAuthConfig: crAuthConfig, - ContainerRuntimeName: crEngine.String(), - } - - buf := strings.Builder{} - if err = tmpl.Execute(&buf, data); err != nil { - return "", fmt.Errorf("failed to execute user-data template: %w", err) - } - - return userdatahelper.CleanupTemplateOutput(buf.String()) -} - -// UserData template. -const userDataTemplate = `#cloud-config -{{ if ne .CloudProviderName "aws" }} -hostname: {{ .MachineSpec.Name }} -{{- /* Never set the hostname on AWS nodes. Kubernetes(kube-proxy) requires the hostname to be the private dns name */}} -{{ end }} - -{{- if .OSConfig.DistUpgradeOnBoot }} -package_upgrade: true -package_reboot_if_required: true -{{- end }} - -ssh_pwauth: false - -{{- if ne (len .ProviderSpec.SSHPublicKeys) 0 }} -ssh_authorized_keys: -{{- range .ProviderSpec.SSHPublicKeys }} - - "{{ . }}" -{{- end }} -{{- end }} - -write_files: -{{- if .HTTPProxy }} -- path: "/etc/environment" - content: | -{{ proxyEnvironment .HTTPProxy .NoProxy | indent 4 }} -{{- end }} - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | -{{ journalDConfig | indent 4 }} - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | -{{ kernelModulesScript | indent 4 }} - -- path: "/etc/sysctl.d/k8s.conf" - content: | -{{ kernelSettings | indent 4 }} - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - -{{- /* As we added some modules and don't want to reboot, restart the service */}} - systemctl restart systemd-modules-load.service - sysctl --system - - {{ if ne .CloudProviderName "aws" }} -{{- /* The normal way of setting it via cloud-init is broken, see */}} -{{- /* https://bugs.launchpad.net/cloud-init/+bug/1662542 */}} - hostnamectl set-hostname {{ .MachineSpec.Name }} - {{ end }} - -{{- /* CentOS 8 has reached EOL and all packages were moved to vault.centos.org -- https://www.centos.org/centos-linux-eol/ */}} - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - {{- if or (eq .CloudProviderName "vsphere") (eq .CloudProviderName "vmware-cloud-director") }} - open-vm-tools \ - {{- end }} - {{- if eq .CloudProviderName "nutanix" }} - iscsi-initiator-utils \ - {{- end }} - ipvsadm - - {{- /* iscsid service is required on Nutanix machines for CSI driver to attach volumes. */}} - {{- if eq .CloudProviderName "nutanix" }} - systemctl enable --now iscsid - {{ end }} -{{ .ContainerRuntimeScript | indent 4 }} - -{{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - {{ if eq .CloudProviderName "vsphere" }} - systemctl enable --now vmtoolsd.service - {{ end -}} - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - {{- if eq .CloudProviderName "kubevirt" }} - systemctl enable --now --no-block restart-kubelet.service - {{ end }} - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - -{{- if ne (len .CloudConfig) 0 }} -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | -{{ .CloudConfig | indent 4 }} -{{- end }} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | -{{ .NodeIPScript | indent 4 }} - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | -{{ .Kubeconfig | indent 4 }} - -- path: "/etc/kubernetes/kubelet.conf" - content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - -- path: "/etc/kubernetes/pki/ca.crt" - content: | -{{ .KubernetesCACert | indent 4 }} - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: {{ .ContainerRuntimeConfigFileName }} - permissions: "0644" - content: | -{{ .ContainerRuntimeConfig | indent 4 }} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | -{{ kubeletHealthCheckSystemdUnit | indent 4 }} - -{{- with .ProviderSpec.CAPublicKey }} - -- path: "/etc/ssh/trusted-user-ca-keys.pem" - content: | -{{ . | indent 4 }} - -- path: "/etc/ssh/sshd_config" - content: | -{{ sshConfigAddendum | indent 4 }} - append: true -{{- end }} - -{{- if eq .CloudProviderName "kubevirt" }} -- path: "/opt/bin/restart-kubelet.sh" - permissions: "0744" - content: | - #!/bin/bash - # Needed for Kubevirt provider because if the virt-launcher pod is deleted, - # the VM and DataVolume states are kept and VM is rebooted. We need to restart the kubelet - # with the new config (new IP) and run this at every boot. - set -xeuo pipefail - - # This helps us avoid an unnecessary restart for kubelet on the first boot - if [ -f /etc/kubelet_needs_restart ]; then - # restart kubelet since it's not the first boot - systemctl daemon-reload - systemctl restart kubelet.service - else - touch /etc/kubelet_needs_restart - fi - -- path: "/etc/systemd/system/restart-kubelet.service" - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - Description=Service responsible for restarting kubelet when the machine is rebooted - - [Service] - Type=oneshot - ExecStart=/opt/bin/restart-kubelet.sh - - [Install] - WantedBy=multi-user.target -{{- end }} - -runcmd: -- systemctl enable --now setup.service -` diff --git a/pkg/userdata/centos/provider_test.go b/pkg/userdata/centos/provider_test.go deleted file mode 100644 index 4b46652c9..000000000 --- a/pkg/userdata/centos/provider_test.go +++ /dev/null @@ -1,288 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for CentOS. -// - -package centos - -import ( - "flag" - "net" - "testing" - - "go.uber.org/zap" - - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - "github.com/kubermatic/machine-controller/pkg/containerruntime" - testhelper "github.com/kubermatic/machine-controller/pkg/test" - "github.com/kubermatic/machine-controller/pkg/userdata/convert" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" -) - -var ( - update = flag.Bool("update", false, "update testdata files") - - pemCertificate = `-----BEGIN CERTIFICATE----- -MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV -BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG -A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 -DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 -NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG -cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv -c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B -AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS -R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT -ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk -JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 -mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW -caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G -A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt -hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB -MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES -MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv -bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h -U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao -eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 -UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD -58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n -sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF -kPe6XoSbiLm/kxk32T0= ------END CERTIFICATE-----` -) - -// fakeCloudConfigProvider simulates cloud config provider for test. -type fakeCloudConfigProvider struct { - config string - name string - err error -} - -func (p *fakeCloudConfigProvider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return p.config, p.name, p.err -} - -// userDataTestCase contains the data for a table-driven test. -type userDataTestCase struct { - name string - spec clusterv1alpha1.MachineSpec - clusterDNSIPs []net.IP - cloudProviderName *string - externalCloudProvider bool - httpProxy string - noProxy string - insecureRegistries string - registryMirrors string - pauseImage string - containerruntime string -} - -// TestUserDataGeneration runs the data generation for different -// environments. -func TestUserDataGeneration(t *testing.T) { - t.Parallel() - - tests := []userDataTestCase{ - { - name: "kubelet-v1.30.0-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - }, - { - name: "kubelet-v1.30.0-nutanix", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - cloudProviderName: stringPtr("nutanix"), - }, - { - name: "kubelet-v1.30.0-aws-external", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - externalCloudProvider: true, - }, - { - name: "kubelet-v1.30.0-vsphere", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - cloudProviderName: stringPtr("vsphere"), - }, - { - name: "kubelet-v1.30.0-vsphere-proxy", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128/", - noProxy: "192.168.1.0", - insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "kubelet-v1.30.0-vsphere-mirrors", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128/", - noProxy: "192.168.1.0", - registryMirrors: "https://registry.docker-cn.com/", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "kubelet-v1.28-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", - }, - }, - }, - { - name: "kubelet-v1.29-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.0", - }, - }, - }, - { - name: "kubelet-v1.27-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.27.0", - }, - }, - }, - } - - defaultCloudProvider := &fakeCloudConfigProvider{ - name: "aws", - config: "{aws-config:true}", - err: nil, - } - kubeconfig := &clientcmdapi.Config{ - Clusters: map[string]*clientcmdapi.Cluster{ - "": { - Server: "https://server/", - CertificateAuthorityData: []byte(pemCertificate), - }, - }, - AuthInfos: map[string]*clientcmdapi.AuthInfo{ - "": { - Token: "my-token", - }, - }, - } - provider := Provider{} - - kubeletFeatureGates := map[string]bool{ - "RotateKubeletServerCertificate": true, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - emptyProviderSpec := clusterv1alpha1.ProviderSpec{ - Value: &runtime.RawExtension{}, - } - test.spec.ProviderSpec = emptyProviderSpec - var cloudProvider *fakeCloudConfigProvider - if test.cloudProviderName != nil { - cloudProvider = &fakeCloudConfigProvider{ - name: *test.cloudProviderName, - config: "{config:true}", - err: nil, - } - } else { - cloudProvider = defaultCloudProvider - } - cloudConfig, cloudProviderName, err := cloudProvider.GetCloudConfig(test.spec) - if err != nil { - t.Fatalf("failed to get cloud config: %v", err) - } - - containerRuntimeOpts := containerruntime.Opts{ - ContainerRuntime: test.containerruntime, - InsecureRegistries: test.insecureRegistries, - RegistryMirrors: test.registryMirrors, - } - containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) - if err != nil { - t.Fatalf("failed to generate container runtime config: %v", err) - } - - req := plugin.UserDataRequest{ - MachineSpec: test.spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: cloudProviderName, - KubeletCloudProviderName: cloudProviderName, - DNSIPs: test.clusterDNSIPs, - ExternalCloudProvider: test.externalCloudProvider, - HTTPProxy: test.httpProxy, - NoProxy: test.noProxy, - PauseImage: test.pauseImage, - KubeletFeatureGates: kubeletFeatureGates, - ContainerRuntime: containerRuntimeConfig, - } - - s, err := provider.UserData(zap.NewNop().Sugar(), req) - if err != nil { - t.Errorf("error getting userdata: '%v'", err) - } - - // Check if we can gzip it. - if _, err := convert.GzipString(s); err != nil { - t.Fatal(err) - } - goldenName := test.name + ".yaml" - testhelper.CompareOutput(t, goldenName, s, *update) - }) - } -} - -// stringPtr returns pointer to given string. -func stringPtr(a string) *string { - return &a -} diff --git a/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml deleted file mode 100644 index 89a34c232..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.27-aws.yaml +++ /dev/null @@ -1,462 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml deleted file mode 100644 index 3791281fe..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.28-aws.yaml +++ /dev/null @@ -1,462 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml deleted file mode 100644 index be17408c0..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.29-aws.yaml +++ /dev/null @@ -1,460 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.30.0-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.30.0-aws-external.yaml deleted file mode 100644 index cc06362a7..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.30.0-aws-external.yaml +++ /dev/null @@ -1,462 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=${KUBELET_HOSTNAME} \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.30.0-aws.yaml b/pkg/userdata/centos/testdata/kubelet-v1.30.0-aws.yaml deleted file mode 100644 index 8b7d461a9..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.30.0-aws.yaml +++ /dev/null @@ -1,460 +0,0 @@ -#cloud-config - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.30.0-nutanix.yaml b/pkg/userdata/centos/testdata/kubelet-v1.30.0-nutanix.yaml deleted file mode 100644 index 8b533f0aa..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.30.0-nutanix.yaml +++ /dev/null @@ -1,468 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - iscsi-initiator-utils \ - ipvsadm - systemctl enable --now iscsid - - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml b/pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml deleted file mode 100644 index 5c696b39b..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml +++ /dev/null @@ -1,477 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false - -write_files: -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry.docker-cn.com/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere-proxy.yaml b/pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere-proxy.yaml deleted file mode 100644 index f814bb806..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere-proxy.yaml +++ /dev/null @@ -1,484 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false - -write_files: -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - [plugins."io.containerd.grpc.v1.cri".registry.configs] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000".tls] - insecure_skip_verify = true - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000".tls] - insecure_skip_verify = true - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere.yaml b/pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere.yaml deleted file mode 100644 index 32dbe0b3a..000000000 --- a/pkg/userdata/centos/testdata/kubelet-v1.30.0-vsphere.yaml +++ /dev/null @@ -1,468 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/convert/ignition-converter.go b/pkg/userdata/convert/ignition-converter.go deleted file mode 100644 index 2d436b5e0..000000000 --- a/pkg/userdata/convert/ignition-converter.go +++ /dev/null @@ -1,44 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package convert - -import ( - "encoding/json" - "fmt" - - ctconfig "github.com/flatcar/container-linux-config-transpiler/config" -) - -func ToIgnition(s string) (string, error) { - // Convert to ignition - cfg, ast, report := ctconfig.Parse([]byte(s)) - if len(report.Entries) > 0 { - return "", fmt.Errorf("failed to validate coreos cloud config: %s", report.String()) - } - - ignCfg, report := ctconfig.Convert(cfg, "", ast) - if len(report.Entries) > 0 { - return "", fmt.Errorf("failed to convert container linux config to ignition: %s", report.String()) - } - - out, err := json.Marshal(ignCfg) - if err != nil { - return "", fmt.Errorf("failed to marshal ignition config: %w", err) - } - - return string(out), nil -} diff --git a/pkg/userdata/flatcar/flatcar.go b/pkg/userdata/flatcar/flatcar.go index 724223581..d55341677 100644 --- a/pkg/userdata/flatcar/flatcar.go +++ b/pkg/userdata/flatcar/flatcar.go @@ -19,8 +19,6 @@ package flatcar import ( "encoding/json" - "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - "k8s.io/apimachinery/pkg/runtime" ) @@ -46,21 +44,10 @@ type Config struct { func DefaultConfig(operatingSystemSpec runtime.RawExtension) runtime.RawExtension { // Webhook has already performed the defaulting at this point. So the value for // cloudProvider and operatingSystemManagerEnabled parameters are insignificant. - return DefaultConfigForCloud(operatingSystemSpec, "", true) -} - -func DefaultConfigForCloud(operatingSystemSpec runtime.RawExtension, cloudProvider types.CloudProvider, externalBootstrapEnabled bool) runtime.RawExtension { - // If userdata is being used from machine-controller and selected cloud provider is AWS then we - // force cloud-init. Because AWS has a very low cap for the maximum size of user-data. In case of ignition, - // we always exceed that limit which prevents new ec2 instances from being created. osSpec := Config{} if operatingSystemSpec.Raw != nil { _ = json.Unmarshal(operatingSystemSpec.Raw, &osSpec) } - // In case of OSM this is not required. - if cloudProvider == types.CloudProviderAWS && !externalBootstrapEnabled { - osSpec.ProvisioningUtility = CloudInit - } // Always default to ignition if no value was provided if osSpec.ProvisioningUtility == "" { diff --git a/pkg/userdata/flatcar/provider.go b/pkg/userdata/flatcar/provider.go deleted file mode 100644 index a8de7033c..000000000 --- a/pkg/userdata/flatcar/provider.go +++ /dev/null @@ -1,834 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for Flatcar. -// - -package flatcar - -import ( - "bytes" - "fmt" - "text/template" - - "github.com/Masterminds/semver/v3" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - "github.com/kubermatic/machine-controller/pkg/userdata/convert" - userdatahelper "github.com/kubermatic/machine-controller/pkg/userdata/helper" -) - -// Provider is a pkg/userdata/plugin.Provider implementation. -type Provider struct{} - -// UserData renders user-data template to string. -func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) (string, error) { - pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) - if err != nil { - return "", fmt.Errorf("failed to get provider config: %w", err) - } - - if pconfig.OverwriteCloudConfig != nil { - req.CloudConfig = *pconfig.OverwriteCloudConfig - } - - flatcarConfig, err := LoadConfig(pconfig.OperatingSystemSpec) - if err != nil { - return "", fmt.Errorf("failed to get flatcar config from provider config: %w", err) - } - - userDataTemplate, err := getUserDataTemplate(flatcarConfig.ProvisioningUtility) - if err != nil { - return "", fmt.Errorf("failed to get an appropriate user-data template: %w", err) - } - - tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap(log)).Parse(userDataTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse user-data template: %w", err) - } - - kubeletVersion, err := semver.NewVersion(req.MachineSpec.Versions.Kubelet) - if err != nil { - return "", fmt.Errorf("invalid kubelet version: %w", err) - } - - kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) - if err != nil { - return "", err - } - - kubernetesCACert, err := userdatahelper.GetCACert(req.Kubeconfig) - if err != nil { - return "", fmt.Errorf("error extracting cacert: %w", err) - } - - if flatcarConfig.DisableAutoUpdate { - flatcarConfig.DisableLocksmithD = true - flatcarConfig.DisableUpdateEngine = true - } - - crEngine := req.ContainerRuntime.Engine() - crScript, err := crEngine.ScriptFor(providerconfigtypes.OperatingSystemFlatcar) - if err != nil { - return "", fmt.Errorf("failed to generate container runtime install script: %w", err) - } - - crConfig, err := crEngine.Config() - if err != nil { - return "", fmt.Errorf("failed to generate container runtime config: %w", err) - } - - crAuthConfig, err := crEngine.AuthConfig() - if err != nil { - return "", fmt.Errorf("failed to generate container runtime auth config: %w", err) - } - - data := struct { - plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - FlatcarConfig *Config - KubeletVersion string - Kubeconfig string - KubernetesCACert string - NodeIPScript string - ExtraKubeletFlags []string - ContainerRuntimeScript string - ContainerRuntimeConfigFileName string - ContainerRuntimeConfig string - ContainerRuntimeAuthConfigFileName string - ContainerRuntimeAuthConfig string - ContainerRuntimeName string - }{ - UserDataRequest: req, - ProviderSpec: pconfig, - FlatcarConfig: flatcarConfig, - KubeletVersion: kubeletVersion.String(), - Kubeconfig: kubeconfigString, - KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(pconfig.Network.GetIPFamily()), - ExtraKubeletFlags: crEngine.KubeletFlags(), - ContainerRuntimeScript: crScript, - ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), - ContainerRuntimeConfig: crConfig, - ContainerRuntimeAuthConfigFileName: crEngine.AuthConfigFileName(), - ContainerRuntimeAuthConfig: crAuthConfig, - ContainerRuntimeName: crEngine.String(), - } - - b := &bytes.Buffer{} - err = tmpl.Execute(b, data) - if err != nil { - return "", fmt.Errorf("failed to execute user-data template: %w", err) - } - - out, err := userdatahelper.CleanupTemplateOutput(b.String()) - if err != nil { - return "", fmt.Errorf("failed to cleanup user-data template: %w", err) - } - - if flatcarConfig.ProvisioningUtility == CloudInit { - return out, nil - } - - return convert.ToIgnition(out) -} - -func getUserDataTemplate(pUtil ProvisioningUtility) (string, error) { - switch pUtil { - case Ignition, "": - return userDataIgnitionTemplate, nil - case CloudInit: - return userDataCloudInitTemplate, nil - default: - return "", fmt.Errorf("invalid provisioning utility %s, allowed values are %s or %s", - pUtil, Ignition, CloudInit) - } -} - -// Ignition template. -const userDataIgnitionTemplate = `passwd: -{{- if ne (len .ProviderSpec.SSHPublicKeys) 0 }} - users: - - name: core - ssh_authorized_keys: - {{range .ProviderSpec.SSHPublicKeys}}- {{.}} - {{end}} -{{- end }} - -{{- if .ProviderSpec.Network.IsStaticIPConfig }} -networkd: - units: - - name: static-nic.network - contents: | - [Match] - # Because of difficulty predicting specific NIC names on different cloud providers, - # we only support static addressing on VSphere. There should be a single NIC attached - # that we will match by name prefix 'en' which denotes ethernet devices. - Name=en* - - [Network] - DHCP=no - Address={{ .ProviderSpec.Network.CIDR }} - Gateway={{ .ProviderSpec.Network.Gateway }} - {{range .ProviderSpec.Network.DNS.Servers}}DNS={{.}} - {{end}} -{{- end }} - -systemd: - units: -{{- if .FlatcarConfig.DisableUpdateEngine }} - - name: update-engine.service - mask: true -{{- end }} -{{- if .FlatcarConfig.DisableLocksmithD }} - - name: locksmithd.service - mask: true -{{- end }} - -{{- if .HTTPProxy }} - - name: update-engine.service - dropins: - - name: 50-proxy.conf - contents: | - [Service] - Environment=ALL_PROXY={{ .HTTPProxy }} -{{- end }} - - - name: setup.service - enabled: true - contents: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - Requires=nodeip.service - After=network-online.target - After=nodeip.service - - Description=Service responsible for configuring the flatcar machine - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/setup.sh - - - name: download-script.service - enabled: true - contents: | - [Unit] - Requires=network-online.target - Requires=setup.service - After=network-online.target - After=setup.service - [Service] - Type=oneshot - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/download.sh - [Install] - WantedBy=multi-user.target - - - name: kubelet-healthcheck.service - enabled: true - dropins: - - name: 40-download.conf - contents: | - [Unit] - Requires=download-script.service - After=download-script.service - contents: | -{{ kubeletHealthCheckSystemdUnit | indent 10 }} - - - name: nodeip.service - enabled: true - contents: | - [Unit] - Description=Setup Kubelet Node IP Env - Requires=network-online.target - After=network-online.target - - [Service] - ExecStart=/opt/bin/setup_net_env.sh - RemainAfterExit=yes - Type=oneshot - [Install] - WantedBy=multi-user.target - -{{- if eq .CloudProviderName "kubevirt" }} - - name: restart-kubelet.service - enabled: true - contents: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - Description=Service responsible for restarting kubelet when the machine is rebooted - - [Service] - Type=oneshot - ExecStart=/opt/bin/restart-kubelet.sh - - [Install] - WantedBy=multi-user.target -{{- end }} - - - name: kubelet.service - enabled: true - dropins: - - name: 10-nodeip.conf - contents: | - [Service] - EnvironmentFile=/etc/kubernetes/nodeip.conf - - name: resolv.conf - contents: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - - name: 40-download.conf - contents: | - [Unit] - Requires=download-script.service - After=download-script.service - contents: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags false | indent 8 }} - -storage: - files: -{{- if .HTTPProxy }} - - path: /etc/environment - filesystem: root - mode: 0644 - contents: - inline: | -{{ proxyEnvironment .HTTPProxy .NoProxy | indent 10 }} -{{- end }} - - - path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - filesystem: root - mode: 0644 - contents: - inline: | -{{ journalDConfig | indent 10 }} - - - path: "/etc/kubernetes/kubelet.conf" - filesystem: root - mode: 0644 - contents: - inline: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 10 }} - - - path: /opt/load-kernel-modules.sh - filesystem: root - mode: 0755 - contents: - inline: | -{{ kernelModulesScript | indent 10 }} - - - path: /etc/sysctl.d/k8s.conf - filesystem: root - mode: 0644 - contents: - inline: | -{{ kernelSettings | indent 10 }} - - - path: /proc/sys/kernel/panic_on_oops - filesystem: root - mode: 0644 - contents: - inline: | - 1 - - - path: /proc/sys/kernel/panic - filesystem: root - mode: 0644 - contents: - inline: | - 10 - - - path: /proc/sys/vm/overcommit_memory - filesystem: root - mode: 0644 - contents: - inline: | - 1 - - - path: "/opt/bin/setup_net_env.sh" - filesystem: root - mode: 0755 - contents: - inline: | -{{ .NodeIPScript | indent 10 }} - - - path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" - filesystem: root - mode: 0755 - contents: - inline: | - [Network] - IPv6AcceptRA=true - - path: /etc/kubernetes/bootstrap-kubelet.conf - filesystem: root - mode: 0400 - contents: - inline: | -{{ .Kubeconfig | indent 10 }} - -{{- if ne (len .CloudConfig) 0 }} - - path: /etc/kubernetes/cloud-config - filesystem: root - mode: 0400 - contents: - inline: | -{{ .CloudConfig | indent 10 }} -{{- end }} - - - path: /etc/kubernetes/pki/ca.crt - filesystem: root - mode: 0644 - contents: - inline: | -{{ .KubernetesCACert | indent 10 }} -{{ if ne .CloudProviderName "aws" }} - - path: /etc/hostname - filesystem: root - mode: 0600 - contents: - inline: '{{ .MachineSpec.Name }}' -{{- end }} - -{{- if eq .CloudProviderName "kubevirt" }} - - path: /opt/bin/restart-kubelet.sh - filesystem: root - mode: 0744 - contents: - inline: | - #!/bin/bash - # Needed for Kubevirt provider because if the virt-launcher pod is deleted, - # the VM and DataVolume states are kept and VM is rebooted. We need to restart the kubelet - # with the new config (new IP) and run this at every boot. - set -xeuo pipefail - - # This helps us avoid an unnecessary restart for kubelet on the first boot - if [ -f /etc/kubelet_needs_restart ]; then - # restart kubelet since it's not the first boot - systemctl daemon-reload - systemctl restart kubelet.service - else - touch /etc/kubelet_needs_restart - fi -{{- end }} - - - path: /etc/ssh/sshd_config - filesystem: root - mode: 0600 - user: - id: 0 - group: - id: 0 - contents: - inline: | - # Use most defaults for sshd configuration. - Subsystem sftp internal-sftp - ClientAliveInterval 180 - UseDNS no - UsePAM yes - PrintLastLog no # handled by PAM - PrintMotd no # handled by PAM - PasswordAuthentication no - ChallengeResponseAuthentication no - -{{- if not .FlatcarConfig.DisableAutoUpdate }} - - path: "/etc/polkit-1/rules.d/60-noreboot_norestart.rules" - filesystem: root - mode: 0644 - contents: - inline: | - polkit.addRule(function(action, subject) { - if (action.id == "org.freedesktop.login1.reboot" || - action.id == "org.freedesktop.login1.reboot-multiple-sessions") { - if (subject.user == "core") { - return polkit.Result.YES; - } else { - return polkit.Result.AUTH_ADMIN; - } - } - }); -{{- end }} - - - path: /opt/bin/setup.sh - filesystem: root - mode: 0755 - contents: - inline: | - #!/bin/bash - set -xeuo pipefail - - # We stop these services here explicitly since masking only removes the symlinks for these services so that they can't be started. - # But that wouldn't "stop" the already running services on the first boot. - - {{- if or .FlatcarConfig.DisableUpdateEngine .FlatcarConfig.DisableAutoUpdate }} - systemctl stop update-engine.service - {{- end }} - - {{- if or .FlatcarConfig.DisableLocksmithD .FlatcarConfig.DisableAutoUpdate }} - systemctl stop locksmithd.service - {{- end }} - systemctl disable setup.service - - # Creates iscsi InitiatorName on Nutanix machines for CSI driver to attach volumes. - {{- if eq .CloudProviderName "nutanix" }} - systemctl start iscsi-init.service - systemctl enable --now iscsid.service - {{- end }} - - - path: /opt/bin/download.sh - filesystem: root - mode: 0755 - contents: - inline: | - #!/bin/bash - set -xeuo pipefail - -{{ safeDownloadBinariesScript .KubeletVersion | indent 10 }} - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - cat < /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: /etc/sysctl.d/k8s.conf - permissions: "0644" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" - permissions: "0755" - content: | - # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar - # so we enable IPv6 Router Advertisement here. - # See for details https://github.com/flatcar-linux/Flatcar/issues/384 - [Network] - IPv6AcceptRA=true - -- path: /etc/kubernetes/bootstrap-kubelet.conf - permissions: "0400" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: /etc/kubernetes/cloud-config - permissions: "0400" - content: | - {anexia-config:true} - -- path: /etc/kubernetes/pki/ca.crt - permissions: "0644" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - -- path: /etc/hostname - permissions: "0600" - content: 'node1' - -- path: /etc/ssh/sshd_config - permissions: "0600" - user: root - content: | - # Use most defaults for sshd configuration. - Subsystem sftp internal-sftp - ClientAliveInterval 180 - UseDNS no - UsePAM yes - PrintLastLog no # handled by PAM - PrintMotd no # handled by PAM - PasswordAuthentication no - ChallengeResponseAuthentication no - -- path: /opt/bin/download.sh - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.2.0}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") - cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.28.0}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat < /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: /etc/sysctl.d/k8s.conf - permissions: "0644" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" - permissions: "0755" - content: | - # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar - # so we enable IPv6 Router Advertisement here. - # See for details https://github.com/flatcar-linux/Flatcar/issues/384 - [Network] - IPv6AcceptRA=true - -- path: /etc/kubernetes/bootstrap-kubelet.conf - permissions: "0400" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: /etc/kubernetes/cloud-config - permissions: "0400" - content: | - {anexia-config:true} - -- path: /etc/kubernetes/pki/ca.crt - permissions: "0644" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - -- path: /etc/hostname - permissions: "0600" - content: 'node1' - -- path: /etc/ssh/sshd_config - permissions: "0600" - user: root - content: | - # Use most defaults for sshd configuration. - Subsystem sftp internal-sftp - ClientAliveInterval 180 - UseDNS no - UsePAM yes - PrintLastLog no # handled by PAM - PrintMotd no # handled by PAM - PasswordAuthentication no - ChallengeResponseAuthentication no - -- path: /opt/bin/download.sh - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.2.0}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") - cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.29.0}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat < /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: /etc/sysctl.d/k8s.conf - permissions: "0644" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" - permissions: "0755" - content: | - # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar - # so we enable IPv6 Router Advertisement here. - # See for details https://github.com/flatcar-linux/Flatcar/issues/384 - [Network] - IPv6AcceptRA=true - -- path: /etc/kubernetes/bootstrap-kubelet.conf - permissions: "0400" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: /etc/kubernetes/cloud-config - permissions: "0400" - content: | - {anexia-config:true} - -- path: /etc/kubernetes/pki/ca.crt - permissions: "0644" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - -- path: /etc/hostname - permissions: "0600" - content: 'node1' - -- path: /etc/ssh/sshd_config - permissions: "0600" - user: root - content: | - # Use most defaults for sshd configuration. - Subsystem sftp internal-sftp - ClientAliveInterval 180 - UseDNS no - UsePAM yes - PrintLastLog no # handled by PAM - PrintMotd no # handled by PAM - PasswordAuthentication no - ChallengeResponseAuthentication no - -- path: /opt/bin/download.sh - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.2.0}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") - cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat < /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: /etc/sysctl.d/k8s.conf - permissions: "0644" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/systemd/network/zz-default.network.d/ipv6-fix.conf" - permissions: "0755" - content: | - # IPv6 autoconfiguration doesn't work out of the box on some versions of Flatcar - # so we enable IPv6 Router Advertisement here. - # See for details https://github.com/flatcar-linux/Flatcar/issues/384 - [Network] - IPv6AcceptRA=true - -- path: /etc/kubernetes/bootstrap-kubelet.conf - permissions: "0400" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: /etc/kubernetes/cloud-config - permissions: "0400" - content: | - - -- path: /etc/kubernetes/pki/ca.crt - permissions: "0644" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - - -- path: /etc/hostname - permissions: "0600" - content: 'node1' - -- path: /etc/ssh/sshd_config - permissions: "0600" - user: root - content: | - # Use most defaults for sshd configuration. - Subsystem sftp internal-sftp - ClientAliveInterval 180 - UseDNS no - UsePAM yes - PrintLastLog no # handled by PAM - PrintMotd no # handled by PAM - PasswordAuthentication no - ChallengeResponseAuthentication no - -- path: /opt/bin/download.sh - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v1.2.0}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") - cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - mkdir -p /etc/systemd/system/containerd.service.d - - cat <"$kube_sum_file" - -for bin in kubelet kubeadm kubectl; do - {{- /* download kube binary */}} - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - - {{- /* download kube binary checksum */}} - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - - {{- /* save kube binary checksum */}} - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" -done - -{{- /* check kube binaries checksum */}} -sha256sum -c "$kube_sum_file" - -for bin in kubelet kubeadm kubectl; do - {{- /* link kube binaries from verioned dir to $opt_bin */}} - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin -done - -if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh -fi -` - - downloadBinariesTpl = `{{- /*setup some common directories */ -}} -mkdir -p /opt/bin/ -mkdir -p /var/lib/calico -mkdir -p /etc/kubernetes/manifests -mkdir -p /etc/cni/net.d -mkdir -p /opt/cni/bin - -{{- /* # cni */}} -if [ ! -f /opt/cni/bin/loopback ]; then - curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - -fi - -{{- if .DownloadKubelet }} -{{- /* kubelet */}} -if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v{{ .KubeletVersion }}/bin/linux/amd64/kubelet - chmod +x /opt/bin/kubelet -fi -{{- end }} - -if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh -fi -` -) - -// SafeDownloadBinariesScript returns the script which is responsible to -// download and check checksums of all required binaries. -func SafeDownloadBinariesScript(log *zap.SugaredLogger, kubeVersion string) (string, error) { - tmpl, err := template.New("download-binaries").Funcs(TxtFuncMap(log)).Parse(safeDownloadBinariesTpl) - if err != nil { - return "", fmt.Errorf("failed to parse download-binaries template: %w", err) - } - - const ( - CNIVersion = "v1.2.0" - CRIToolsVersion = "v1.27.0" - ) - - // force v in case if it's not there - if !strings.HasPrefix(kubeVersion, "v") { - kubeVersion = "v" + kubeVersion - } - - data := struct { - KubeVersion string - CNIVersion string - CRIToolsVersion string - }{ - KubeVersion: kubeVersion, - CNIVersion: CNIVersion, - CRIToolsVersion: CRIToolsVersion, - } - - b := &bytes.Buffer{} - err = tmpl.Execute(b, data) - if err != nil { - return "", fmt.Errorf("failed to execute download-binaries template: %w", err) - } - - return b.String(), nil -} - -// DownloadBinariesScript returns the script which is responsible to download -// all required binaries. -func DownloadBinariesScript(log *zap.SugaredLogger, kubeletVersion string, downloadKubelet bool) (string, error) { - tmpl, err := template.New("download-binaries").Funcs(TxtFuncMap(log)).Parse(downloadBinariesTpl) - if err != nil { - return "", fmt.Errorf("failed to parse download-binaries template: %w", err) - } - - data := struct { - KubeletVersion string - DownloadKubelet bool - }{ - KubeletVersion: kubeletVersion, - DownloadKubelet: downloadKubelet, - } - b := &bytes.Buffer{} - err = tmpl.Execute(b, data) - if err != nil { - return "", fmt.Errorf("failed to execute download-binaries template: %w", err) - } - - return b.String(), nil -} diff --git a/pkg/userdata/helper/download_binaries_script_test.go b/pkg/userdata/helper/download_binaries_script_test.go deleted file mode 100644 index c2f3795cd..000000000 --- a/pkg/userdata/helper/download_binaries_script_test.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helper - -import ( - "fmt" - "testing" - - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/test" -) - -const goldenExtension = ".golden" - -func TestDownloadBinariesScript(t *testing.T) { - for _, version := range versions { - name := fmt.Sprintf("download_binaries_%s", version.Original()) - t.Run(name, func(t *testing.T) { - script, err := DownloadBinariesScript(zap.NewNop().Sugar(), version.String(), true) - if err != nil { - t.Error(err) - } - goldenName := name + goldenExtension - test.CompareOutput(t, goldenName, script, *update) - }) - } -} - -func TestSafeDownloadBinariesScript(t *testing.T) { - name := "safe_download_binaries_v1.30.0" - t.Run(name, func(t *testing.T) { - script, err := SafeDownloadBinariesScript(zap.NewNop().Sugar(), "v1.30.0") - if err != nil { - t.Error(err) - } - goldenName := name + goldenExtension - test.CompareOutput(t, goldenName, script, *update) - }) -} diff --git a/pkg/userdata/helper/helper.go b/pkg/userdata/helper/helper.go deleted file mode 100644 index 59c8af94f..000000000 --- a/pkg/userdata/helper/helper.go +++ /dev/null @@ -1,216 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helper - -import ( - "encoding/json" - "fmt" - "strings" - - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" -) - -const ( - DefaultDockerContainerLogMaxFiles = "5" - DefaultDockerContainerLogMaxSize = "100m" -) - -func GetCACert(kubeconfig *clientcmdapi.Config) (string, error) { - if len(kubeconfig.Clusters) != 1 { - return "", fmt.Errorf("kubeconfig does not contain exactly one cluster, can not extract server address") - } - // Clusters is a map so we have to use range here. - for _, clusterConfig := range kubeconfig.Clusters { - return string(clusterConfig.CertificateAuthorityData), nil - } - - return "", fmt.Errorf("no CACert found") -} - -// StringifyKubeconfig marshals a kubeconfig to its text form. -func StringifyKubeconfig(kubeconfig *clientcmdapi.Config) (string, error) { - kubeconfigBytes, err := clientcmd.Write(*kubeconfig) - if err != nil { - return "", fmt.Errorf("error writing kubeconfig: %w", err) - } - - return string(kubeconfigBytes), nil -} - -// LoadKernelModules returns a script which is responsible for loading all required kernel modules -// The nf_conntrack_ipv4 module get removed in newer kernel versions. -func LoadKernelModulesScript() string { - return `#!/usr/bin/env bash -set -euo pipefail - -modprobe ip_vs -modprobe ip_vs_rr -modprobe ip_vs_wrr -modprobe ip_vs_sh - -if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 -else - modprobe nf_conntrack -fi -` -} - -// KernelSettings returns the list of kernel settings required for a kubernetes worker node -// inotify changes according to https://github.com/kubernetes/kubernetes/issues/10421 - better than letting the kubelet die. -func KernelSettings() string { - return `net.bridge.bridge-nf-call-ip6tables = 1 -net.bridge.bridge-nf-call-iptables = 1 -kernel.panic_on_oops = 1 -kernel.panic = 10 -net.ipv4.ip_forward = 1 -vm.overcommit_memory = 1 -fs.inotify.max_user_watches = 1048576 -fs.inotify.max_user_instances = 8192 -` -} - -// JournalDConfig returns the journal config preferable on every node. -func JournalDConfig() string { - // JournaldMaxUse defines the maximum space that journalD logs can occupy. - // https://www.freedesktop.org/software/systemd/man/journald.conf.html#SystemMaxUse= - return `[Journal] -SystemMaxUse=5G -` -} - -type dockerConfig struct { - ExecOpts []string `json:"exec-opts,omitempty"` - StorageDriver string `json:"storage-driver,omitempty"` - StorageOpts []string `json:"storage-opts,omitempty"` - LogDriver string `json:"log-driver,omitempty"` - LogOpts map[string]string `json:"log-opts,omitempty"` - InsecureRegistries []string `json:"insecure-registries,omitempty"` - RegistryMirrors []string `json:"registry-mirrors,omitempty"` -} - -// DockerConfig returns the docker daemon.json. -func DockerConfig(insecureRegistries, registryMirrors []string, logMaxFiles string, logMaxSize string) (string, error) { - if len(logMaxSize) > 0 { - // Parse log max size to ensure that it has the correct units - logMaxSize = strings.ToLower(logMaxSize) - logMaxSize = strings.ReplaceAll(logMaxSize, "ki", "k") - logMaxSize = strings.ReplaceAll(logMaxSize, "mi", "m") - logMaxSize = strings.ReplaceAll(logMaxSize, "gi", "g") - } else { - logMaxSize = DefaultDockerContainerLogMaxSize - } - - // Default if value is not provided - if len(logMaxFiles) == 0 { - logMaxFiles = DefaultDockerContainerLogMaxFiles - } - - cfg := dockerConfig{ - ExecOpts: []string{"native.cgroupdriver=systemd"}, - StorageDriver: "overlay2", - LogDriver: "json-file", - LogOpts: map[string]string{ - "max-size": logMaxSize, - "max-file": logMaxFiles, - }, - InsecureRegistries: insecureRegistries, - RegistryMirrors: registryMirrors, - } - - b, err := json.Marshal(cfg) - return string(b), err -} - -func ProxyEnvironment(proxy, noProxy string) string { - return fmt.Sprintf(`HTTP_PROXY=%s -http_proxy=%s -HTTPS_PROXY=%s -https_proxy=%s -NO_PROXY=%s -no_proxy=%s`, proxy, proxy, proxy, proxy, noProxy, noProxy) -} - -func SetupNodeIPEnvScript(ipFamily util.IPFamily) string { - const defaultIfcIPv4 = `DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+")` - - var defaultIfcIP string - switch ipFamily { - case util.IPFamilyIPv4: - defaultIfcIP = defaultIfcIPv4 - case util.IPFamilyIPv6: - defaultIfcIP = `DEFAULT_IFC_IP=$(ip -o -6 route get 1:: | grep -oP "src \K\S+")` - case util.IPFamilyIPv4IPv6: - defaultIfcIP = `DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") -DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") -if [ -z "${DEFAULT_IFC_IP6}" ] -then - echodate "Failed to get IPv6 address for the default route interface" - exit 1 -fi -DEFAULT_IFC_IP=$DEFAULT_IFC_IP,$DEFAULT_IFC_IP6` - case util.IPFamilyIPv6IPv4: - defaultIfcIP = `DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") -DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") -if [ -z "${DEFAULT_IFC_IP6}" ] -then - echodate "Failed to get IPv6 address for the default route interface" - exit 1 -fi -DEFAULT_IFC_IP=$DEFAULT_IFC_IP6,$DEFAULT_IFC_IP` - default: - defaultIfcIP = defaultIfcIPv4 - } - return `#!/usr/bin/env bash -echodate() { - echo "[$(date -Is)]" "$@" -} - -# get the default interface IP address -` + defaultIfcIP + ` - -# get the full hostname -FULL_HOSTNAME=$(hostname -f) - -if [ -z "${DEFAULT_IFC_IP}" ] -then - echodate "Failed to get IP address for the default route interface" - exit 1 -fi - -# write the nodeip_env file -# we need the line below because flatcar has the same string "coreos" in that file -if grep -q coreos /etc/os-release -then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf -elif [ ! -d /etc/systemd/system/kubelet.service.d ] -then - echodate "Can't find kubelet service extras directory" - exit 1 -else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf -fi - ` -} - -func SSHConfigAddendum() string { - return `TrustedUserCAKeys /etc/ssh/trusted-user-ca-keys.pem -CASignatureAlgorithms ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,ssh-ed25519,rsa-sha2-512,rsa-sha2-256,ssh-rsa` -} diff --git a/pkg/userdata/helper/kubelet.go b/pkg/userdata/helper/kubelet.go deleted file mode 100644 index 57bcbdf15..000000000 --- a/pkg/userdata/helper/kubelet.go +++ /dev/null @@ -1,379 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helper - -import ( - "fmt" - "net" - "strconv" - "strings" - "text/template" - - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - kubeletv1b1 "k8s.io/kubelet/config/v1beta1" - "k8s.io/utils/ptr" - kyaml "sigs.k8s.io/yaml" -) - -const ( - defaultKubeletContainerLogMaxSize = "100Mi" -) - -func kubeletFlagsTpl(withNodeIP bool) string { - flagsTemplate := `--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ ---kubeconfig=/var/lib/kubelet/kubeconfig \ ---config=/etc/kubernetes/kubelet.conf \ ---cert-dir=/etc/kubernetes/pki \` - - flagsTemplate += ` -{{- if .IsExternal }} ---cloud-provider=external \ -{{- /* In-tree cloud providers have been disabled starting from k8s 1.29. For more information: https://github.com/kubernetes/kubernetes/pull/117503 */}} -{{- else if and (.CloudProvider) (semverCompare "<1.29" .KubeletVersion) }} ---cloud-provider={{- .CloudProvider }} \ ---cloud-config=/etc/kubernetes/cloud-config \ -{{- end }}` - - flagsTemplate += `{{- if and (.Hostname) (ne .CloudProvider "aws") }} ---hostname-override={{ .Hostname }} \ -{{- else if and (eq .CloudProvider "aws") (.IsExternal) }} ---hostname-override=${KUBELET_HOSTNAME} \ -{{- end }} ---exit-on-lock-contention \ ---lock-file=/tmp/kubelet.lock \ -{{- if .PauseImage }} ---pod-infra-container-image={{ .PauseImage }} \ -{{- end }} -{{- if .InitialTaints }} ---register-with-taints={{- .InitialTaints }} \ -{{- end }} -{{- range .ExtraKubeletFlags }} -{{ . }} \ -{{- end }}` - - if withNodeIP { - flagsTemplate += ` ---node-ip ${KUBELET_NODE_IP}` - } - - return flagsTemplate -} - -const ( - kubeletSystemdUnitTpl = `[Unit] -After={{ .ContainerRuntime }}.service -Requires={{ .ContainerRuntime }}.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh -{{ if .DisableSwap }} -ExecStartPre=/bin/bash /opt/disable-swap.sh -{{ end }} -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ -{{ kubeletFlags .KubeletVersion .CloudProvider .Hostname .ClusterDNSIPs .IsExternal .IPFamily .PauseImage .InitialTaints .ExtraKubeletFlags | indent 2 }} - -[Install] -WantedBy=multi-user.target` - - containerRuntimeHealthCheckSystemdUnitTpl = `[Unit] -Requires={{ .ContainerRuntime }}.service -After={{ .ContainerRuntime }}.service - -[Service] -ExecStart=/opt/bin/health-monitor.sh container-runtime - -[Install] -WantedBy=multi-user.target` -) - -// List of allowed TLS cipher suites for kubelet. -var kubeletTLSCipherSuites = []string{ - // TLS 1.3 cipher suites - "TLS_AES_128_GCM_SHA256", - "TLS_AES_256_GCM_SHA384", - "TLS_CHACHA20_POLY1305_SHA256", - // TLS 1.0 - 1.2 cipher suites - "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", - "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305", - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256", - "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384", - "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305", -} - -func withNodeIPFlag(ipFamily util.IPFamily, cloudProvider string, external bool) bool { - // If external or in-tree CCM is in use we don't need to set --node-ip - // as the cloud provider will know what IPs to return. - if ipFamily.IsDualstack() { - if external || cloudProvider != "" { - return false - } - } - return true -} - -// KubeletSystemdUnit returns the systemd unit for the kubelet. -func KubeletSystemdUnit(log *zap.SugaredLogger, containerRuntime, kubeletVersion, cloudProvider, hostname string, dnsIPs []net.IP, external bool, ipFamily util.IPFamily, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string, disableSwap bool) (string, error) { - tmpl, err := template.New("kubelet-systemd-unit").Funcs(TxtFuncMap(log)).Parse(kubeletSystemdUnitTpl) - if err != nil { - return "", fmt.Errorf("failed to parse kubelet-systemd-unit template: %w", err) - } - - data := struct { - ContainerRuntime string - KubeletVersion string - CloudProvider string - Hostname string - ClusterDNSIPs []net.IP - IsExternal bool - IPFamily util.IPFamily - PauseImage string - InitialTaints []corev1.Taint - ExtraKubeletFlags []string - DisableSwap bool - }{ - ContainerRuntime: containerRuntime, - KubeletVersion: kubeletVersion, - CloudProvider: cloudProvider, - Hostname: hostname, - ClusterDNSIPs: dnsIPs, - IsExternal: external, - IPFamily: ipFamily, - PauseImage: pauseImage, - InitialTaints: initialTaints, - ExtraKubeletFlags: extraKubeletFlags, - DisableSwap: disableSwap, - } - - var buf strings.Builder - if err = tmpl.Execute(&buf, data); err != nil { - return "", fmt.Errorf("failed to execute kubelet-systemd-unit template: %w", err) - } - - return buf.String(), nil -} - -// kubeletConfiguration returns marshaled kubelet.config.k8s.io/v1beta1 KubeletConfiguration. -func kubeletConfiguration(log *zap.SugaredLogger, clusterDomain string, clusterDNS []net.IP, featureGates map[string]bool, kubeletConfigs map[string]string) (string, error) { - clusterDNSstr := make([]string, 0, len(clusterDNS)) - for _, ip := range clusterDNS { - clusterDNSstr = append(clusterDNSstr, ip.String()) - } - - cfg := kubeletv1b1.KubeletConfiguration{ - TypeMeta: metav1.TypeMeta{ - Kind: "KubeletConfiguration", - APIVersion: kubeletv1b1.SchemeGroupVersion.String(), - }, - Authentication: kubeletv1b1.KubeletAuthentication{ - X509: kubeletv1b1.KubeletX509Authentication{ - ClientCAFile: "/etc/kubernetes/pki/ca.crt", - }, - Webhook: kubeletv1b1.KubeletWebhookAuthentication{ - Enabled: ptr.To(true), - }, - Anonymous: kubeletv1b1.KubeletAnonymousAuthentication{ - Enabled: ptr.To(false), - }, - }, - Authorization: kubeletv1b1.KubeletAuthorization{ - Mode: kubeletv1b1.KubeletAuthorizationModeWebhook, - }, - CgroupDriver: "systemd", - ClusterDNS: clusterDNSstr, - ClusterDomain: clusterDomain, - FeatureGates: featureGates, - ProtectKernelDefaults: true, - ReadOnlyPort: 0, - RotateCertificates: true, - ServerTLSBootstrap: true, - StaticPodPath: "/etc/kubernetes/manifests", - KubeReserved: map[string]string{"cpu": "200m", "memory": "200Mi", "ephemeral-storage": "1Gi"}, - SystemReserved: map[string]string{"cpu": "200m", "memory": "200Mi", "ephemeral-storage": "1Gi"}, - EvictionHard: map[string]string{"memory.available": "100Mi", "nodefs.available": "10%", "nodefs.inodesFree": "5%", "imagefs.available": "15%"}, - VolumePluginDir: "/var/lib/kubelet/volumeplugins", - TLSCipherSuites: kubeletTLSCipherSuites, - ContainerLogMaxSize: defaultKubeletContainerLogMaxSize, - } - - if kubeReserved, ok := kubeletConfigs[common.KubeReservedKubeletConfig]; ok { - for _, krPair := range strings.Split(kubeReserved, ",") { - krKV := strings.SplitN(krPair, "=", 2) - if len(krKV) != 2 { - continue - } - cfg.KubeReserved[krKV[0]] = krKV[1] - } - } - - if systemReserved, ok := kubeletConfigs[common.SystemReservedKubeletConfig]; ok { - for _, srPair := range strings.Split(systemReserved, ",") { - srKV := strings.SplitN(srPair, "=", 2) - if len(srKV) != 2 { - continue - } - cfg.SystemReserved[srKV[0]] = srKV[1] - } - } - - if evictionHard, ok := kubeletConfigs[common.EvictionHardKubeletConfig]; ok { - for _, ehPair := range strings.Split(evictionHard, ",") { - ehKV := strings.SplitN(ehPair, "<", 2) - if len(ehKV) != 2 { - continue - } - cfg.EvictionHard[ehKV[0]] = ehKV[1] - } - } - - if maxPods, ok := kubeletConfigs[common.MaxPodsKubeletConfig]; ok { - mp, err := strconv.ParseInt(maxPods, 10, 32) - if err != nil { - // Instead of breaking the workflow, just print a warning and skip the configuration - log.Info("Skipping invalid MaxPods value for Kubelet configuration", "value", maxPods) - } else { - cfg.MaxPods = int32(mp) - } - } - - if containerLogMaxSize, ok := kubeletConfigs[common.ContainerLogMaxSizeKubeletConfig]; ok { - cfg.ContainerLogMaxSize = containerLogMaxSize - } - if containerLogMaxFiles, ok := kubeletConfigs[common.ContainerLogMaxFilesKubeletConfig]; ok { - maxFiles, err := strconv.Atoi(containerLogMaxFiles) - if err != nil || maxFiles < 0 { - // Instead of breaking the workflow, just print a warning and skip the configuration - log.Infow("Skipping invalid ContainerLogMaxSize value for Kubelet configuration", "value", containerLogMaxFiles) - } else { - cfg.ContainerLogMaxFiles = ptr.To(int32(maxFiles)) - } - } - - if enabled, ok := featureGates["SeccompDefault"]; ok && enabled { - cfg.SeccompDefault = ptr.To(true) - } - - buf, err := kyaml.Marshal(cfg) - return string(buf), err -} - -// KubeletFlags returns the kubelet flags. -// --node-ip and --cloud-provider kubelet flags conflict in the dualstack setup. -// In general, it is not expected to need to use --node-ip with external CCMs, -// as the cloud provider is expected to know the correct IPs to return. -// For details read kubernetes/sig-networking channel discussion -// https://kubernetes.slack.com/archives/C09QYUH5W/p1654003958331739 -func KubeletFlags(log *zap.SugaredLogger, version, cloudProvider, hostname string, dnsIPs []net.IP, external bool, ipFamily util.IPFamily, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string) (string, error) { - withNodeIPFlag := withNodeIPFlag(ipFamily, cloudProvider, external) - - tmpl, err := template.New("kubelet-flags").Funcs(TxtFuncMap(log)). - Parse(kubeletFlagsTpl(withNodeIPFlag)) - if err != nil { - return "", fmt.Errorf("failed to parse kubelet-flags template: %w", err) - } - - initialTaintsArgs := []string{} - for _, taint := range initialTaints { - initialTaintsArgs = append(initialTaintsArgs, fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect)) - } - - kubeletFlags := make([]string, len(extraKubeletFlags)) - copy(kubeletFlags, extraKubeletFlags) - - data := struct { - CloudProvider string - Hostname string - ClusterDNSIPs []net.IP - KubeletVersion string - IsExternal bool - IPFamily util.IPFamily - PauseImage string - InitialTaints string - ExtraKubeletFlags []string - }{ - CloudProvider: cloudProvider, - Hostname: hostname, - ClusterDNSIPs: dnsIPs, - KubeletVersion: version, - IsExternal: external, - IPFamily: ipFamily, - PauseImage: pauseImage, - InitialTaints: strings.Join(initialTaintsArgs, ","), - ExtraKubeletFlags: kubeletFlags, - } - - var buf strings.Builder - if err = tmpl.Execute(&buf, data); err != nil { - return "", fmt.Errorf("failed to execute kubelet-flags template: %w", err) - } - - return buf.String(), nil -} - -// KubeletHealthCheckSystemdUnit kubelet health checking systemd unit. -func KubeletHealthCheckSystemdUnit() string { - return `[Unit] -Requires=kubelet.service -After=kubelet.service - -[Service] -ExecStart=/opt/bin/health-monitor.sh kubelet - -[Install] -WantedBy=multi-user.target -` -} - -// ContainerRuntimeHealthCheckSystemdUnit container-runtime health checking systemd unit. -func ContainerRuntimeHealthCheckSystemdUnit(log *zap.SugaredLogger, containerRuntime string) (string, error) { - tmpl, err := template.New("container-runtime-healthcheck-systemd-unit").Funcs(TxtFuncMap(log)).Parse(containerRuntimeHealthCheckSystemdUnitTpl) - if err != nil { - return "", fmt.Errorf("failed to parse container-runtime-healthcheck-systemd-unit template: %w", err) - } - - data := struct { - ContainerRuntime string - }{ - ContainerRuntime: containerRuntime, - } - - var buf strings.Builder - if err = tmpl.Execute(&buf, data); err != nil { - return "", fmt.Errorf("failed to execute container-runtime-healthcheck-systemd-unit template: %w", err) - } - return buf.String(), nil -} diff --git a/pkg/userdata/helper/kubelet_test.go b/pkg/userdata/helper/kubelet_test.go deleted file mode 100644 index cf6f78d8e..000000000 --- a/pkg/userdata/helper/kubelet_test.go +++ /dev/null @@ -1,145 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helper - -import ( - "fmt" - "net" - "testing" - - "github.com/Masterminds/semver/v3" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - testhelper "github.com/kubermatic/machine-controller/pkg/test" - - corev1 "k8s.io/api/core/v1" -) - -type kubeletFlagTestCase struct { - name string - containerRuntime string - version *semver.Version - dnsIPs []net.IP - hostname string - cloudProvider string - external bool - ipFamily util.IPFamily - pauseImage string - initialTaints []corev1.Taint - extraFlags []string -} - -func TestKubeletSystemdUnit(t *testing.T) { - var tests []kubeletFlagTestCase - for _, version := range versions { - tests = append(tests, - kubeletFlagTestCase{ - name: fmt.Sprintf("version-%s", version.Original()), - version: version, - dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, - hostname: "some-test-node", - }, - kubeletFlagTestCase{ - name: fmt.Sprintf("version-%s-external", version.Original()), - version: version, - dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, - hostname: "some-test-node", - external: true, - }, - ) - } - tests = append(tests, []kubeletFlagTestCase{ - { - name: "multiple-dns-servers", - version: semver.MustParse("v1.30.0"), - dnsIPs: []net.IP{ - net.ParseIP("10.10.10.10"), - net.ParseIP("10.10.10.11"), - net.ParseIP("10.10.10.12"), - }, - hostname: "some-test-node", - }, - { - name: "cloud-provider-set", - version: semver.MustParse("v1.30.0"), - dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, - hostname: "some-test-node", - cloudProvider: "aws", - }, - { - name: "pause-image-set", - version: semver.MustParse("v1.30.0"), - dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, - hostname: "some-test-node", - cloudProvider: "aws", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "taints-set", - version: semver.MustParse("v1.30.0"), - dnsIPs: []net.IP{net.ParseIP("10.10.10.10")}, - hostname: "some-test-node", - cloudProvider: "aws", - initialTaints: []corev1.Taint{ - { - Key: "key1", - Value: "value1", - Effect: corev1.TaintEffectNoSchedule, - }, - { - Key: "key2", - Value: "value2", - Effect: corev1.TaintEffectNoExecute, - }, - }, - }, - }...) - - for _, test := range tests { - name := fmt.Sprintf("kublet_systemd_unit_%s", test.name) - t.Run(name, func(t *testing.T) { - out, err := KubeletSystemdUnit( - zap.NewNop().Sugar(), - defaultTo(test.containerRuntime, "docker"), - test.version.String(), - test.cloudProvider, - test.hostname, - test.dnsIPs, - test.external, - test.ipFamily, - test.pauseImage, - test.initialTaints, - test.extraFlags, - true, - ) - if err != nil { - t.Error(err) - } - goldenName := name + ".golden" - testhelper.CompareOutput(t, goldenName, out, *update) - }) - } -} - -func defaultTo(in string, defaultValue string) string { - if in == "" { - return defaultValue - } - - return in -} diff --git a/pkg/userdata/helper/template_functions.go b/pkg/userdata/helper/template_functions.go deleted file mode 100644 index 74062fba4..000000000 --- a/pkg/userdata/helper/template_functions.go +++ /dev/null @@ -1,81 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package helper - -import ( - "net" - "regexp" - "text/template" - - "github.com/Masterminds/sprig/v3" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - - corev1 "k8s.io/api/core/v1" -) - -// TxtFuncMap returns an aggregated template function map. Currently (custom functions + sprig). -func TxtFuncMap(log *zap.SugaredLogger) template.FuncMap { - funcMap := sprig.TxtFuncMap() - - // use inline wrappers to inject the logger without forcing the templates to keep track of it - - funcMap["downloadBinariesScript"] = func(kubeletVersion string, downloadKubelet bool) (string, error) { - return DownloadBinariesScript(log, kubeletVersion, downloadKubelet) - } - - funcMap["safeDownloadBinariesScript"] = func(kubeVersion string) (string, error) { - return SafeDownloadBinariesScript(log, kubeVersion) - } - - funcMap["kubeletSystemdUnit"] = func(containerRuntime, kubeletVersion, cloudProvider, hostname string, dnsIPs []net.IP, external bool, ipFamily util.IPFamily, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string, disableSwap bool) (string, error) { - return KubeletSystemdUnit(log, containerRuntime, kubeletVersion, cloudProvider, hostname, dnsIPs, external, ipFamily, pauseImage, initialTaints, extraKubeletFlags, disableSwap) - } - - funcMap["kubeletConfiguration"] = func(clusterDomain string, clusterDNS []net.IP, featureGates map[string]bool, kubeletConfigs map[string]string) (string, error) { - return kubeletConfiguration(log, clusterDomain, clusterDNS, featureGates, kubeletConfigs) - } - - funcMap["kubeletFlags"] = func(version, cloudProvider, hostname string, dnsIPs []net.IP, external bool, ipFamily util.IPFamily, pauseImage string, initialTaints []corev1.Taint, extraKubeletFlags []string) (string, error) { - return KubeletFlags(log, version, cloudProvider, hostname, dnsIPs, external, ipFamily, pauseImage, initialTaints, extraKubeletFlags) - } - - funcMap["containerRuntimeHealthCheckSystemdUnit"] = func(containerRuntime string) (string, error) { - return ContainerRuntimeHealthCheckSystemdUnit(log, containerRuntime) - } - - funcMap["kernelModulesScript"] = LoadKernelModulesScript - funcMap["kernelSettings"] = KernelSettings - funcMap["journalDConfig"] = JournalDConfig - funcMap["kubeletHealthCheckSystemdUnit"] = KubeletHealthCheckSystemdUnit - funcMap["dockerConfig"] = DockerConfig - funcMap["proxyEnvironment"] = ProxyEnvironment - funcMap["sshConfigAddendum"] = SSHConfigAddendum - - return funcMap -} - -// CleanupTemplateOutput postprocesses the output of the template processing. Those -// may exist due to the working of template functions like those of the sprig package -// or template condition. -func CleanupTemplateOutput(output string) (string, error) { - // Valid YAML files are not allowed to have empty lines containing spaces or tabs. - // So far only cleanup. - woBlankLines := regexp.MustCompile(`(?m)^[ \t]+$`).ReplaceAllString(output, "") - return woBlankLines, nil -} diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.27.0.golden b/pkg/userdata/helper/testdata/download_binaries_v1.27.0.golden deleted file mode 100644 index 1f219d364..000000000 --- a/pkg/userdata/helper/testdata/download_binaries_v1.27.0.golden +++ /dev/null @@ -1,17 +0,0 @@ -mkdir -p /opt/bin/ -mkdir -p /var/lib/calico -mkdir -p /etc/kubernetes/manifests -mkdir -p /etc/cni/net.d -mkdir -p /opt/cni/bin -if [ ! -f /opt/cni/bin/loopback ]; then - curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - -fi -if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.27.0/bin/linux/amd64/kubelet - chmod +x /opt/bin/kubelet -fi - -if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh -fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.28.0.golden b/pkg/userdata/helper/testdata/download_binaries_v1.28.0.golden deleted file mode 100644 index ffd12c3ae..000000000 --- a/pkg/userdata/helper/testdata/download_binaries_v1.28.0.golden +++ /dev/null @@ -1,17 +0,0 @@ -mkdir -p /opt/bin/ -mkdir -p /var/lib/calico -mkdir -p /etc/kubernetes/manifests -mkdir -p /etc/cni/net.d -mkdir -p /opt/cni/bin -if [ ! -f /opt/cni/bin/loopback ]; then - curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - -fi -if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.28.0/bin/linux/amd64/kubelet - chmod +x /opt/bin/kubelet -fi - -if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh -fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.29.0.golden b/pkg/userdata/helper/testdata/download_binaries_v1.29.0.golden deleted file mode 100644 index fe412fdad..000000000 --- a/pkg/userdata/helper/testdata/download_binaries_v1.29.0.golden +++ /dev/null @@ -1,17 +0,0 @@ -mkdir -p /opt/bin/ -mkdir -p /var/lib/calico -mkdir -p /etc/kubernetes/manifests -mkdir -p /etc/cni/net.d -mkdir -p /opt/cni/bin -if [ ! -f /opt/cni/bin/loopback ]; then - curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - -fi -if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.29.0/bin/linux/amd64/kubelet - chmod +x /opt/bin/kubelet -fi - -if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh -fi diff --git a/pkg/userdata/helper/testdata/download_binaries_v1.30.0.golden b/pkg/userdata/helper/testdata/download_binaries_v1.30.0.golden deleted file mode 100644 index 6603bc840..000000000 --- a/pkg/userdata/helper/testdata/download_binaries_v1.30.0.golden +++ /dev/null @@ -1,17 +0,0 @@ -mkdir -p /opt/bin/ -mkdir -p /var/lib/calico -mkdir -p /etc/kubernetes/manifests -mkdir -p /etc/cni/net.d -mkdir -p /opt/cni/bin -if [ ! -f /opt/cni/bin/loopback ]; then - curl -L https://github.com/containernetworking/plugins/releases/download/v0.8.7/cni-plugins-linux-amd64-v0.8.7.tgz | tar -xvzC /opt/cni/bin -f - -fi -if [ ! -f /opt/bin/kubelet ]; then - curl -Lfo /opt/bin/kubelet https://dl.k8s.io/v1.30.0/bin/linux/amd64/kubelet - chmod +x /opt/bin/kubelet -fi - -if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/8b5b66e4910a6228dfaecccaa0a3b05ec4902f8e/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh -fi diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden deleted file mode 100644 index 434a9081e..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_cloud-provider-set.golden +++ /dev/null @@ -1,34 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden deleted file mode 100644 index e70567560..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_multiple-dns-servers.golden +++ /dev/null @@ -1,35 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_pause-image-set.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_pause-image-set.golden deleted file mode 100644 index afcb8e0a3..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_pause-image-set.golden +++ /dev/null @@ -1,35 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_taints-set.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_taints-set.golden deleted file mode 100644 index 0581b535e..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_taints-set.golden +++ /dev/null @@ -1,35 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --register-with-taints=key1=value1:NoSchedule,key2=value2:NoExecute \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.0-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.0-external.golden deleted file mode 100644 index 50f4f5138..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.0-external.golden +++ /dev/null @@ -1,36 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.0.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.0.golden deleted file mode 100644 index e70567560..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.27.0.golden +++ /dev/null @@ -1,35 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0-external.golden deleted file mode 100644 index 50f4f5138..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0-external.golden +++ /dev/null @@ -1,36 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0.golden deleted file mode 100644 index e70567560..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.28.0.golden +++ /dev/null @@ -1,35 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.0-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.0-external.golden deleted file mode 100644 index 50f4f5138..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.0-external.golden +++ /dev/null @@ -1,36 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.0.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.0.golden deleted file mode 100644 index e70567560..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.29.0.golden +++ /dev/null @@ -1,35 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.30.0-external.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.30.0-external.golden deleted file mode 100644 index 50f4f5138..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.30.0-external.golden +++ /dev/null @@ -1,36 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.30.0.golden b/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.30.0.golden deleted file mode 100644 index e70567560..000000000 --- a/pkg/userdata/helper/testdata/kublet_systemd_unit_version-v1.30.0.golden +++ /dev/null @@ -1,35 +0,0 @@ -[Unit] -After=docker.service -Requires=docker.service - -Description=kubelet: The Kubernetes Node Agent -Documentation=https://kubernetes.io/docs/home/ - -[Service] -User=root -Restart=always -StartLimitInterval=0 -RestartSec=10 -CPUAccounting=true -MemoryAccounting=true - -Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" -EnvironmentFile=-/etc/environment - -ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - -ExecStartPre=/bin/bash /opt/disable-swap.sh - -ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh -ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=some-test-node \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --node-ip ${KUBELET_NODE_IP} - -[Install] -WantedBy=multi-user.target \ No newline at end of file diff --git a/pkg/userdata/helper/testdata/safe_download_binaries_v1.30.0.golden b/pkg/userdata/helper/testdata/safe_download_binaries_v1.30.0.golden deleted file mode 100644 index 6c0b33dc7..000000000 --- a/pkg/userdata/helper/testdata/safe_download_binaries_v1.30.0.golden +++ /dev/null @@ -1,65 +0,0 @@ -opt_bin=/opt/bin -usr_local_bin=/usr/local/bin -cni_bin_dir=/opt/cni/bin -mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" -arch=${HOST_ARCH-} -if [ -z "$arch" ] -then -case $(uname -m) in -x86_64) - arch="amd64" - ;; -aarch64) - arch="arm64" - ;; -*) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; -esac -fi -CNI_VERSION="${CNI_VERSION:-v1.2.0}" -cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" -cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" -curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" -cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") -cd "$cni_bin_dir" -sha256sum -c <<<"$cni_sum" -tar xvf "$cni_filename" -rm -f "$cni_filename" -cd - -CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.27.0}" -cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/$%7BCRI_TOOLS_RELEASE%7D" -cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" -curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" -cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") -cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" -cd "$opt_bin" -sha256sum -c <<<"$cri_tools_sum" -tar xvf "$cri_tools_filename" -rm -f "$cri_tools_filename" -ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" -cd - -KUBE_VERSION="${KUBE_VERSION:-v1.30.0}" -kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" -kube_base_url="https://dl.k8s.io/$KUBE_VERSION/bin/linux/$arch" -kube_sum_file="$kube_dir/sha256" -mkdir -p "$kube_dir" -: >"$kube_sum_file" - -for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" -done -sha256sum -c "$kube_sum_file" - -for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin -done - -if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh -fi diff --git a/pkg/userdata/manager/manager.go b/pkg/userdata/manager/manager.go deleted file mode 100644 index e53e0cfbf..000000000 --- a/pkg/userdata/manager/manager.go +++ /dev/null @@ -1,98 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin manager. -// - -// Package manager provides the instantiation and -// running of the plugins on machine controller side. -package manager - -import ( - "errors" - "flag" - - "go.uber.org/zap" - - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -var ( - // ErrLocatingPlugins is returned when a new manager cannot locate - // the plugins for the supported operating systems. - ErrLocatingPlugins = errors.New("one or more user data plugins not found") - - // ErrPluginNotFound describes an invalid operating system for - // a user data plugin. Here directory has to be checked if - // correct ones are installed. - ErrPluginNotFound = errors.New("no user data plugin for the given operating system found") - - // supportedOS contains a list of operating systems the machine - // controller supports. - supportedOS = []providerconfigtypes.OperatingSystem{ - providerconfigtypes.OperatingSystemAmazonLinux2, - providerconfigtypes.OperatingSystemCentOS, - providerconfigtypes.OperatingSystemFlatcar, - providerconfigtypes.OperatingSystemRHEL, - providerconfigtypes.OperatingSystemUbuntu, - providerconfigtypes.OperatingSystemRockyLinux, - } -) - -// Manager inits and manages the userdata plugins. -type Manager struct { - debug bool - log *zap.SugaredLogger - plugins map[providerconfigtypes.OperatingSystem]*Plugin -} - -// New returns an initialised plugin manager. -func New(log *zap.SugaredLogger) (*Manager, error) { - m := &Manager{ - log: log, - plugins: make(map[providerconfigtypes.OperatingSystem]*Plugin), - } - flag.BoolVar(&m.debug, "plugin-debug", false, "Switch for enabling the plugin debugging") - m.locatePlugins() - if len(m.plugins) < len(supportedOS) { - return nil, ErrLocatingPlugins - } - return m, nil -} - -// ForOS returns the plugin for the given operating system. -func (m *Manager) ForOS(os providerconfigtypes.OperatingSystem) (p *Plugin, err error) { - var found bool - if p, found = m.plugins[os]; !found { - return nil, ErrPluginNotFound - } - return p, nil -} - -// locatePlugins tries to find the plugins and inits their wrapper. -func (m *Manager) locatePlugins() { - for _, os := range supportedOS { - osLog := m.log.With("os", os) - - plugin, err := newPlugin(osLog, os, m.debug) - if err != nil { - osLog.Errorw("Cannot use plugin", zap.Error(err)) - continue - } - m.plugins[os] = plugin - } -} diff --git a/pkg/userdata/manager/plugin.go b/pkg/userdata/manager/plugin.go deleted file mode 100644 index 6f7014fd6..000000000 --- a/pkg/userdata/manager/plugin.go +++ /dev/null @@ -1,144 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin manager. -// - -package manager - -import ( - "encoding/json" - "fmt" - "os" - "os/exec" - "path/filepath" - "strings" - - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -const ( - // pluginPrefix has to be the prefix of all plugin filenames. - pluginPrefix = "machine-controller-userdata-" -) - -// Plugin looks for the plugin executable and calls it for -// each request. -type Plugin struct { - debug bool - log *zap.SugaredLogger - command string -} - -// newPlugin creates a new plugin manager. It starts the named -// binary and connects to it via net/rpc. -func newPlugin(log *zap.SugaredLogger, os providerconfigtypes.OperatingSystem, debug bool) (*Plugin, error) { - p := &Plugin{ - debug: debug, - log: log, - } - if err := p.findPlugin(string(os)); err != nil { - return nil, err - } - return p, nil -} - -// UserData retrieves the user data of the given resource via -// plugin handling the communication. -func (p *Plugin) UserData(_ *zap.SugaredLogger, req plugin.UserDataRequest) (string, error) { - // Prepare command. - var argv []string - if p.debug { - argv = append(argv, "-debug") - } - cmd := exec.Command(p.command, argv...) - // Set environment. - reqj, err := json.Marshal(req) - if err != nil { - return "", err - } - cmd.Env = append(os.Environ(), fmt.Sprintf("%s=%s", plugin.EnvUserDataRequest, string(reqj))) - // Execute command. - out, err := cmd.CombinedOutput() - if err != nil { - return "", fmt.Errorf("failed to execute command %q: output: %q error: %w", p.command, string(out), err) - } - var resp plugin.UserDataResponse - err = json.Unmarshal(out, &resp) - if err != nil { - return "", err - } - if resp.Err != "" { - return "", fmt.Errorf("%s", resp.Err) - } - return resp.UserData, nil -} - -// findPlugin tries to find the executable of the plugin. -func (p *Plugin) findPlugin(name string) error { - filename := pluginPrefix + name - pluginLog := p.log.With("plugin", filename) - pluginLog.Infow("Looking for plugin") - // Create list to search in. - var dirs []string - envDir := os.Getenv(plugin.EnvPluginDir) - if envDir != "" { - dirs = append(dirs, envDir) - } - executable, err := os.Executable() - if err != nil { - return err - } - ownDir, _ := filepath.Split(executable) - ownDir, err = filepath.Abs(ownDir) - if err != nil { - return err - } - dirs = append(dirs, ownDir) - workingDir, err := os.Getwd() - if err != nil { - return err - } - dirs = append(dirs, workingDir) - path := os.Getenv("PATH") - pathDirs := strings.Split(path, string(os.PathListSeparator)) - dirs = append(dirs, pathDirs...) - // Now take a look. - for _, dir := range dirs { - command := dir + string(os.PathSeparator) + filename - pluginLog.Debugw("Checking directory", "directory", dir) - fi, err := os.Stat(command) - if err != nil { - if os.IsNotExist(err) { - continue - } - return fmt.Errorf("error when looking for %q: %w", command, err) - } - if fi.IsDir() || (fi.Mode()&0111 == 0) { - pluginLog.Infow("Found file, but is no executable", "filename", command) - continue - } - p.command = command - p.log.Infow("Found plugin", "filename", command) - return nil - } - pluginLog.Error("Did not find plugin") - return ErrPluginNotFound -} diff --git a/pkg/userdata/plugin/plugin.go b/pkg/userdata/plugin/plugin.go deleted file mode 100644 index b7471f10d..000000000 --- a/pkg/userdata/plugin/plugin.go +++ /dev/null @@ -1,90 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// Core UserData plugin. -// - -// Package plugin provides the plugin side of the plugin mechanism. -// Individual plugins have to implement the provider interface, -// pass it to a new plugin instance, and call run. -package plugin - -import ( - "encoding/json" - "fmt" - "os" - - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/plugin" -) - -// Provider defines the interface each plugin has to implement -// for the retrieval of the userdata based on the given arguments. -type Provider interface { - UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) (string, error) -} - -// Plugin implements a convenient helper to map the request to the given -// provider and return the response. -type Plugin struct { - provider Provider - debug bool -} - -// New creates a new plugin. -func New(provider Provider, debug bool) *Plugin { - return &Plugin{ - provider: provider, - debug: debug, - } -} - -// Run looks for the given request and executes it. -func (p *Plugin) Run(log *zap.SugaredLogger) error { - reqEnv := os.Getenv(plugin.EnvUserDataRequest) - if reqEnv == "" { - resp := plugin.ErrorResponse{ - Err: fmt.Sprintf("environment variable '%s' not set", plugin.EnvUserDataRequest), - } - return p.printResponse(resp) - } - // Handle the request for user data. - var req plugin.UserDataRequest - err := json.Unmarshal([]byte(reqEnv), &req) - if err != nil { - return err - } - userData, err := p.provider.UserData(log, req) - var resp plugin.UserDataResponse - if err != nil { - resp.Err = err.Error() - } else { - resp.UserData = userData - } - return p.printResponse(resp) -} - -// printResponse marshals the response and prints it to stdout. -func (p *Plugin) printResponse(resp interface{}) error { - bs, err := json.Marshal(resp) - if err != nil { - return err - } - _, err = fmt.Printf("%s", string(bs)) - return err -} diff --git a/pkg/userdata/rhel/provider.go b/pkg/userdata/rhel/provider.go deleted file mode 100644 index 218c8b3ec..000000000 --- a/pkg/userdata/rhel/provider.go +++ /dev/null @@ -1,432 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for RHEL. -// - -package rhel - -import ( - "errors" - "fmt" - "strings" - "text/template" - - "github.com/Masterminds/semver/v3" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - userdatahelper "github.com/kubermatic/machine-controller/pkg/userdata/helper" -) - -// Provider is a pkg/userdata/plugin.Provider implementation. -type Provider struct{} - -// UserData renders user-data template to string. -func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) (string, error) { - tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap(log)).Parse(userDataTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse user-data template: %w", err) - } - - kubeletVersion, err := semver.NewVersion(req.MachineSpec.Versions.Kubelet) - if err != nil { - return "", fmt.Errorf("invalid kubelet version: %w", err) - } - - pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) - if err != nil { - return "", fmt.Errorf("failed to get provider config: %w", err) - } - - if pconfig.OverwriteCloudConfig != nil { - req.CloudConfig = *pconfig.OverwriteCloudConfig - } - - if pconfig.Network.IsStaticIPConfig() { - return "", errors.New("static IP config is not supported with RHEL") - } - - rhelConfig, err := LoadConfig(pconfig.OperatingSystemSpec) - if err != nil { - return "", fmt.Errorf("failed to parse OperatingSystemSpec: %w", err) - } - - kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) - if err != nil { - return "", err - } - - kubernetesCACert, err := userdatahelper.GetCACert(req.Kubeconfig) - if err != nil { - return "", fmt.Errorf("error extracting cacert: %w", err) - } - - crEngine := req.ContainerRuntime.Engine() - crScript, err := crEngine.ScriptFor(providerconfigtypes.OperatingSystemRHEL) - if err != nil { - return "", fmt.Errorf("failed to generate container runtime install script: %w", err) - } - - crConfig, err := crEngine.Config() - if err != nil { - return "", fmt.Errorf("failed to generate container runtime config: %w", err) - } - - crAuthConfig, err := crEngine.AuthConfig() - if err != nil { - return "", fmt.Errorf("failed to generate container runtime auth config: %w", err) - } - - data := struct { - plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - OSConfig *Config - KubeletVersion string - Kubeconfig string - KubernetesCACert string - NodeIPScript string - ExtraKubeletFlags []string - ContainerRuntimeScript string - ContainerRuntimeConfigFileName string - ContainerRuntimeConfig string - ContainerRuntimeAuthConfigFileName string - ContainerRuntimeAuthConfig string - ContainerRuntimeName string - }{ - UserDataRequest: req, - ProviderSpec: pconfig, - OSConfig: rhelConfig, - KubeletVersion: kubeletVersion.String(), - Kubeconfig: kubeconfigString, - KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(pconfig.Network.GetIPFamily()), - ExtraKubeletFlags: crEngine.KubeletFlags(), - ContainerRuntimeScript: crScript, - ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), - ContainerRuntimeConfig: crConfig, - ContainerRuntimeAuthConfigFileName: crEngine.AuthConfigFileName(), - ContainerRuntimeAuthConfig: crAuthConfig, - ContainerRuntimeName: crEngine.String(), - } - - var buf strings.Builder - if err = tmpl.Execute(&buf, data); err != nil { - return "", fmt.Errorf("failed to execute user-data template: %w", err) - } - - return userdatahelper.CleanupTemplateOutput(buf.String()) -} - -// UserData template. -const userDataTemplate = `#cloud-config -bootcmd: -- modprobe ip_tables -{{ if ne .CloudProviderName "aws" }} -hostname: {{ .MachineSpec.Name }} -fqdn: {{ .MachineSpec.Name }} -{{- /* Never set the hostname on AWS nodes. Kubernetes(kube-proxy) requires the hostname to be the private dns name */}} -{{ end }} - -{{- if .OSConfig.DistUpgradeOnBoot }} -package_upgrade: true -package_reboot_if_required: true -{{- end }} - -ssh_pwauth: false - -{{- if ne (len .ProviderSpec.SSHPublicKeys) 0 }} -ssh_authorized_keys: -{{- range .ProviderSpec.SSHPublicKeys }} - - "{{ . }}" -{{- end }} -{{- end }} - -write_files: -{{- if .HTTPProxy }} -- path: "/etc/environment" - content: | -{{ proxyEnvironment .HTTPProxy .NoProxy | indent 4 }} -{{- end }} - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | -{{ journalDConfig | indent 4 }} - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | -{{ kernelModulesScript | indent 4 }} - -- path: "/etc/sysctl.d/k8s.conf" - content: | -{{ kernelSettings | indent 4 }} - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - -{{- /* As we added some modules and don't want to reboot, restart the service */}} - systemctl restart systemd-modules-load.service - sysctl --system - {{ if ne .CloudProviderName "aws" }} -{{- /* The normal way of setting it via cloud-init is broken, see */}} -{{- /* https://bugs.launchpad.net/cloud-init/+bug/1662542 */}} - hostnamectl set-hostname {{ .MachineSpec.Name }} - {{ end }} - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - {{- if or (eq .CloudProviderName "vsphere") (eq .CloudProviderName "vmware-cloud-director") }} - open-vm-tools \ - {{- end }} - {{- if eq .CloudProviderName "nutanix" }} - iscsi-initiator-utils \ - {{- end }} - ipvsadm - - {{- /* iscsid service is required on Nutanix machines for CSI driver to attach volumes. */}} - {{- if eq .CloudProviderName "nutanix" }} - systemctl enable --now iscsid - {{ end }} -{{ .ContainerRuntimeScript | indent 4 }} -{{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - {{ if eq .CloudProviderName "vsphere" }} - systemctl enable --now vmtoolsd.service - {{ end -}} - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - {{- if eq .CloudProviderName "kubevirt" }} - systemctl enable --now --no-block restart-kubelet.service - {{ end }} - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - -{{- if ne (len .CloudConfig) 0 }} -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | -{{ .CloudConfig | indent 4 }} -{{- end }} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | -{{ .NodeIPScript | indent 4 }} - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | -{{ .Kubeconfig | indent 4 }} - -- path: "/etc/kubernetes/kubelet.conf" - content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - -- path: "/etc/kubernetes/pki/ca.crt" - content: | -{{ .KubernetesCACert | indent 4 }} - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: {{ .ContainerRuntimeConfigFileName }} - permissions: "0644" - content: | -{{ .ContainerRuntimeConfig | indent 4 }} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | -{{ kubeletHealthCheckSystemdUnit | indent 4 }} - -{{- with .ProviderSpec.CAPublicKey }} - -- path: "/etc/ssh/trusted-user-ca-keys.pem" - content: | -{{ . | indent 4 }} - -- path: "/etc/ssh/sshd_config" - content: | -{{ sshConfigAddendum | indent 4 }} - append: true -{{- end }} - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -{{- if eq .CloudProviderName "kubevirt" }} -- path: "/opt/bin/restart-kubelet.sh" - permissions: "0744" - content: | - #!/bin/bash - # Needed for Kubevirt provider because if the virt-launcher pod is deleted, - # the VM and DataVolume states are kept and VM is rebooted. We need to restart the kubelet - # with the new config (new IP) and run this at every boot. - set -xeuo pipefail - - # This helps us avoid an unnecessary restart for kubelet on the first boot - if [ -f /etc/kubelet_needs_restart ]; then - # restart kubelet since it's not the first boot - systemctl daemon-reload - systemctl restart kubelet.service - else - touch /etc/kubelet_needs_restart - fi - -- path: "/etc/systemd/system/restart-kubelet.service" - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - Description=Service responsible for restarting kubelet when the machine is rebooted - - [Service] - Type=oneshot - ExecStart=/opt/bin/restart-kubelet.sh - - [Install] - WantedBy=multi-user.target -{{- end }} - -rh_subscription: -{{- if .OSConfig.RHELUseSatelliteServer }} - org: "{{.OSConfig.RHELOrganizationName}}" - activation-key: "{{.OSConfig.RHELActivationKey}}" - server-hostname: {{ .OSConfig.RHELSatelliteServer }} - rhsm-baseurl: https://{{ .OSConfig.RHELSatelliteServer }}/pulp/repos -{{- else }} - username: "{{.OSConfig.RHELSubscriptionManagerUser}}" - password: "{{.OSConfig.RHELSubscriptionManagerPassword}}" - auto-attach: {{.OSConfig.AttachSubscription}} -{{- end }} - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service -` diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go deleted file mode 100644 index 6412245b5..000000000 --- a/pkg/userdata/rhel/provider_test.go +++ /dev/null @@ -1,298 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for RHEL. -// - -package rhel - -import ( - "flag" - "net" - "testing" - - "go.uber.org/zap" - - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - "github.com/kubermatic/machine-controller/pkg/containerruntime" - testhelper "github.com/kubermatic/machine-controller/pkg/test" - "github.com/kubermatic/machine-controller/pkg/userdata/convert" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" -) - -var ( - update = flag.Bool("update", false, "update testdata files") - - pemCertificate = `-----BEGIN CERTIFICATE----- -MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV -BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG -A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 -DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 -NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG -cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv -c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B -AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS -R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT -ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk -JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 -mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW -caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G -A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt -hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB -MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES -MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv -bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h -U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao -eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 -UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD -58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n -sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF -kPe6XoSbiLm/kxk32T0= ------END CERTIFICATE-----` -) - -// fakeCloudConfigProvider simulates cloud config provider for test. -type fakeCloudConfigProvider struct { - config string - name string - err error -} - -func (p *fakeCloudConfigProvider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return p.config, p.name, p.err -} - -// userDataTestCase contains the data for a table-driven test. -type userDataTestCase struct { - name string - spec clusterv1alpha1.MachineSpec - clusterDNSIPs []net.IP - cloudProviderName *string - externalCloudProvider bool - httpProxy string - noProxy string - insecureRegistries string - registryMirrors string - pauseImage string - containerruntime string -} - -// TestUserDataGeneration runs the data generation for different -// environments. -func TestUserDataGeneration(t *testing.T) { - t.Parallel() - - tests := []userDataTestCase{ - { - name: "kubelet-v1.28-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", - }, - }, - }, - { - name: "kubelet-v1.29-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.0", - }, - }, - }, - { - name: "kubelet-v1.30.0-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - }, - { - name: "kubelet-v1.30.0-aws-external", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - externalCloudProvider: true, - }, - { - name: "kubelet-v1.30.0-vsphere", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - cloudProviderName: stringPtr("vsphere"), - }, - { - name: "kubelet-v1.30.0-vsphere-proxy", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128/", - noProxy: "192.168.1.0", - insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "kubelet-v1.30.0-vsphere-mirrors", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128/", - noProxy: "192.168.1.0", - registryMirrors: "https://registry.docker-cn.com/", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "kubelet-v1.28-nutanix", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", - }, - }, - cloudProviderName: stringPtr("nutanix"), - }, - { - name: "pod-cidr-azure-rhel", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.0", - }, - }, - cloudProviderName: stringPtr("azure"), - }, - { - name: "kubelet-v1.29-nutanix", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.0", - }, - }, - cloudProviderName: stringPtr("nutanix"), - }, - } - - defaultCloudProvider := &fakeCloudConfigProvider{ - name: "aws", - config: "{aws-config:true}", - err: nil, - } - kubeconfig := &clientcmdapi.Config{ - Clusters: map[string]*clientcmdapi.Cluster{ - "": { - Server: "https://server/", - CertificateAuthorityData: []byte(pemCertificate), - }, - }, - AuthInfos: map[string]*clientcmdapi.AuthInfo{ - "": { - Token: "my-token", - }, - }, - } - provider := Provider{} - - kubeletFeatureGates := map[string]bool{ - "RotateKubeletServerCertificate": true, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - emptyProviderSpec := clusterv1alpha1.ProviderSpec{ - Value: &runtime.RawExtension{}, - } - test.spec.ProviderSpec = emptyProviderSpec - var cloudProvider *fakeCloudConfigProvider - if test.cloudProviderName != nil { - cloudProvider = &fakeCloudConfigProvider{ - name: *test.cloudProviderName, - config: "{config:true}", - err: nil, - } - } else { - cloudProvider = defaultCloudProvider - } - cloudConfig, cloudProviderName, err := cloudProvider.GetCloudConfig(test.spec) - if err != nil { - t.Fatalf("failed to get cloud config: %v", err) - } - - containerRuntimeOpts := containerruntime.Opts{ - ContainerRuntime: test.containerruntime, - InsecureRegistries: test.insecureRegistries, - RegistryMirrors: test.registryMirrors, - } - containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) - if err != nil { - t.Fatalf("failed to generate container runtime config: %v", err) - } - - req := plugin.UserDataRequest{ - MachineSpec: test.spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: cloudProviderName, - KubeletCloudProviderName: cloudProviderName, - DNSIPs: test.clusterDNSIPs, - ExternalCloudProvider: test.externalCloudProvider, - HTTPProxy: test.httpProxy, - NoProxy: test.noProxy, - PauseImage: test.pauseImage, - KubeletFeatureGates: kubeletFeatureGates, - ContainerRuntime: containerRuntimeConfig, - } - s, err := provider.UserData(zap.NewNop().Sugar(), req) - if err != nil { - t.Errorf("error getting userdata: '%v'", err) - } - - // Check if we can gzip it. - if _, err := convert.GzipString(s); err != nil { - t.Fatal(err) - } - goldenName := test.name + ".yaml" - testhelper.CompareOutput(t, goldenName, s, *update) - }) - } -} - -// stringPtr returns pointer to given string. -func stringPtr(a string) *string { - return &a -} diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.28-aws.yaml deleted file mode 100644 index d3d5d387c..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.28-aws.yaml +++ /dev/null @@ -1,503 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml deleted file mode 100644 index 3d321946f..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.28-nutanix.yaml +++ /dev/null @@ -1,512 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 -fqdn: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - iscsi-initiator-utils \ - ipvsadm - systemctl enable --now iscsid - - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=nutanix \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.29-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29-aws.yaml deleted file mode 100644 index 3197af4b8..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.29-aws.yaml +++ /dev/null @@ -1,501 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.29-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.29-nutanix.yaml deleted file mode 100644 index f58782ffc..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.29-nutanix.yaml +++ /dev/null @@ -1,510 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 -fqdn: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - iscsi-initiator-utils \ - ipvsadm - systemctl enable --now iscsid - - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.30.0-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.30.0-aws-external.yaml deleted file mode 100644 index e9b71441f..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.30.0-aws-external.yaml +++ /dev/null @@ -1,503 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=${KUBELET_HOSTNAME} \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.30.0-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.30.0-aws.yaml deleted file mode 100644 index 7cdd4a479..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.30.0-aws.yaml +++ /dev/null @@ -1,501 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml deleted file mode 100644 index 17c83b356..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml +++ /dev/null @@ -1,519 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 -fqdn: node1 - - -ssh_pwauth: false - -write_files: -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry.docker-cn.com/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.30.0-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.30.0-vsphere-proxy.yaml deleted file mode 100644 index 9dd133846..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.30.0-vsphere-proxy.yaml +++ /dev/null @@ -1,526 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 -fqdn: node1 - - -ssh_pwauth: false - -write_files: -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - [plugins."io.containerd.grpc.v1.cri".registry.configs] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000".tls] - insecure_skip_verify = true - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000".tls] - insecure_skip_verify = true - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.30.0-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.30.0-vsphere.yaml deleted file mode 100644 index 6a6dd7f63..000000000 --- a/pkg/userdata/rhel/testdata/kubelet-v1.30.0-vsphere.yaml +++ /dev/null @@ -1,510 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 -fqdn: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml deleted file mode 100644 index e38dc657f..000000000 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ /dev/null @@ -1,507 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 -fqdn: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - hostnamectl set-hostname node1 - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - systemctl disable disable-nm-cloud-setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: "/opt/bin/disable-nm-cloud-setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl status 'nm-cloud-setup.timer' 2> /dev/null | grep -Fq "Active:"; then - systemctl stop nm-cloud-setup.timer - systemctl disable nm-cloud-setup.service - systemctl disable nm-cloud-setup.timer - reboot - fi - -- path: "/etc/systemd/system/disable-nm-cloud-setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/disable-nm-cloud-setup - -rh_subscription: - username: "" - password: "" - auto-attach: false - -runcmd: -- systemctl enable --now setup.service -- systemctl enable --now disable-nm-cloud-setup.service diff --git a/pkg/userdata/rockylinux/provider.go b/pkg/userdata/rockylinux/provider.go deleted file mode 100644 index 06f2633de..000000000 --- a/pkg/userdata/rockylinux/provider.go +++ /dev/null @@ -1,352 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for RockyLinux. -// - -package rockylinux - -import ( - "errors" - "fmt" - "strings" - "text/template" - - "github.com/Masterminds/semver/v3" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - userdatahelper "github.com/kubermatic/machine-controller/pkg/userdata/helper" -) - -// Provider is a pkg/userdata/plugin.Provider implementation. -type Provider struct{} - -// UserData renders user-data template to string. -func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) (string, error) { - tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap(log)).Parse(userDataTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse user-data template: %w", err) - } - - kubeletVersion, err := semver.NewVersion(req.MachineSpec.Versions.Kubelet) - if err != nil { - return "", fmt.Errorf("invalid kubelet version: %w", err) - } - - pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) - if err != nil { - return "", fmt.Errorf("failed to get provider config: %w", err) - } - - if pconfig.OverwriteCloudConfig != nil { - req.CloudConfig = *pconfig.OverwriteCloudConfig - } - - if pconfig.Network.IsStaticIPConfig() { - return "", errors.New("static IP config is not supported with RockyLinux") - } - - rockyLinuxConfig, err := LoadConfig(pconfig.OperatingSystemSpec) - if err != nil { - return "", fmt.Errorf("failed to parse OperatingSystemSpec: %w", err) - } - - kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) - if err != nil { - return "", err - } - - kubernetesCACert, err := userdatahelper.GetCACert(req.Kubeconfig) - if err != nil { - return "", fmt.Errorf("error extracting cacert: %w", err) - } - - crEngine := req.ContainerRuntime.Engine() - crScript, err := crEngine.ScriptFor(providerconfigtypes.OperatingSystemRockyLinux) - if err != nil { - return "", fmt.Errorf("failed to generate container runtime install script: %w", err) - } - - crConfig, err := crEngine.Config() - if err != nil { - return "", fmt.Errorf("failed to generate container runtime config: %w", err) - } - - crAuthConfig, err := crEngine.AuthConfig() - if err != nil { - return "", fmt.Errorf("failed to generate container runtime auth config: %w", err) - } - - data := struct { - plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - OSConfig *Config - KubeletVersion string - Kubeconfig string - KubernetesCACert string - NodeIPScript string - ExtraKubeletFlags []string - ContainerRuntimeScript string - ContainerRuntimeConfigFileName string - ContainerRuntimeConfig string - ContainerRuntimeAuthConfigFileName string - ContainerRuntimeAuthConfig string - ContainerRuntimeName string - }{ - UserDataRequest: req, - ProviderSpec: pconfig, - OSConfig: rockyLinuxConfig, - KubeletVersion: kubeletVersion.String(), - Kubeconfig: kubeconfigString, - KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(pconfig.Network.GetIPFamily()), - ExtraKubeletFlags: crEngine.KubeletFlags(), - ContainerRuntimeScript: crScript, - ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), - ContainerRuntimeConfig: crConfig, - ContainerRuntimeAuthConfigFileName: crEngine.AuthConfigFileName(), - ContainerRuntimeAuthConfig: crAuthConfig, - ContainerRuntimeName: crEngine.String(), - } - - buf := strings.Builder{} - if err = tmpl.Execute(&buf, data); err != nil { - return "", fmt.Errorf("failed to execute user-data template: %w", err) - } - - return userdatahelper.CleanupTemplateOutput(buf.String()) -} - -// UserData template. -const userDataTemplate = `#cloud-config -bootcmd: -- modprobe ip_tables -{{ if ne .CloudProviderName "aws" }} -hostname: {{ .MachineSpec.Name }} -{{- /* Never set the hostname on AWS nodes. Kubernetes(kube-proxy) requires the hostname to be the private dns name */}} -{{ end }} - -{{- if .OSConfig.DistUpgradeOnBoot }} -package_upgrade: true -package_reboot_if_required: true -{{- end }} - -ssh_pwauth: false - -{{- if ne (len .ProviderSpec.SSHPublicKeys) 0 }} -ssh_authorized_keys: -{{- range .ProviderSpec.SSHPublicKeys }} - - "{{ . }}" -{{- end }} -{{- end }} - -write_files: -{{- if .HTTPProxy }} -- path: "/etc/environment" - content: | -{{ proxyEnvironment .HTTPProxy .NoProxy | indent 4 }} -{{- end }} - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | -{{ journalDConfig | indent 4 }} - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | -{{ kernelModulesScript | indent 4 }} - -- path: "/etc/sysctl.d/k8s.conf" - content: | -{{ kernelSettings | indent 4 }} - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - -{{- /* As we added some modules and don't want to reboot, restart the service */}} - systemctl restart systemd-modules-load.service - sysctl --system - - {{ if ne .CloudProviderName "aws" }} -{{- /* The normal way of setting it via cloud-init is broken, see */}} -{{- /* https://bugs.launchpad.net/cloud-init/+bug/1662542 */}} - hostnamectl set-hostname {{ .MachineSpec.Name }} - {{ end -}} - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - {{- if or (eq .CloudProviderName "vsphere") (eq .CloudProviderName "vmware-cloud-director") }} - open-vm-tools \ - {{- end }} - {{- if eq .CloudProviderName "nutanix" }} - iscsi-initiator-utils \ - {{- end }} - ipvsadm - - {{- /* iscsid service is required on Nutanix machines for CSI driver to attach volumes. */}} - {{- if eq .CloudProviderName "nutanix" }} - systemctl enable --now iscsid - {{ end }} -{{ .ContainerRuntimeScript | indent 4 }} -{{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - {{ if eq .CloudProviderName "vsphere" }} - systemctl enable --now vmtoolsd.service - {{ end -}} - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - -{{- if ne (len .CloudConfig) 0 }} -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | -{{ .CloudConfig | indent 4 }} -{{- end }} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | -{{ .NodeIPScript | indent 4 }} - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | -{{ .Kubeconfig | indent 4 }} - -- path: "/etc/kubernetes/kubelet.conf" - content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - -- path: "/etc/kubernetes/pki/ca.crt" - content: | -{{ .KubernetesCACert | indent 4 }} - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: {{ .ContainerRuntimeConfigFileName }} - permissions: "0644" - content: | -{{ .ContainerRuntimeConfig | indent 4 }} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | -{{ kubeletHealthCheckSystemdUnit | indent 4 }} - -{{- with .ProviderSpec.CAPublicKey }} - -- path: "/etc/ssh/trusted-user-ca-keys.pem" - content: | -{{ . | indent 4 }} - -- path: "/etc/ssh/sshd_config" - content: | -{{ sshConfigAddendum | indent 4 }} - append: true -{{- end }} - -runcmd: -- systemctl enable --now setup.service -` diff --git a/pkg/userdata/rockylinux/provider_test.go b/pkg/userdata/rockylinux/provider_test.go deleted file mode 100644 index 5172756ca..000000000 --- a/pkg/userdata/rockylinux/provider_test.go +++ /dev/null @@ -1,270 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for RockyLinux. -// - -package rockylinux - -import ( - "flag" - "net" - "testing" - - "go.uber.org/zap" - - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - "github.com/kubermatic/machine-controller/pkg/containerruntime" - testhelper "github.com/kubermatic/machine-controller/pkg/test" - "github.com/kubermatic/machine-controller/pkg/userdata/convert" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" -) - -var ( - update = flag.Bool("update", false, "update testdata files") - - pemCertificate = `-----BEGIN CERTIFICATE----- -MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV -BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG -A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 -DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 -NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG -cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv -c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B -AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS -R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT -ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk -JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 -mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW -caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G -A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt -hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB -MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES -MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv -bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h -U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao -eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 -UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD -58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n -sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF -kPe6XoSbiLm/kxk32T0= ------END CERTIFICATE-----` -) - -// fakeCloudConfigProvider simulates cloud config provider for test. -type fakeCloudConfigProvider struct { - config string - name string - err error -} - -func (p *fakeCloudConfigProvider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return p.config, p.name, p.err -} - -// userDataTestCase contains the data for a table-driven test. -type userDataTestCase struct { - name string - spec clusterv1alpha1.MachineSpec - clusterDNSIPs []net.IP - cloudProviderName *string - externalCloudProvider bool - httpProxy string - noProxy string - insecureRegistries string - registryMirrors string - pauseImage string - containerruntime string -} - -// TestUserDataGeneration runs the data generation for different -// environments. -func TestUserDataGeneration(t *testing.T) { - t.Parallel() - - tests := []userDataTestCase{ - { - name: "kubelet-v1.28-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.28.0", - }, - }, - }, - { - name: "kubelet-v1.30.0-aws", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - }, - { - name: "kubelet-v1.30.0-aws-external", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - externalCloudProvider: true, - }, - { - name: "kubelet-v1.30.0-vsphere", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - cloudProviderName: stringPtr("vsphere"), - }, - { - name: "kubelet-v1.30.0-vsphere-proxy", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128/", - noProxy: "192.168.1.0", - insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "kubelet-v1.30.0-vsphere-mirrors", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - cloudProviderName: stringPtr("vsphere"), - httpProxy: "http://192.168.100.100:3128/", - noProxy: "192.168.1.0", - registryMirrors: "https://registry.docker-cn.com/", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "kubelet-v1.30.0-nutanix", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.30.0", - }, - }, - cloudProviderName: stringPtr("nutanix"), - }, - } - - defaultCloudProvider := &fakeCloudConfigProvider{ - name: "aws", - config: "{aws-config:true}", - err: nil, - } - kubeconfig := &clientcmdapi.Config{ - Clusters: map[string]*clientcmdapi.Cluster{ - "": { - Server: "https://server/", - CertificateAuthorityData: []byte(pemCertificate), - }, - }, - AuthInfos: map[string]*clientcmdapi.AuthInfo{ - "": { - Token: "my-token", - }, - }, - } - provider := Provider{} - - kubeletFeatureGates := map[string]bool{ - "RotateKubeletServerCertificate": true, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - emptyProviderSpec := clusterv1alpha1.ProviderSpec{ - Value: &runtime.RawExtension{}, - } - test.spec.ProviderSpec = emptyProviderSpec - var cloudProvider *fakeCloudConfigProvider - if test.cloudProviderName != nil { - cloudProvider = &fakeCloudConfigProvider{ - name: *test.cloudProviderName, - config: "{config:true}", - err: nil, - } - } else { - cloudProvider = defaultCloudProvider - } - cloudConfig, cloudProviderName, err := cloudProvider.GetCloudConfig(test.spec) - if err != nil { - t.Fatalf("failed to get cloud config: %v", err) - } - - containerRuntimeOpts := containerruntime.Opts{ - ContainerRuntime: test.containerruntime, - InsecureRegistries: test.insecureRegistries, - RegistryMirrors: test.registryMirrors, - } - containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) - if err != nil { - t.Fatalf("failed to generate container runtime config: %v", err) - } - - req := plugin.UserDataRequest{ - MachineSpec: test.spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: cloudProviderName, - KubeletCloudProviderName: cloudProviderName, - DNSIPs: test.clusterDNSIPs, - ExternalCloudProvider: test.externalCloudProvider, - HTTPProxy: test.httpProxy, - NoProxy: test.noProxy, - PauseImage: test.pauseImage, - KubeletFeatureGates: kubeletFeatureGates, - ContainerRuntime: containerRuntimeConfig, - } - - s, err := provider.UserData(zap.NewNop().Sugar(), req) - if err != nil { - t.Errorf("error getting userdata: '%v'", err) - } - - // Check if we can gzip it. - if _, err := convert.GzipString(s); err != nil { - t.Fatal(err) - } - goldenName := test.name + ".yaml" - testhelper.CompareOutput(t, goldenName, s, *update) - }) - } -} - -// stringPtr returns pointer to given string. -func stringPtr(a string) *string { - return &a -} diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml deleted file mode 100644 index 43e619442..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.28-aws.yaml +++ /dev/null @@ -1,469 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=aws \ - --cloud-config=/etc/kubernetes/cloud-config \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-aws-external.yaml deleted file mode 100644 index 89cfc07d6..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-aws-external.yaml +++ /dev/null @@ -1,469 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=${KUBELET_HOSTNAME} \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-aws.yaml deleted file mode 100644 index a8fb80046..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-aws.yaml +++ /dev/null @@ -1,467 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {aws-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-nutanix.yaml deleted file mode 100644 index 81478aaf1..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-nutanix.yaml +++ /dev/null @@ -1,475 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - iscsi-initiator-utils \ - ipvsadm - systemctl enable --now iscsid - - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml deleted file mode 100644 index 2a3fd2368..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere-mirrors.yaml +++ /dev/null @@ -1,484 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 - - -ssh_pwauth: false - -write_files: -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry.docker-cn.com/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere-proxy.yaml deleted file mode 100644 index 9875e6e11..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere-proxy.yaml +++ /dev/null @@ -1,491 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 - - -ssh_pwauth: false - -write_files: -- path: "/etc/environment" - content: | - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - [plugins."io.containerd.grpc.v1.cri".registry.configs] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000".tls] - insecure_skip_verify = true - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000".tls] - insecure_skip_verify = true - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere.yaml deleted file mode 100644 index edf34abf3..000000000 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.30.0-vsphere.yaml +++ /dev/null @@ -1,475 +0,0 @@ -#cloud-config -bootcmd: -- modprobe ip_tables - -hostname: node1 - - -ssh_pwauth: false - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - tar \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - # Enable IPv6 and DHCPv6 on the default interface - grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE - grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE - grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE - - # Restart NetworkManager to apply for IPv6 configs - systemctl restart NetworkManager - # Let NetworkManager apply the DHCPv6 configs - sleep 3 - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/scripts/health-monitor.sh b/pkg/userdata/scripts/health-monitor.sh deleted file mode 100644 index 515610346..000000000 --- a/pkg/userdata/scripts/health-monitor.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2016 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is for master and node instance health monitoring, which is -# packed in kube-manifest tarball. It is executed through a systemd service -# in cluster/gce/gci/.yaml. The env variables come from an env -# file provided by the systemd service. - -# This script is a slightly adjusted version of -# https://github.com/kubernetes/kubernetes/blob/e1a1aa211224fcd9b213420b80b2ae680669683d/cluster/gce/gci/health-monitor.sh -# Adjustments are: -# * Kubelet health port is 10248 not 10255 -# * Removal of all all references to the KUBE_ENV file - -set -o nounset -set -o pipefail - -# We simply kill the process when there is a failure. Another systemd service will -# automatically restart the process. -function container_runtime_monitoring() { - local -r max_attempts=5 - local attempt=1 - local -r container_runtime_name="${CONTAINER_RUNTIME_NAME:-docker}" - # We still need to use 'docker ps' when container runtime is "docker". This is because - # dockershim is still part of kubelet today. When kubelet is down, crictl pods - # will also fail, and docker will be killed. This is undesirable especially when - # docker live restore is disabled. - local healthcheck_command="docker ps" - if [[ "${CONTAINER_RUNTIME:-docker}" != "docker" ]]; then - healthcheck_command="crictl pods" - fi - # Container runtime startup takes time. Make initial attempts before starting - # killing the container runtime. - until timeout 60 ${healthcheck_command} > /dev/null; do - if ((attempt == max_attempts)); then - echo "Max attempt ${max_attempts} reached! Proceeding to monitor container runtime healthiness." - break - fi - echo "$attempt initial attempt \"${healthcheck_command}\"! Trying again in $attempt seconds..." - sleep "$((2 ** attempt++))" - done - while true; do - if ! timeout 60 ${healthcheck_command} > /dev/null; then - echo "Container runtime ${container_runtime_name} failed!" - if [[ "$container_runtime_name" == "docker" ]]; then - # Dump stack of docker daemon for investigation. - # Log file name looks like goroutine-stacks-TIMESTAMP and will be saved to - # the exec root directory, which is /var/run/docker/ on Ubuntu and COS. - pkill -SIGUSR1 dockerd - fi - systemctl kill --kill-who=main "${container_runtime_name}" - # Wait for a while, as we don't want to kill it again before it is really up. - sleep 120 - else - sleep "${SLEEP_SECONDS}" - fi - done -} - -function kubelet_monitoring() { - echo "Wait for 2 minutes for kubelet to be functional" - # TODO(andyzheng0831): replace it with a more reliable method if possible. - sleep 120 - local -r max_seconds=10 - local output="" - while true; do - local failed=false - - if journalctl -u kubelet -n 1 | grep -q "use of closed network connection"; then - failed=true - echo "Kubelet stopped posting node status. Restarting" - elif ! output=$(curl -m "${max_seconds}" -f -s -S http://127.0.0.1:10248/healthz 2>&1); then - failed=true - # Print the response and/or errors. - echo "$output" - fi - - if [[ "$failed" == "true" ]]; then - echo "Kubelet is unhealthy!" - systemctl kill kubelet - # Wait for a while, as we don't want to kill it again before it is really up. - sleep 60 - else - sleep "${SLEEP_SECONDS}" - fi - done -} - -############## Main Function ################ -if [[ "$#" -ne 1 ]]; then - echo "Usage: health-monitor.sh " - exit 1 -fi - -SLEEP_SECONDS=10 -component=$1 -echo "Start kubernetes health monitoring for ${component}" -if [[ "${component}" == "container-runtime" ]]; then - container_runtime_monitoring -elif [[ "${component}" == "kubelet" ]]; then - kubelet_monitoring -else - echo "Health monitoring for component ${component} is not supported!" -fi diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go deleted file mode 100644 index 47dbc0107..000000000 --- a/pkg/userdata/ubuntu/provider.go +++ /dev/null @@ -1,380 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for Ubuntu. -// - -package ubuntu - -import ( - "errors" - "fmt" - "strings" - "text/template" - - "github.com/Masterminds/semver/v3" - "go.uber.org/zap" - - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - userdatahelper "github.com/kubermatic/machine-controller/pkg/userdata/helper" -) - -// Provider is a pkg/userdata/plugin.Provider implementation. -type Provider struct{} - -// UserData renders user-data template to string. -func (p Provider) UserData(log *zap.SugaredLogger, req plugin.UserDataRequest) (string, error) { - tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap(log)).Parse(userDataTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse user-data template: %w", err) - } - - kubeletVersion, err := semver.NewVersion(req.MachineSpec.Versions.Kubelet) - if err != nil { - return "", fmt.Errorf("invalid kubelet version: %w", err) - } - - pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) - if err != nil { - return "", fmt.Errorf("failed to get providerSpec: %w", err) - } - - if pconfig.OverwriteCloudConfig != nil { - req.CloudConfig = *pconfig.OverwriteCloudConfig - } - - if pconfig.Network.IsStaticIPConfig() { - return "", errors.New("static IP config is not supported with Ubuntu") - } - - ubuntuConfig, err := LoadConfig(pconfig.OperatingSystemSpec) - if err != nil { - return "", fmt.Errorf("failed to get ubuntu config from provider config: %w", err) - } - - kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) - if err != nil { - return "", err - } - - kubernetesCACert, err := userdatahelper.GetCACert(req.Kubeconfig) - if err != nil { - return "", fmt.Errorf("error extracting cacert: %w", err) - } - - crEngine := req.ContainerRuntime.Engine() - crScript, err := crEngine.ScriptFor(providerconfigtypes.OperatingSystemUbuntu) - if err != nil { - return "", fmt.Errorf("failed to generate container runtime install script: %w", err) - } - - crConfig, err := crEngine.Config() - if err != nil { - return "", fmt.Errorf("failed to generate container runtime config: %w", err) - } - - crAuthConfig, err := crEngine.AuthConfig() - if err != nil { - return "", fmt.Errorf("failed to generate container runtime auth config: %w", err) - } - - data := struct { - plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - OSConfig *Config - KubeletVersion string - Kubeconfig string - KubernetesCACert string - NodeIPScript string - ExtraKubeletFlags []string - ContainerRuntimeScript string - ContainerRuntimeConfigFileName string - ContainerRuntimeConfig string - ContainerRuntimeAuthConfigFileName string - ContainerRuntimeAuthConfig string - ContainerRuntimeName string - }{ - UserDataRequest: req, - ProviderSpec: pconfig, - OSConfig: ubuntuConfig, - KubeletVersion: kubeletVersion.String(), - Kubeconfig: kubeconfigString, - KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(pconfig.Network.GetIPFamily()), - ExtraKubeletFlags: crEngine.KubeletFlags(), - ContainerRuntimeScript: crScript, - ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), - ContainerRuntimeConfig: crConfig, - ContainerRuntimeAuthConfigFileName: crEngine.AuthConfigFileName(), - ContainerRuntimeAuthConfig: crAuthConfig, - ContainerRuntimeName: crEngine.String(), - } - - var buf strings.Builder - if err = tmpl.Execute(&buf, data); err != nil { - return "", fmt.Errorf("failed to execute user-data template: %w", err) - } - - return userdatahelper.CleanupTemplateOutput(buf.String()) -} - -// UserData template. -const userDataTemplate = `#cloud-config -{{ if ne .CloudProviderName "aws" }} -hostname: {{ .MachineSpec.Name }} -{{- /* Never set the hostname on AWS nodes. Kubernetes(kube-proxy) requires the hostname to be the private dns name */}} -{{ end }} - -{{- if .OSConfig.DistUpgradeOnBoot }} -package_upgrade: true -package_reboot_if_required: true -{{- end }} - -ssh_pwauth: false - -{{- if .ProviderSpec.SSHPublicKeys }} -ssh_authorized_keys: -{{- range .ProviderSpec.SSHPublicKeys }} -- "{{ . }}" -{{- end }} -{{- end }} - -write_files: -{{- if .HTTPProxy }} -- path: "/etc/environment" - content: | - PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" -{{ proxyEnvironment .HTTPProxy .NoProxy | indent 4 }} -{{- end }} - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | -{{ journalDConfig | indent 4 }} - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | -{{ kernelModulesScript | indent 4 }} - -- path: "/etc/sysctl.d/k8s.conf" - content: | -{{ kernelSettings | indent 4 }} - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - -{{- /* As we added some modules and don't want to reboot, restart the service */}} - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - {{- if or (eq .CloudProviderName "vsphere") (eq .CloudProviderName "vmware-cloud-director") }} - open-vm-tools \ - {{- end }} - {{- if eq .CloudProviderName "nutanix" }} - open-iscsi \ - {{- end }} - ipvsadm - - {{- /* iscsid service is required on Nutanix machines for CSI driver to attach volumes. */}} - {{- if eq .CloudProviderName "nutanix" }} - systemctl enable --now iscsid - {{ end }} - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - {{ if eq .CloudProviderName "alibaba" }} - if grep -v -q swapaccount=1 /proc/cmdline - then - echo "Reboot system if not alibaba cloud" - update-grub - touch /var/run/reboot-required - fi - {{ end }} -{{ .ContainerRuntimeScript | indent 4 }} - -{{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - {{- if eq .CloudProviderName "kubevirt" }} - systemctl enable --now --no-block restart-kubelet.service - {{ end }} - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -{{- if ne (len .CloudConfig) 0 }} -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | -{{ .CloudConfig | indent 4 }} -{{- end }} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | -{{ .NodeIPScript | indent 4 }} - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | -{{ .Kubeconfig | indent 4 }} - -- path: "/etc/kubernetes/pki/ca.crt" - content: | -{{ .KubernetesCACert | indent 4 }} - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: {{ .ContainerRuntimeConfigFileName }} - permissions: "0644" - content: | -{{ .ContainerRuntimeConfig | indent 4 }} - -- path: "/etc/kubernetes/kubelet.conf" - content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | -{{ kubeletHealthCheckSystemdUnit | indent 4 }} - -{{- with .ProviderSpec.CAPublicKey }} - -- path: "/etc/ssh/trusted-user-ca-keys.pem" - content: | -{{ . | indent 4 }} - -- path: "/etc/ssh/sshd_config" - content: | -{{ sshConfigAddendum | indent 4 }} - append: true -{{- end }} - -{{- if eq .CloudProviderName "kubevirt" }} -- path: "/opt/bin/restart-kubelet.sh" - permissions: "0744" - content: | - #!/bin/bash - # Needed for Kubevirt provider because if the virt-launcher pod is deleted, - # the VM and DataVolume states are kept and VM is rebooted. We need to restart the kubelet - # with the new config (new IP) and run this at every boot. - set -xeuo pipefail - - # This helps us avoid an unnecessary restart for kubelet on the first boot - if [ -f /etc/kubelet_needs_restart ]; then - # restart kubelet since it's not the first boot - systemctl daemon-reload - systemctl restart kubelet.service - else - touch /etc/kubelet_needs_restart - fi - -- path: "/etc/systemd/system/restart-kubelet.service" - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - Description=Service responsible for restarting kubelet when the machine is rebooted - - [Service] - Type=oneshot - ExecStart=/opt/bin/restart-kubelet.sh - - [Install] - WantedBy=multi-user.target -{{- end }} - -runcmd: -- systemctl enable --now setup.service -` diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go deleted file mode 100644 index eef43a2b3..000000000 --- a/pkg/userdata/ubuntu/provider_test.go +++ /dev/null @@ -1,693 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for Ubuntu. -// - -package ubuntu - -import ( - "encoding/json" - "flag" - "fmt" - "net" - "testing" - - "github.com/Masterminds/semver/v3" - "go.uber.org/zap" - - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - "github.com/kubermatic/machine-controller/pkg/containerruntime" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - testhelper "github.com/kubermatic/machine-controller/pkg/test" - "github.com/kubermatic/machine-controller/pkg/userdata/cloud" - "github.com/kubermatic/machine-controller/pkg/userdata/convert" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" -) - -var ( - update = flag.Bool("update", false, "update testdata files") - - pemCertificate = `-----BEGIN CERTIFICATE----- -MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV -BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG -A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 -DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 -NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG -cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv -c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B -AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS -R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT -ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk -JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 -mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW -caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G -A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt -hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB -MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES -MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv -bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h -U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao -eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 -UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD -58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n -sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF -kPe6XoSbiLm/kxk32T0= ------END CERTIFICATE-----` - - kubeconfig = &clientcmdapi.Config{ - Clusters: map[string]*clientcmdapi.Cluster{ - "": { - Server: "https://server/", - CertificateAuthorityData: []byte(pemCertificate), - }, - }, - AuthInfos: map[string]*clientcmdapi.AuthInfo{ - "": { - Token: "my-token", - }, - }, - } - - kubeletFeatureGates = map[string]bool{ - "RotateKubeletServerCertificate": true, - } -) - -const ( - defaultVersion = "1.29.0" -) - -type fakeCloudConfigProvider struct { - config string - name string - err error -} - -func (p *fakeCloudConfigProvider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return p.config, p.name, p.err -} - -// userDataTestCase contains the data for a table-driven test. -type userDataTestCase struct { - name string - spec clusterv1alpha1.MachineSpec - ccProvider cloud.ConfigProvider - osConfig *Config - providerSpec *providerconfigtypes.Config - DNSIPs []net.IP - kubernetesCACert string - externalCloudProvider bool - httpProxy string - noProxy string - insecureRegistries string - registryMirrors string - containerdRegistryMirrors containerruntime.RegistryMirrorsFlags - registryCredentials map[string]containerruntime.AuthConfig - pauseImage string - containerruntime string -} - -func simpleVersionTests() []userDataTestCase { - versions := []*semver.Version{ - semver.MustParse("v1.27.0"), - semver.MustParse("v1.28.0"), - semver.MustParse("v1.29.0"), - semver.MustParse("v1.30.0"), - } - - var tests []userDataTestCase - for _, v := range versions { - tests = append(tests, userDataTestCase{ - name: fmt.Sprintf("version-%s", v.String()), - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: v.String(), - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }) - } - - return tests -} - -// TestUserDataGeneration runs the data generation for different -// environments. -func TestUserDataGeneration(t *testing.T) { - t.Parallel() - - tests := simpleVersionTests() - tests = append(tests, []userDataTestCase{ - { - name: "dist-upgrade-on-boot", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: true, - }, - }, - { - name: "multiple-dns-servers", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "kubelet-version-without-v-prefix", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.27.0", - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "multiple-ssh-keys", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD", "ssh-rsa EEEFFF"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.27.0", - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "openstack", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "openstack", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "openstack", - config: "{openstack-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "openstack-dualstack", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "openstack", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - Network: &providerconfigtypes.NetworkConfig{ - IPFamily: util.IPFamilyIPv4IPv6, - }, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "openstack", - config: "{openstack-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - externalCloudProvider: true, - }, - { - name: "digitalocean-dualstack", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "digitalocean", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - Network: &providerconfigtypes.NetworkConfig{ - IPFamily: util.IPFamilyIPv4IPv6, - }, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - config: "{digitalocean-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "openstack-dualstack-IPv6+IPv4", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "openstack", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - Network: &providerconfigtypes.NetworkConfig{ - IPFamily: util.IPFamilyIPv6IPv4, - }, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "openstack", - config: "{openstack-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - externalCloudProvider: true, - }, - { - name: "digitalocean-dualstack-IPv6+IPv4", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "digitalocean", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - Network: &providerconfigtypes.NetworkConfig{ - IPFamily: util.IPFamilyIPv6IPv4, - }, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - config: "{digitalocean-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "openstack-overwrite-cloud-config", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "openstack", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: stringPtr("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.27.0", - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "openstack", - config: "{openstack-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "vsphere", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: stringPtr("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.27.0", - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "vsphere-proxy", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: stringPtr("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.27.0", - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - httpProxy: "http://192.168.100.100:3128/", - noProxy: "192.168.1.0", - insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "vsphere-mirrors", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: stringPtr("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.27.0", - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - httpProxy: "http://192.168.100.100:3128/", - noProxy: "192.168.1.0", - registryMirrors: "https://registry.docker-cn.com/", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "containerd", - containerruntime: "containerd", - registryCredentials: map[string]containerruntime.AuthConfig{ - "docker.io": { - Username: "login1", - Password: "passwd1", - }, - }, - insecureRegistries: "k8s.gcr.io", - containerdRegistryMirrors: map[string][]string{ - "k8s.gcr.io": {"https://intranet.local/"}, - }, - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: true, - }, - }, - { - name: "docker", - containerruntime: "docker", - registryCredentials: map[string]containerruntime.AuthConfig{ - "docker.io": { - Username: "login1", - Password: "passwd1", - }, - }, - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: true, - }, - }, - { - name: "nutanix", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "nutanix", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: stringPtr("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.29.0", - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "nutanix", - config: "{nutanix-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - }...) - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - rProviderSpec := test.providerSpec - osConfigByte, err := json.Marshal(test.osConfig) - if err != nil { - t.Fatal(err) - } - rProviderSpec.OperatingSystemSpec = runtime.RawExtension{ - Raw: osConfigByte, - } - - providerSpecRaw, err := json.Marshal(rProviderSpec) - if err != nil { - t.Fatal(err) - } - test.spec.ProviderSpec = clusterv1alpha1.ProviderSpec{ - Value: &runtime.RawExtension{ - Raw: providerSpecRaw, - }, - } - provider := Provider{} - - cloudConfig, cloudProviderName, err := test.ccProvider.GetCloudConfig(test.spec) - if err != nil { - t.Fatalf("failed to get cloud config: %v", err) - } - - containerRuntimeOpts := containerruntime.Opts{ - ContainerRuntime: test.containerruntime, - InsecureRegistries: test.insecureRegistries, - RegistryMirrors: test.registryMirrors, - ContainerdRegistryMirrors: test.containerdRegistryMirrors, - } - containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) - if err != nil { - t.Fatalf("failed to generate container runtime config: %v", err) - } - containerRuntimeConfig.RegistryCredentials = test.registryCredentials - - req := plugin.UserDataRequest{ - MachineSpec: test.spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: cloudProviderName, - KubeletCloudProviderName: cloudProviderName, - DNSIPs: test.DNSIPs, - ExternalCloudProvider: test.externalCloudProvider, - HTTPProxy: test.httpProxy, - NoProxy: test.noProxy, - PauseImage: test.pauseImage, - KubeletFeatureGates: kubeletFeatureGates, - ContainerRuntime: containerRuntimeConfig, - } - s, err := provider.UserData(zap.NewNop().Sugar(), req) - if err != nil { - t.Fatal(err) - } - - // Check if we can gzip it. - if _, err := convert.GzipString(s); err != nil { - t.Fatal(err) - } - goldenName := test.name + ".yaml" - testhelper.CompareOutput(t, goldenName, s, *update) - }) - } -} - -// stringPtr returns pointer to given string. -func stringPtr(str string) *string { - return &str -} diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml deleted file mode 100644 index f22f8e9b3..000000000 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ /dev/null @@ -1,473 +0,0 @@ -#cloud-config - -hostname: node1 - -package_upgrade: true -package_reboot_if_required: true - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"] - endpoint = ["https://intranet.local/"] - [plugins."io.containerd.grpc.v1.cri".registry.configs] - [plugins."io.containerd.grpc.v1.cri".registry.configs."docker.io"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."docker.io".auth] - username = "login1" - password = "passwd1" - [plugins."io.containerd.grpc.v1.cri".registry.configs."k8s.gcr.io"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."k8s.gcr.io".tls] - insecure_skip_verify = true - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml deleted file mode 100644 index 0efa6733d..000000000 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml +++ /dev/null @@ -1,474 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {digitalocean-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") - if [ -z "${DEFAULT_IFC_IP6}" ] - then - echodate "Failed to get IPv6 address for the default route interface" - exit 1 - fi - DEFAULT_IFC_IP=$DEFAULT_IFC_IP6,$DEFAULT_IFC_IP - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - - 10.10.10.11 - - 10.10.10.12 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml deleted file mode 100644 index a219cc83f..000000000 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ /dev/null @@ -1,474 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {digitalocean-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") - if [ -z "${DEFAULT_IFC_IP6}" ] - then - echodate "Failed to get IPv6 address for the default route interface" - exit 1 - fi - DEFAULT_IFC_IP=$DEFAULT_IFC_IP,$DEFAULT_IFC_IP6 - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - - 10.10.10.11 - - 10.10.10.12 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml deleted file mode 100644 index 4ecb2c10c..000000000 --- a/pkg/userdata/ubuntu/testdata/dist-upgrade-on-boot.yaml +++ /dev/null @@ -1,463 +0,0 @@ -#cloud-config - -hostname: node1 - -package_upgrade: true -package_reboot_if_required: true - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/docker.yaml b/pkg/userdata/ubuntu/testdata/docker.yaml deleted file mode 100644 index afc0ca39e..000000000 --- a/pkg/userdata/ubuntu/testdata/docker.yaml +++ /dev/null @@ -1,468 +0,0 @@ -#cloud-config - -hostname: node1 - -package_upgrade: true -package_reboot_if_required: true - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - [plugins."io.containerd.grpc.v1.cri".registry.configs] - [plugins."io.containerd.grpc.v1.cri".registry.configs."docker.io"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."docker.io".auth] - username = "login1" - password = "passwd1" - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml deleted file mode 100644 index 64bdb9fa0..000000000 --- a/pkg/userdata/ubuntu/testdata/kubelet-version-without-v-prefix.yaml +++ /dev/null @@ -1,461 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml b/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml deleted file mode 100644 index d6bcc6537..000000000 --- a/pkg/userdata/ubuntu/testdata/multiple-dns-servers.yaml +++ /dev/null @@ -1,463 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - - 10.10.10.11 - - 10.10.10.12 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml b/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml deleted file mode 100644 index 193977e44..000000000 --- a/pkg/userdata/ubuntu/testdata/multiple-ssh-keys.yaml +++ /dev/null @@ -1,463 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" -- "ssh-rsa CCCDDD" -- "ssh-rsa EEEFFF" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/nutanix.yaml b/pkg/userdata/ubuntu/testdata/nutanix.yaml deleted file mode 100644 index 6dd5c6aa0..000000000 --- a/pkg/userdata/ubuntu/testdata/nutanix.yaml +++ /dev/null @@ -1,470 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - open-iscsi \ - ipvsadm - systemctl enable --now iscsid - - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml deleted file mode 100644 index 7c24e7fc0..000000000 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack-IPv6+IPv4.yaml +++ /dev/null @@ -1,474 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {openstack-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") - if [ -z "${DEFAULT_IFC_IP6}" ] - then - echodate "Failed to get IPv6 address for the default route interface" - exit 1 - fi - DEFAULT_IFC_IP=$DEFAULT_IFC_IP6,$DEFAULT_IFC_IP - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - - 10.10.10.11 - - 10.10.10.12 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml deleted file mode 100644 index a45955b95..000000000 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml +++ /dev/null @@ -1,474 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=external \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {openstack-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") - if [ -z "${DEFAULT_IFC_IP6}" ] - then - echodate "Failed to get IPv6 address for the default route interface" - exit 1 - fi - DEFAULT_IFC_IP=$DEFAULT_IFC_IP,$DEFAULT_IFC_IP6 - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - - 10.10.10.11 - - 10.10.10.12 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml deleted file mode 100644 index b21555d8d..000000000 --- a/pkg/userdata/ubuntu/testdata/openstack-overwrite-cloud-config.yaml +++ /dev/null @@ -1,469 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=openstack \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/openstack.yaml b/pkg/userdata/ubuntu/testdata/openstack.yaml deleted file mode 100644 index 08decba36..000000000 --- a/pkg/userdata/ubuntu/testdata/openstack.yaml +++ /dev/null @@ -1,467 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {openstack-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - - 10.10.10.11 - - 10.10.10.12 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.27.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.27.0.yaml deleted file mode 100644 index 64bdb9fa0..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.27.0.yaml +++ /dev/null @@ -1,461 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.28.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.28.0.yaml deleted file mode 100644 index 2b496a23b..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.28.0.yaml +++ /dev/null @@ -1,461 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.29.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.29.0.yaml deleted file mode 100644 index 5c3f6533b..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.29.0.yaml +++ /dev/null @@ -1,461 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/version-1.30.0.yaml b/pkg/userdata/ubuntu/testdata/version-1.30.0.yaml deleted file mode 100644 index e450b0951..000000000 --- a/pkg/userdata/ubuntu/testdata/version-1.30.0.yaml +++ /dev/null @@ -1,461 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml b/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml deleted file mode 100644 index ff6e90790..000000000 --- a/pkg/userdata/ubuntu/testdata/vsphere-mirrors.yaml +++ /dev/null @@ -1,480 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: -- path: "/etc/environment" - content: | - PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - open-vm-tools \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry.docker-cn.com/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml b/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml deleted file mode 100644 index 275591c13..000000000 --- a/pkg/userdata/ubuntu/testdata/vsphere-proxy.yaml +++ /dev/null @@ -1,487 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: -- path: "/etc/environment" - content: | - PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - open-vm-tools \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - [plugins."io.containerd.grpc.v1.cri".registry.configs] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000".tls] - insecure_skip_verify = true - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000"] - [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000".tls] - insecure_skip_verify = true - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/vsphere.yaml b/pkg/userdata/ubuntu/testdata/vsphere.yaml deleted file mode 100644 index 5c858de6d..000000000 --- a/pkg/userdata/ubuntu/testdata/vsphere.yaml +++ /dev/null @@ -1,470 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/etc/default/grub.d/60-swap-accounting.cfg" - content: | - # Added by kubermatic machine-controller - # Enable cgroups memory and swap accounting - GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - if systemctl is-active ufw; then systemctl stop ufw; fi - systemctl mask ufw - systemctl restart systemd-modules-load.service - sysctl --system - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ - curl \ - ca-certificates \ - ceph-common \ - cifs-utils \ - conntrack \ - e2fsprogs \ - ebtables \ - ethtool \ - glusterfs-client \ - iptables \ - jq \ - kmod \ - openssh-client \ - nfs-common \ - socat \ - util-linux \ - open-vm-tools \ - ipvsadm - - # Update grub to include kernel command options to enable swap accounting. - # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 - - - apt-get update - apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io/"] - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - containerRuntimeEndpoint: "" - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMaximumGCAge: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - text: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index d41bc76ae..f1eeaa32c 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -415,10 +415,6 @@ func TestAWSProvisioningE2E(t *testing.T) { t.Parallel() provisioningUtility := flatcar.Ignition - // `OPERATING_SYSTEM_MANAGER` will be false when legacy machine-controller userdata should be used for E2E tests. - if v := os.Getenv("OPERATING_SYSTEM_MANAGER"); v == "false" { - provisioningUtility = flatcar.CloudInit - } // test data awsKeyID := os.Getenv("AWS_E2E_TESTS_KEY_ID") From 31463a80af435b2b670e9a816de56372db221592 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 6 May 2024 15:26:49 +0500 Subject: [PATCH 386/489] Fix namespace for machinesets (#1792) Signed-off-by: Waleed Malik --- pkg/controller/util/machine.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/util/machine.go b/pkg/controller/util/machine.go index a8e9b090a..69a2a833b 100644 --- a/pkg/controller/util/machine.go +++ b/pkg/controller/util/machine.go @@ -39,7 +39,7 @@ func GetMachineDeploymentNameAndRevisionForMachine(ctx context.Context, machine if machineSetName != "" { machineSet := &clusterv1alpha1.MachineSet{} - if err := c.Get(ctx, types.NamespacedName{Name: machineSetName, Namespace: "kube-system"}, machineSet); err != nil { + if err := c.Get(ctx, types.NamespacedName{Name: machineSetName, Namespace: machine.Namespace}, machineSet); err != nil { return "", "", err } From f7184855150359fac14c5003f11a6fb7bc00efeb Mon Sep 17 00:00:00 2001 From: Mara Sophie Grosch Date: Tue, 14 May 2024 13:47:58 +0200 Subject: [PATCH 387/489] Anexia: various patches (request/response logging, CPU performance type, 404 fix on delete) (#1797) * Anexia: really handle 404 response for DELETE There already was a check for a 404 error being returned, but the client library does not actually return 404 as error. Workaround is needed, as that won't be fixed in go-anxcloud, as it's the legacy client - but nicely commented and good workaround. Signed-off-by: Mara Sophie Grosch * Anexia: use cloudproviderutil.HttpClient This way we get request/response logging if required - even prefixed with the Machine name, if applicable. Signed-off-by: Mara Sophie Grosch * Anexia: allow to specify CPU performance type Signed-off-by: Mara Sophie Grosch --------- Signed-off-by: Mara Sophie Grosch Co-authored-by: Mara Sophie Grosch --- examples/anexia-machinedeployment.yaml | 5 ++ pkg/cloudprovider/provider/anexia/provider.go | 46 +++++++++++++++---- .../provider/anexia/types/types.go | 5 +- 3 files changed, 45 insertions(+), 11 deletions(-) diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index 3affb6ab9..f1366e358 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -39,6 +39,11 @@ spec: cpus: 2 memory: 2048 + # this defaults to "performance", but you can set anything + # supported by the Anexia Engine here - or not set this attribute + # at all + cpuPerformanceType: standard + disks: - size: 60 performanceType: ENT6 diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index ea1017797..c989dcdc7 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -48,6 +48,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + cloudproviderutil "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "k8s.io/apimachinery/pkg/api/meta" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -107,7 +108,7 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * Machine: machine, }) - _, client, err := getClient(config.Token) + _, client, err := getClient(config.Token, &machine.Name) if err != nil { return nil, err } @@ -161,6 +162,10 @@ func provisionVM(ctx context.Context, log *zap.SugaredLogger, client anxclient.C vm.DiskType = config.Disks[0].PerformanceType + if config.CPUPerformanceType != "" { + vm.CPUPerformanceType = config.CPUPerformanceType + } + for _, disk := range config.Disks[1:] { vm.AdditionalDisks = append(vm.AdditionalDisks, anxvm.AdditionalDisk{ SizeGBs: disk.Size, @@ -334,7 +339,7 @@ func (p *provider) resolveConfig(ctx context.Context, log *zap.SugaredLogger, co // when "templateID" is not set, we expect "template" to be if ret.TemplateID == "" { - a, _, err := getClient(ret.Token) + a, _, err := getClient(ret.Token, nil) if err != nil { return nil, fmt.Errorf("failed initializing API clients: %w", err) } @@ -467,7 +472,7 @@ func (p *provider) Get(ctx context.Context, log *zap.SugaredLogger, machine *clu return nil, newError(common.InvalidConfigurationMachineError, "failed to retrieve config: %v", err) } - _, cli, err := getClient(config.Token) + _, cli, err := getClient(config.Token, &machine.Name) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, "failed to create Anexia client: %v", err) } @@ -550,7 +555,7 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine return false, newError(common.InvalidConfigurationMachineError, "failed to parse MachineSpec: %v", err) } - _, cli, err := getClient(config.Token) + _, cli, err := getClient(config.Token, &machine.Name) if err != nil { return false, newError(common.InvalidConfigurationMachineError, "failed to create Anexia client: %v", err) } @@ -565,10 +570,20 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine response, err := vsphereAPI.Provisioning().VM().Deprovision(deleteCtx, status.InstanceID, false) if err != nil { var respErr *anxclient.ResponseError + // Only error if the error was not "not found" if !(errors.As(err, &respErr) && respErr.ErrorData.Code == http.StatusNotFound) { return false, newError(common.DeleteMachineError, "failed to delete machine: %v", err) } + + // good thinking checking for a "not found" error, but go-anxcloud does only + // return >= 500 && < 600 errors (: + // since that's the legacy client in go-anxcloud and the new one is not yet available, + // this will not be fixed there but we have a nice workaround here: + + if response.Identifier == "" { + return true, nil + } } status.DeprovisioningID = response.Identifier } @@ -606,16 +621,29 @@ func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } -func getClient(token string) (api.API, anxclient.Client, error) { - tokenOpt := anxclient.TokenFromString(token) - client := anxclient.HTTPClient(&http.Client{Timeout: 120 * time.Second}) +func getClient(token string, machineName *string) (api.API, anxclient.Client, error) { + logPrefix := "[Anexia API]" + + if machineName != nil { + logPrefix = fmt.Sprintf("[Anexia API for Machine %q]", *machineName) + } + + httpClient := cloudproviderutil.HTTPClientConfig{ + Timeout: 120 * time.Second, + LogPrefix: logPrefix, + }.New() + + legacyClientOptions := []anxclient.Option{ + anxclient.TokenFromString(token), + anxclient.HTTPClient(&httpClient), + } - a, err := api.NewAPI(api.WithClientOptions(client, tokenOpt)) + a, err := api.NewAPI(api.WithClientOptions(legacyClientOptions...)) if err != nil { return nil, nil, fmt.Errorf("error creating generic API client: %w", err) } - legacyClient, err := anxclient.New(tokenOpt, client) + legacyClient, err := anxclient.New(legacyClientOptions...) if err != nil { return nil, nil, fmt.Errorf("error creating legacy client: %w", err) } diff --git a/pkg/cloudprovider/provider/anexia/types/types.go b/pkg/cloudprovider/provider/anexia/types/types.go index f97a5389a..04cb2c992 100644 --- a/pkg/cloudprovider/provider/anexia/types/types.go +++ b/pkg/cloudprovider/provider/anexia/types/types.go @@ -63,8 +63,9 @@ type RawConfig struct { Template providerconfigtypes.ConfigVarString `json:"template"` TemplateBuild providerconfigtypes.ConfigVarString `json:"templateBuild"` - CPUs int `json:"cpus"` - Memory int `json:"memory"` + CPUs int `json:"cpus"` + CPUPerformanceType string `json:"cpuPerformanceType"` + Memory int `json:"memory"` // Deprecated, use Disks instead. DiskSize int `json:"diskSize"` From 762be54fc9813e53bc56715d3026cb2eea49ea52 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Fri, 31 May 2024 13:30:07 +0200 Subject: [PATCH 388/489] update to controller-runtime 0.18.2 (#1786) --- go.mod | 17 +---- go.sum | 56 +-------------- pkg/cloudprovider/provider/anexia/provider.go | 2 +- pkg/cloudprovider/provider/edge/provider.go | 1 + pkg/controller/machine/controller.go | 71 +++++++++---------- .../machinedeployment/controller.go | 62 ++++++---------- pkg/controller/machineset/controller.go | 58 ++++++--------- pkg/controller/nodecsrapprover/controller.go | 28 ++++---- 8 files changed, 91 insertions(+), 204 deletions(-) diff --git a/go.mod b/go.mod index 972fbec4c..37b7d00b9 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/aws/smithy-go v1.20.1 github.com/davecgh/go-spew v1.1.1 github.com/digitalocean/godo v1.110.0 - github.com/flatcar/container-linux-config-transpiler v0.9.4 github.com/go-logr/logr v1.4.1 github.com/go-logr/zapr v1.3.0 github.com/go-test/deep v1.1.0 @@ -61,13 +60,10 @@ require ( k8s.io/client-go v0.30.0 k8s.io/cloud-provider v0.30.0 k8s.io/klog v1.0.0 - k8s.io/kubelet v0.30.0 k8s.io/utils v0.0.0-20240310230437-4693a0247e57 kubevirt.io/api v1.2.0 kubevirt.io/containerized-data-importer-api v1.58.1 - // Pinned due to a breaking change in k8s.io/client-go/tools/leaderelection in v0.30.0 - // TODO: Update to the latest semver version when https://github.com/kubernetes-sigs/controller-runtime/pull/2693 is released - sigs.k8s.io/controller-runtime v0.17.1-0.20240418082203-04706074d2f1 + sigs.k8s.io/controller-runtime v0.18.2 sigs.k8s.io/yaml v1.4.0 ) @@ -87,8 +83,6 @@ require ( github.com/Masterminds/goutils v1.1.1 // indirect github.com/PaesslerAG/gval v1.2.2 // indirect github.com/PaesslerAG/jsonpath v0.1.1 // indirect - github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect - github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3 // indirect @@ -99,10 +93,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/coreos/go-semver v0.3.1 // indirect - github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/distribution/reference v0.5.0 // indirect github.com/docker/distribution v2.8.3+incompatible // indirect @@ -110,7 +101,6 @@ require ( github.com/evanphx/json-patch v5.7.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/flatcar/ignition v0.36.2 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.20.2 // indirect @@ -135,7 +125,6 @@ require ( github.com/hashicorp/go-version v1.6.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.16 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -163,8 +152,6 @@ require ( github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cast v1.6.0 // indirect - github.com/spf13/cobra v1.8.0 // indirect - github.com/vincent-petithory/dataurl v1.0.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect @@ -173,7 +160,6 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/sync v0.6.0 // indirect @@ -192,7 +178,6 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/component-base v0.30.0 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect diff --git a/go.sum b/go.sum index 28edabc29..4d8f9a6e0 100644 --- a/go.sum +++ b/go.sum @@ -110,18 +110,12 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/agnivade/levenshtein v1.1.0 h1:n6qGwyHG61v3ABce1rPVZklEYRT8NFpCMrpZdBUbYGM= github.com/agnivade/levenshtein v1.1.0/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= -github.com/ajeddeloh/go-json v0.0.0-20160803184958-73d058cf8437/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= -github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 h1:4SPQljF/GJ8Q+QlCWMWxRBepub4DresnOm4eI2ebFGc= -github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 h1:ez/4by2iGztzR4L0zgAOR8lTQK9VlyBVVd7G4omaOQs= -github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/aliyun/alibaba-cloud-sdk-go v1.62.695 h1:Lk9qjMhhkzZaD4eyx23v0E2+4nAIfwreJ/ecKdaTU6E= github.com/aliyun/alibaba-cloud-sdk-go v1.62.695/go.mod h1:CJJYa1ZMxjlN/NbXEwmejEnBkhi0DV+Yb3B2lxf+74o= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= @@ -132,7 +126,6 @@ github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoU github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/aws/aws-sdk-go v1.42.23/go.mod h1:gyRszuZ/icHmHAVE4gc/r+cfCmhA1AD+vqfWbgI+eHs= github.com/aws/aws-sdk-go-v2 v1.25.3 h1:xYiLpZTQs1mzvz5PaI6uR0Wh57ippuEthxS4iK5v0n0= github.com/aws/aws-sdk-go-v2 v1.25.3/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I= @@ -167,8 +160,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= -github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bnkamalesh/webgo/v4 v4.1.11/go.mod h1:taIAonQTzao8G5rnB22WgKmQuIOWHpQ0n/YLAidBXlM= github.com/bnkamalesh/webgo/v6 v6.2.2/go.mod h1:2Y+dEdTp1xC/ra+3PAVZV6hh4sCI+iPK7mcHt+t9bfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -189,16 +180,8 @@ github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/coreos/go-semver v0.1.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= -github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= -github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creasty/defaults v1.5.2 h1:/VfB6uxpyp6h0fr7SPp7n8WJBoV8jfxQXPCnkVSjyls= github.com/creasty/defaults v1.5.2/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY= @@ -240,10 +223,6 @@ github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0 github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flatcar/container-linux-config-transpiler v0.9.4 h1:yXQ0NB8PeNrKJPrZvbv5/DV63PNhTqt8vaf8YxmX/RA= -github.com/flatcar/container-linux-config-transpiler v0.9.4/go.mod h1:LxanhPvXkWgHG9PrkT4rX/p7YhUPdDGGsUdkNpV3L5U= -github.com/flatcar/ignition v0.36.2 h1:xGHgScUe0P4Fkprjqv7L2CE58emiQgP833OCCn9z2v4= -github.com/flatcar/ignition v0.36.2/go.mod h1:uk1tpzLFRXus4RrvzgMI+IqmmB8a/RGFSBlI+tMTbbA= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -265,7 +244,6 @@ github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= @@ -319,7 +297,6 @@ github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncV github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/goccy/go-json v0.9.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -428,7 +405,6 @@ github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2c github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gophercloud/gophercloud v1.11.0 h1:ls0O747DIq1D8SUHc7r2vI8BFbMLeLFuENaAIfEx7OM= github.com/gophercloud/gophercloud v1.11.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -461,11 +437,8 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -483,7 +456,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= @@ -607,13 +579,11 @@ github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046 h1:zF+CUhv8LMpqTFFp github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046/go.mod h1:W/xTaqgJ2kJCwayvm3BF3bOj9ku0F5DjjYnZaioxnOk= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterhellberg/link v1.2.0 h1:UA5pg3Gp/E0F2WdX7GERiNrPQrM1K6CVJUUWfHa4t6c= github.com/peterhellberg/link v1.2.0/go.mod h1:gYfAh+oJgQu2SrZHg5hROVRQe1ICoK0/HHJTcE0edxc= -github.com/pin/tftp v2.1.0+incompatible/go.mod h1:xVpZOMCXTy+A5QMjEVN0Glwa1sUvaJhFXbr/aAxuxGY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -659,7 +629,6 @@ github.com/rollbar/rollbar-go v1.4.2/go.mod h1:kLQ9gP3WCRGrvJmF0ueO3wK9xWocej8GR github.com/rollbar/rollbar-go/errors v0.0.0-20210929193720-32947096267e/go.mod h1:Ie0xEc1Cyj+T4XMO8s0Vf7pMfvSAAy1sb4AYc8aJsao= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25 h1:/8rfZAdFfafRXOgz+ZpMZZWZ5pYggCY9t7e/BvjaBHM= @@ -672,20 +641,14 @@ github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sigma/bdoor v0.0.0-20160202064022-babf2a4017b0/go.mod h1:WBu7REWbxC/s/J06jsk//d+9DOz9BbsmcIrimuGRFbs= -github.com/sigma/vmw-guestinfo v0.0.0-20160204083807-95dd4126d6e8/go.mod h1:JrRFFC0veyh0cibh0DAhriSY7/gV3kDdNaVUOmfx01U= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= -github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= @@ -724,14 +687,10 @@ github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+ github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/vektah/gqlparser/v2 v2.2.0 h1:bAc3slekAAJW6sZTi07aGq0OrfaCjj4jxARAaC7g2EM= github.com/vektah/gqlparser/v2 v2.2.0/go.mod h1:i3mQIGIrbK2PD1RrCeMTlVbkF2FJ6WkU1KJlJlC+3F4= -github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= -github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/vmware/go-vcloud-director/v2 v2.22.0 h1:i1yFCoQZl/mTKViWLpT8mC9tlOAbupip703K0q1gQT0= github.com/vmware/go-vcloud-director/v2 v2.22.0/go.mod h1:QPxGFgrUcSyzy9IlpwDE4UNT3tsOy2047tJOPEJ4nlw= github.com/vmware/govmomi v0.36.1 h1:+E/nlfteQ8JvC0xhuKAfpnMsuIeGeGj7rJwqENUcWm8= github.com/vmware/govmomi v0.36.1/go.mod h1:mtGWtM+YhTADHlCgJBiskSRPOZRsN9MSjPzaZLte/oQ= -github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= -github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= github.com/vultr/govultr/v3 v3.6.4 h1:unvY9eXlBw667ECQZDbBDOIaWB8wkk6Bx+yB0IMKXJ4= github.com/vultr/govultr/v3 v3.6.4/go.mod h1:rt9v2x114jZmmLAE/h5N5jnxTmsK9ewwS2oQZ0UBQzM= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= @@ -798,10 +757,6 @@ go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go4.org v0.0.0-20160314031811-03efcb870d84/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= -go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= -go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= -go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -875,7 +830,6 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -921,7 +875,6 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= @@ -1075,7 +1028,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -1325,8 +1277,6 @@ k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY= k8s.io/cloud-provider v0.30.0 h1:hz1MXkFjsyO167sRZVchXEi2YYMQ6kolBi79nuICjzw= k8s.io/cloud-provider v0.30.0/go.mod h1:iyVcGvDfmZ7m5cliI9TTHj0VTjYDNpc/K71Gp6hukjU= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/component-base v0.30.0 h1:cj6bp38g0ainlfYtaOQuRELh5KSYjhKxM+io7AUIk4o= -k8s.io/component-base v0.30.0/go.mod h1:V9x/0ePFNaKeKYA3bOvIbrNoluTSG+fSJKjLdjOoeXQ= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= @@ -1344,8 +1294,6 @@ k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lV k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/kubelet v0.30.0 h1:/pqHVR2Rn8ExCpn211wL3pMtqRFpcBcJPl4+1INbIMk= -k8s.io/kubelet v0.30.0/go.mod h1:WukdKqbQxnj+csn3K8XOKeX7Sh60J/da25IILjvvB5s= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= @@ -1362,8 +1310,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.17.1-0.20240418082203-04706074d2f1 h1:W15Y5zHVUsH1YJvstRqy6lG0KquU7kS2ooGC5poLnrU= -sigs.k8s.io/controller-runtime v0.17.1-0.20240418082203-04706074d2f1/go.mod h1:umEFUKWCSYpq2U4tNN7riBXU6iiulk7bdF0XZq9LzvU= +sigs.k8s.io/controller-runtime v0.18.2 h1:RqVW6Kpeaji67CY5nPEfRz6ZfFMk0lWQlNrLqlNpx+Q= +sigs.k8s.io/controller-runtime v0.18.2/go.mod h1:tuAt1+wbVsXIT8lPtk5RURxqAnq7xkpv2Mhttslg7Hw= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index c989dcdc7..bef8ee17b 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -45,10 +45,10 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + cloudproviderutil "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - cloudproviderutil "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "k8s.io/apimachinery/pkg/api/meta" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/cloudprovider/provider/edge/provider.go b/pkg/cloudprovider/provider/edge/provider.go index b47f535af..f4456a7c5 100644 --- a/pkg/cloudprovider/provider/edge/provider.go +++ b/pkg/cloudprovider/provider/edge/provider.go @@ -18,6 +18,7 @@ package edge import ( "context" + "go.uber.org/zap" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" diff --git a/pkg/controller/machine/controller.go b/pkg/controller/machine/controller.go index ad85ace9e..53eb38511 100644 --- a/pkg/controller/machine/controller.go +++ b/pkg/controller/machine/controller.go @@ -64,6 +64,7 @@ import ( "k8s.io/client-go/tools/reference" "k8s.io/client-go/util/retry" ccmapi "k8s.io/cloud-provider/api" + "sigs.k8s.io/controller-runtime/pkg/builder" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/event" @@ -71,7 +72,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" ) const ( @@ -209,50 +209,45 @@ func Add( reconciler.metrics.Errors.Add(1) }) - c, err := controller.New(ControllerName, mgr, controller.Options{ - Reconciler: reconciler, - MaxConcurrentReconciles: numWorkers, - LogConstructor: func(*reconcile.Request) logr.Logger { - // we log ourselves - return zapr.NewLogger(zap.NewNop()) - }, - }) - if err != nil { - return err - } - if err := c.Watch(source.Kind(mgr.GetCache(), &clusterv1alpha1.Machine{}), - &handler.EnqueueRequestForObject{}); err != nil { - return err - } - metrics.Workers.Set(float64(numWorkers)) - return c.Watch( - source.Kind(mgr.GetCache(), &corev1.Node{}), - enqueueRequestsForNodes(ctx, log, mgr), - predicate.Funcs{UpdateFunc: func(e event.UpdateEvent) bool { - oldNode := e.ObjectOld.(*corev1.Node) - newNode := e.ObjectNew.(*corev1.Node) - if newNode.ResourceVersion == oldNode.ResourceVersion { - return false + nodePredicate := predicate.Funcs{UpdateFunc: func(e event.UpdateEvent) bool { + oldNode := e.ObjectOld.(*corev1.Node) + newNode := e.ObjectNew.(*corev1.Node) + if newNode.ResourceVersion == oldNode.ResourceVersion { + return false + } + // Don't do anything if the ready condition hasn't changed + for _, newCondition := range newNode.Status.Conditions { + if newCondition.Type != corev1.NodeReady { + continue } - // Don't do anything if the ready condition hasn't changed - for _, newCondition := range newNode.Status.Conditions { - if newCondition.Type != corev1.NodeReady { + for _, oldCondition := range oldNode.Status.Conditions { + if oldCondition.Type != corev1.NodeReady { continue } - for _, oldCondition := range oldNode.Status.Conditions { - if oldCondition.Type != corev1.NodeReady { - continue - } - if newCondition.Status == oldCondition.Status { - return false - } + if newCondition.Status == oldCondition.Status { + return false } } - return true - }}, - ) + } + return true + }} + + _, err := builder.ControllerManagedBy(mgr). + Named(ControllerName). + WithOptions(controller.Options{ + MaxConcurrentReconciles: numWorkers, + LogConstructor: func(*reconcile.Request) logr.Logger { + // we log ourselves + return zapr.NewLogger(zap.NewNop()) + }, + }). + For(&clusterv1alpha1.Machine{}). + Watches(&corev1.Node{}, enqueueRequestsForNodes(ctx, log, mgr), builder.WithPredicates(nodePredicate)). + Build(reconciler) + + return err } func enqueueRequestsForNodes(ctx context.Context, log *zap.SugaredLogger, mgr manager.Manager) handler.EventHandler { diff --git a/pkg/controller/machinedeployment/controller.go b/pkg/controller/machinedeployment/controller.go index f37fee3fb..73530f52d 100644 --- a/pkg/controller/machinedeployment/controller.go +++ b/pkg/controller/machinedeployment/controller.go @@ -35,12 +35,12 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" ctrlruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" ) // controllerName is the name of this controller. @@ -77,47 +77,25 @@ func Add(mgr manager.Manager, log *zap.SugaredLogger) error { // add adds a new Controller to mgr with r as the reconcile.Reconciler. func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) error { - // Create a new controller. - c, err := controller.New(controllerName, mgr, controller.Options{ - Reconciler: r, - LogConstructor: func(*reconcile.Request) logr.Logger { - // we log ourselves - return zapr.NewLogger(zap.NewNop()) - }, - }) - if err != nil { - return err - } - - // Watch for changes to MachineDeployment. - err = c.Watch(source.Kind(mgr.GetCache(), &v1alpha1.MachineDeployment{}), - &handler.EnqueueRequestForObject{}, - ) - if err != nil { - return err - } - - // Watch for changes to MachineSet and reconcile the owner MachineDeployment. - err = c.Watch( - source.Kind(mgr.GetCache(), &v1alpha1.MachineSet{}), - handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &v1alpha1.MachineDeployment{}, handler.OnlyControllerOwner()), - ) - if err != nil { - return err - } - - // Watch for changes to MachineSets using a mapping function to MachineDeployment. - // This watcher is required for use cases like adoption. In case a MachineSet doesn't have - // a controller reference, it'll look for potential matching MachineDeployments to reconcile. - err = c.Watch( - source.Kind(mgr.GetCache(), &v1alpha1.MachineSet{}), - handler.EnqueueRequestsFromMapFunc(mapFn), - ) - if err != nil { - return err - } - - return nil + _, err := builder.ControllerManagedBy(mgr). + Named(controllerName). + WithOptions(controller.Options{ + LogConstructor: func(*reconcile.Request) logr.Logger { + // we log ourselves + return zapr.NewLogger(zap.NewNop()) + }, + }). + // Watch for changes to MachineDeployment. + For(&v1alpha1.MachineDeployment{}). + // Watch for changes to MachineSet and reconcile the owner MachineDeployment. + Owns(&v1alpha1.MachineSet{}). + // Watch for changes to MachineSets using a mapping function to MachineDeployment. + // This watcher is required for use cases like adoption. In case a MachineSet doesn't have + // a controller reference, it'll look for potential matching MachineDeployments to reconcile. + Watches(&v1alpha1.MachineSet{}, handler.EnqueueRequestsFromMapFunc(mapFn)). + Build(r) + + return err } // Reconcile reads that state of the cluster for a MachineDeployment object and makes changes based on the state read diff --git a/pkg/controller/machineset/controller.go b/pkg/controller/machineset/controller.go index 8d6ca3f10..3290a56bd 100644 --- a/pkg/controller/machineset/controller.go +++ b/pkg/controller/machineset/controller.go @@ -38,12 +38,12 @@ import ( "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/tools/record" ctrlruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" ) // controllerName is the name of this controller. @@ -80,43 +80,25 @@ func newReconciler(mgr manager.Manager, log *zap.SugaredLogger) *ReconcileMachin // add adds a new Controller to mgr with r as the reconcile.Reconciler. func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) error { - // Create a new controller. - c, err := controller.New(controllerName, mgr, controller.Options{ - Reconciler: r, - LogConstructor: func(*reconcile.Request) logr.Logger { - // we log ourselves - return zapr.NewLogger(zap.NewNop()) - }, - }) - if err != nil { - return err - } - - // Watch for changes to MachineSet. - err = c.Watch( - source.Kind(mgr.GetCache(), &clusterv1alpha1.MachineSet{}), - &handler.EnqueueRequestForObject{}, - ) - if err != nil { - return err - } - - // Watch for changes to Machines and reconcile the owner MachineSet. - err = c.Watch( - source.Kind(mgr.GetCache(), &clusterv1alpha1.Machine{}), - handler.EnqueueRequestForOwner(mgr.GetScheme(), mgr.GetRESTMapper(), &clusterv1alpha1.MachineSet{}, handler.OnlyControllerOwner()), - ) - if err != nil { - return err - } - - // Watch for changes to Machines using a mapping function to MachineSets. - // This watcher is required for use cases like adoption. In case a Machine doesn't have - // a controller reference, it'll look for potential matching MachineSet to reconcile. - return c.Watch( - source.Kind(mgr.GetCache(), &clusterv1alpha1.Machine{}), - handler.EnqueueRequestsFromMapFunc(mapFn), - ) + _, err := builder.ControllerManagedBy(mgr). + Named(controllerName). + WithOptions(controller.Options{ + LogConstructor: func(*reconcile.Request) logr.Logger { + // we log ourselves + return zapr.NewLogger(zap.NewNop()) + }, + }). + // Watch for changes to MachineSet. + For(&clusterv1alpha1.MachineSet{}). + // Watch for changes to Machines and reconcile the owner MachineSet. + Owns(&clusterv1alpha1.Machine{}). + // Watch for changes to Machines using a mapping function to MachineSets. + // This watcher is required for use cases like adoption. In case a Machine doesn't have + // a controller reference, it'll look for potential matching MachineSet to reconcile. + Watches(&clusterv1alpha1.Machine{}, handler.EnqueueRequestsFromMapFunc(mapFn)). + Build(r) + + return err } // ReconcileMachineSet reconciles a MachineSet object. diff --git a/pkg/controller/nodecsrapprover/controller.go b/pkg/controller/nodecsrapprover/controller.go index 3f9178063..97ccb1357 100644 --- a/pkg/controller/nodecsrapprover/controller.go +++ b/pkg/controller/nodecsrapprover/controller.go @@ -35,12 +35,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" certificatesv1client "k8s.io/client-go/kubernetes/typed/certificates/v1" + "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" ) const ( @@ -81,20 +80,19 @@ func Add(mgr manager.Manager, log *zap.SugaredLogger) error { log: log.Named(ControllerName), certClient: certClient.CertificateSigningRequests(), } - watchType := &certificatesv1.CertificateSigningRequest{} - - cntrl, err := controller.New(ControllerName, mgr, controller.Options{ - Reconciler: rec, - LogConstructor: func(*reconcile.Request) logr.Logger { - // we log ourselves - return zapr.NewLogger(zap.NewNop()) - }, - }) - if err != nil { - return fmt.Errorf("failed to construct controller: %w", err) - } - return cntrl.Watch(source.Kind(mgr.GetCache(), watchType), &handler.EnqueueRequestForObject{}) + _, err = builder.ControllerManagedBy(mgr). + Named(ControllerName). + WithOptions(controller.Options{ + LogConstructor: func(*reconcile.Request) logr.Logger { + // we log ourselves + return zapr.NewLogger(zap.NewNop()) + }, + }). + For(&certificatesv1.CertificateSigningRequest{}). + Build(rec) + + return err } func (r *reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { From d4db18ed84475701f70057e6db459f4ab3352573 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 3 Jun 2024 13:12:02 +0500 Subject: [PATCH 389/489] Fix AMI filter for rocky linux (#1801) Signed-off-by: Waleed Malik --- pkg/cloudprovider/provider/aws/provider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 1320f6147..86dd70292 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -115,12 +115,12 @@ var ( }, providerconfigtypes.OperatingSystemRockyLinux: { awstypes.CPUArchitectureX86_64: { - description: "*Rocky-8-ec2-8*.x86_64", + description: "*Rocky-8-EC2-8*.x86_64", // The AWS marketplace ID from Rocky Linux Community Platform Engineering (CPE) owner: "792107900819", }, awstypes.CPUArchitectureARM64: { - description: "*Rocky-8-ec2-8*.aarch64", + description: "*Rocky-8-EC2-8*.aarch64", // The AWS marketplace ID from Rocky Linux Community Platform Engineering (CPE) owner: "792107900819", }, From 24477483799684149f0b2633a967f0ef85278a6a Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 3 Jun 2024 15:02:01 +0500 Subject: [PATCH 390/489] Update AMI filter for rocky linux (#1803) Signed-off-by: Waleed Malik --- pkg/cloudprovider/provider/aws/provider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 86dd70292..862ea8676 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -115,12 +115,12 @@ var ( }, providerconfigtypes.OperatingSystemRockyLinux: { awstypes.CPUArchitectureX86_64: { - description: "*Rocky-8-EC2-8*.x86_64", + description: "*Rocky-8-EC2-*.x86_64", // The AWS marketplace ID from Rocky Linux Community Platform Engineering (CPE) owner: "792107900819", }, awstypes.CPUArchitectureARM64: { - description: "*Rocky-8-EC2-8*.aarch64", + description: "*Rocky-8-EC2-*.aarch64", // The AWS marketplace ID from Rocky Linux Community Platform Engineering (CPE) owner: "792107900819", }, From ec6b90321786f4fe1974fed3a41a742e43a9b0a5 Mon Sep 17 00:00:00 2001 From: Mohamed Rafraf <81432497+mohamed-rafraf@users.noreply.github.com> Date: Wed, 5 Jun 2024 23:44:04 +0100 Subject: [PATCH 391/489] Delete provSpec redundant check in MC providers (#1791) * Delete provSpec check in baremetal provider No need for this check in the `getConfig()` function ``` if provSpec.Value == nil { return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } ``` because it already exists in `providerconfigtypes.GetConfig()` function Signed-off-by: Mohamed Rafraf <81432497+mohamed-rafraf@users.noreply.github.com> * delete redundant check --------- Signed-off-by: Mohamed Rafraf <81432497+mohamed-rafraf@users.noreply.github.com> --- pkg/cloudprovider/provider/alibaba/provider.go | 4 ---- pkg/cloudprovider/provider/anexia/provider.go | 3 --- pkg/cloudprovider/provider/aws/provider.go | 4 ---- pkg/cloudprovider/provider/azure/provider.go | 4 ---- pkg/cloudprovider/provider/baremetal/provider.go | 4 ---- pkg/cloudprovider/provider/digitalocean/provider.go | 4 ---- pkg/cloudprovider/provider/equinixmetal/provider.go | 4 ---- pkg/cloudprovider/provider/gce/config.go | 4 ---- pkg/cloudprovider/provider/hetzner/provider.go | 4 ---- pkg/cloudprovider/provider/kubevirt/provider.go | 4 ---- pkg/cloudprovider/provider/linode/provider.go | 4 ---- pkg/cloudprovider/provider/nutanix/provider.go | 4 ---- pkg/cloudprovider/provider/opennebula/provider.go | 4 ---- pkg/cloudprovider/provider/openstack/provider.go | 8 -------- pkg/cloudprovider/provider/scaleway/provider.go | 4 ---- .../provider/vmwareclouddirector/provider.go | 8 -------- pkg/cloudprovider/provider/vsphere/provider.go | 4 ---- pkg/cloudprovider/provider/vultr/provider.go | 4 ---- 18 files changed, 79 deletions(-) diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index f66e7b848..7ed32a008 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -347,10 +347,6 @@ func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { } func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, errors.New("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, fmt.Errorf("failed to decode providers config: %w", err) diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index bef8ee17b..ec8227de0 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -387,9 +387,6 @@ func (p *provider) resolveConfig(ctx context.Context, log *zap.SugaredLogger, co } func (p *provider) getConfig(ctx context.Context, log *zap.SugaredLogger, provSpec clusterv1alpha1.ProviderSpec) (*resolvedConfig, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerSpec.value is nil") - } pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 862ea8676..5f6fc5fc9 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -1099,10 +1099,6 @@ func awsErrorToTerminalError(err error, msg string) error { } func setProviderSpec(rawConfig awstypes.RawConfig, provSpec clusterv1alpha1.ProviderSpec) (*runtime.RawExtension, error) { - if provSpec.Value == nil { - return nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, err diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 9d857c074..581d5d526 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -241,10 +241,6 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes } func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err diff --git a/pkg/cloudprovider/provider/baremetal/provider.go b/pkg/cloudprovider/provider/baremetal/provider.go index fedca37f1..15bdae2bf 100644 --- a/pkg/cloudprovider/provider/baremetal/provider.go +++ b/pkg/cloudprovider/provider/baremetal/provider.go @@ -88,10 +88,6 @@ type Config struct { } func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index 649da5acc..97d9c614a 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -104,10 +104,6 @@ func getClient(ctx context.Context, token string) *godo.Client { } func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index bcaf075f2..03f4c8d3e 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -80,10 +80,6 @@ type provider struct { } func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *equinixmetaltypes.RawConfig, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, nil, err diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index 0ba7252db..e23c78b58 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -72,10 +72,6 @@ const ( // given ProviderSpec. func newCloudProviderSpec(provSpec v1alpha1.ProviderSpec) (*gcetypes.CloudProviderSpec, *providerconfigtypes.Config, error) { // Retrieve provider configuration from machine specification. - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, fmt.Errorf("cannot unmarshal machine.spec.providerconfig.value: %w", err) diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 9e3a7d1aa..6dde36495 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -86,10 +86,6 @@ func getClient(token string) *hcloud.Client { } func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index c4f81e5c6..e7c9b0157 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -190,10 +190,6 @@ func (k *kubeVirtServer) Status() instance.Status { var _ instance.Instance = &kubeVirtServer{} func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index eac81b491..f25086850 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -109,10 +109,6 @@ func getClient(ctx context.Context, token string) linodego.Client { } func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index cdf112739..78b322fca 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -110,10 +110,6 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes } func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *nutanixtypes.RawConfig, error) { - if provSpec.Value == nil { - return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, nil, err diff --git a/pkg/cloudprovider/provider/opennebula/provider.go b/pkg/cloudprovider/provider/opennebula/provider.go index 1a3f442fd..35259d4e0 100644 --- a/pkg/cloudprovider/provider/opennebula/provider.go +++ b/pkg/cloudprovider/provider/opennebula/provider.go @@ -84,10 +84,6 @@ func getClient(config *Config) *goca.Client { } func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 78f381b0d..8cd021920 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -178,10 +178,6 @@ func (p *provider) getConfigAuth(c *Config, rawConfig *openstacktypes.RawConfig) } func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *openstacktypes.RawConfig, error) { - if provSpec.Value == nil { - return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, nil, err @@ -297,10 +293,6 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } func setProviderSpec(rawConfig openstacktypes.RawConfig, provSpec clusterv1alpha1.ProviderSpec) (*runtime.RawExtension, error) { - if provSpec.Value == nil { - return nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, err diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go index f91af1deb..f010ca758 100644 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ b/pkg/cloudprovider/provider/scaleway/provider.go @@ -87,10 +87,6 @@ func getImageNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { } func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go index 325fda982..c40472520 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -314,10 +314,6 @@ func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, } func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *vcdtypes.RawConfig, error) { - if provSpec.Value == nil { - return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, nil, err @@ -557,10 +553,6 @@ func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec cluste } func setProviderSpec(rawConfig vcdtypes.RawConfig, provSpec clusterv1alpha1.ProviderSpec) (*runtime.RawExtension, error) { - if provSpec.Value == nil { - return nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, err diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index c8d828276..5bf072739 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -121,10 +121,6 @@ func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.Machin } func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *vspheretypes.RawConfig, error) { - if provSpec.Value == nil { - return nil, nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, nil, err diff --git a/pkg/cloudprovider/provider/vultr/provider.go b/pkg/cloudprovider/provider/vultr/provider.go index fcad5b23b..de4f05ef1 100644 --- a/pkg/cloudprovider/provider/vultr/provider.go +++ b/pkg/cloudprovider/provider/vultr/provider.go @@ -99,10 +99,6 @@ func getClient(ctx context.Context, apiKey string) *govultr.Client { } func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err From b14dc1ca6eb19ba3bca57b6b55ce1e7c35800502 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 6 Jun 2024 12:10:03 +0500 Subject: [PATCH 392/489] Drop cloud provider specific cloud-configs (#1802) OSM is responsible for generating and mantaining these cloud-configs Signed-off-by: Waleed Malik --- docs/howto-provider.md | 6 - .../provider/alibaba/provider.go | 4 - pkg/cloudprovider/provider/anexia/provider.go | 4 - pkg/cloudprovider/provider/aws/provider.go | 22 -- .../provider/aws/types/cloudconfig.go | 83 ------- .../provider/aws/types/cloudconfig_test.go | 71 ------ .../aws/types/testdata/simple-config.golden | 12 -- pkg/cloudprovider/provider/azure/provider.go | 38 ---- .../provider/azure/types/cloudconfig.go | 50 ----- .../provider/baremetal/provider.go | 4 - .../provider/digitalocean/provider.go | 4 - pkg/cloudprovider/provider/edge/provider.go | 4 - .../provider/equinixmetal/provider.go | 4 - pkg/cloudprovider/provider/fake/provider.go | 4 - pkg/cloudprovider/provider/gce/provider.go | 28 --- .../provider/gce/types/cloudconfig.go | 79 ------- .../provider/gce/types/cloudconfig_test.go | 73 ------- .../provider/hetzner/provider.go | 4 - .../provider/kubevirt/provider.go | 14 -- .../provider/kubevirt/types/cloudconfig.go | 36 ---- pkg/cloudprovider/provider/linode/provider.go | 4 - .../provider/nutanix/provider.go | 5 - .../provider/opennebula/provider.go | 4 - .../provider/openstack/provider.go | 39 ---- .../provider/openstack/types/cloudconfig.go | 144 ------------- .../openstack/types/cloudconfig_test.go | 204 ------------------ .../testdata/bs-defaulting-config.golden | 20 -- .../testdata/config-with-special-chars.golden | 26 --- .../types/testdata/simple-config.golden | 21 -- ...ication-credentials-ignore-userpass.golden | 17 -- .../use-application-credentials.golden | 17 -- .../use-octavia-explicitly-disabled.golden | 22 -- .../use-octavia-explicitly-enabled.golden | 22 -- .../provider/scaleway/provider.go | 4 - .../provider/vmwareclouddirector/provider.go | 4 - .../provider/vsphere/provider.go | 63 ------ .../provider/vsphere/types/cloudconfig.go | 143 ------------ .../vsphere/types/cloudconfig_test.go | 138 ------------ .../types/testdata/2-virtual-centers.golden | 33 --- .../types/testdata/3-dual-stack.golden | 29 --- .../types/testdata/simple-config.golden | 21 -- pkg/cloudprovider/provider/vultr/provider.go | 4 - pkg/cloudprovider/types/types.go | 3 - pkg/cloudprovider/validationwrapper.go | 5 - pkg/userdata/cloud/provider.go | 25 --- 45 files changed, 1561 deletions(-) delete mode 100644 pkg/cloudprovider/provider/aws/types/cloudconfig.go delete mode 100644 pkg/cloudprovider/provider/aws/types/cloudconfig_test.go delete mode 100644 pkg/cloudprovider/provider/aws/types/testdata/simple-config.golden delete mode 100644 pkg/cloudprovider/provider/azure/types/cloudconfig.go delete mode 100644 pkg/cloudprovider/provider/gce/types/cloudconfig.go delete mode 100644 pkg/cloudprovider/provider/gce/types/cloudconfig_test.go delete mode 100644 pkg/cloudprovider/provider/kubevirt/types/cloudconfig.go delete mode 100644 pkg/cloudprovider/provider/openstack/types/cloudconfig.go delete mode 100644 pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go delete mode 100644 pkg/cloudprovider/provider/openstack/types/testdata/bs-defaulting-config.golden delete mode 100644 pkg/cloudprovider/provider/openstack/types/testdata/config-with-special-chars.golden delete mode 100644 pkg/cloudprovider/provider/openstack/types/testdata/simple-config.golden delete mode 100644 pkg/cloudprovider/provider/openstack/types/testdata/use-application-credentials-ignore-userpass.golden delete mode 100644 pkg/cloudprovider/provider/openstack/types/testdata/use-application-credentials.golden delete mode 100644 pkg/cloudprovider/provider/openstack/types/testdata/use-octavia-explicitly-disabled.golden delete mode 100644 pkg/cloudprovider/provider/openstack/types/testdata/use-octavia-explicitly-enabled.golden delete mode 100644 pkg/cloudprovider/provider/vsphere/types/cloudconfig.go delete mode 100644 pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go delete mode 100644 pkg/cloudprovider/provider/vsphere/types/testdata/2-virtual-centers.golden delete mode 100644 pkg/cloudprovider/provider/vsphere/types/testdata/3-dual-stack.golden delete mode 100644 pkg/cloudprovider/provider/vsphere/types/testdata/simple-config.golden delete mode 100644 pkg/userdata/cloud/provider.go diff --git a/docs/howto-provider.md b/docs/howto-provider.md index f797c4260..886620ae3 100644 --- a/docs/howto-provider.md +++ b/docs/howto-provider.md @@ -28,12 +28,6 @@ Get(machine *v1alpha1.Machine) (instance.Instance, error) In case the instance cannot be found, the returned error has to be `github.com/kubermatic/machine-controller/pkg/cloudprovider/errors.ErrInstanceNotFound` for proper evaluation by the machine controller. -```go -GetCloudConfig(spec v1alpha1.MachineSpec) (config string, name string, err error) -``` - -`GetCloudConfig` will return the cloud provider specific cloud-config, which gets consumed by the kubelet. - ```go Create(machine *v1alpha1.Machine, data *cloud.MachineCreateDeleteData, userdata string) (instance.Instance, error) ``` diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index 7ed32a008..472eac148 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -200,10 +200,6 @@ func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *cluster return nil, fmt.Errorf("instance %v is not ready", foundInstance.InstanceId) } -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return "", "", nil -} - func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index ec8227de0..a5c283603 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -523,10 +523,6 @@ func (p *provider) Get(ctx context.Context, log *zap.SugaredLogger, machine *clu return &instance, nil } -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string, error) { - return "", "", nil -} - func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (isDeleted bool, retErr error) { if inst, err := p.Get(ctx, log, machine, data); err != nil { if cloudprovidererrors.IsNotFound(err) { diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 5f6fc5fc9..6f9ad9a4b 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -917,28 +917,6 @@ func (p *provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (* return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { - c, _, _, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return "", "", fmt.Errorf("failed to parse config: %w", err) - } - - cc := &awstypes.CloudConfig{ - Global: awstypes.GlobalOpts{ - VPC: c.VpcID, - SubnetID: c.SubnetID, - Zone: c.AvailabilityZone, - }, - } - - s, err := awstypes.CloudConfigToString(cc) - if err != nil { - return "", "", fmt.Errorf("failed to convert cloud-config to string: %w", err) - } - - return s, "aws", nil -} - func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) diff --git a/pkg/cloudprovider/provider/aws/types/cloudconfig.go b/pkg/cloudprovider/provider/aws/types/cloudconfig.go deleted file mode 100644 index 40de37151..000000000 --- a/pkg/cloudprovider/provider/aws/types/cloudconfig.go +++ /dev/null @@ -1,83 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "bytes" - "fmt" - "text/template" - - "github.com/Masterminds/sprig/v3" - - "github.com/kubermatic/machine-controller/pkg/ini" -) - -const ( - cloudConfigTpl = `[global] -Zone={{ .Global.Zone | iniEscape }} -VPC={{ .Global.VPC | iniEscape }} -SubnetID={{ .Global.SubnetID | iniEscape }} -RouteTableID={{ .Global.RouteTableID | iniEscape }} -RoleARN={{ .Global.RoleARN | iniEscape }} -KubernetesClusterID={{ .Global.KubernetesClusterID | iniEscape }} -DisableSecurityGroupIngress={{ .Global.DisableSecurityGroupIngress }} -ElbSecurityGroup={{ .Global.ElbSecurityGroup | iniEscape }} -{{- if .Global.DisableStrictZoneCheck }} -DisableStrictZoneCheck=true -{{- end }} -{{- range .Global.NodeIPFamilies }} -NodeIPFamilies={{ . | iniEscape}} -{{- end }} -` -) - -type CloudConfig struct { - Global GlobalOpts -} - -type GlobalOpts struct { - Zone string - VPC string - SubnetID string - RouteTableID string - RoleARN string - KubernetesClusterTag string - KubernetesClusterID string - ElbSecurityGroup string - DisableSecurityGroupIngress bool - // DisableStrictZoneCheck has been removed in Kubernetes 1.27+. - // See https://github.com/kubernetes/cloud-provider-aws/pull/573 for more information. - DisableStrictZoneCheck bool - NodeIPFamilies []string -} - -func CloudConfigToString(c *CloudConfig) (string, error) { - funcMap := sprig.TxtFuncMap() - funcMap["iniEscape"] = ini.Escape - - tpl, err := template.New("cloud-config").Funcs(funcMap).Parse(cloudConfigTpl) - if err != nil { - return "", fmt.Errorf("failed to parse the cloud config template: %w", err) - } - - buf := &bytes.Buffer{} - if err := tpl.Execute(buf, c); err != nil { - return "", fmt.Errorf("failed to execute cloud config template: %w", err) - } - - return buf.String(), nil -} diff --git a/pkg/cloudprovider/provider/aws/types/cloudconfig_test.go b/pkg/cloudprovider/provider/aws/types/cloudconfig_test.go deleted file mode 100644 index f9eaa1cfb..000000000 --- a/pkg/cloudprovider/provider/aws/types/cloudconfig_test.go +++ /dev/null @@ -1,71 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "flag" - "testing" - - "gopkg.in/gcfg.v1" - - testhelper "github.com/kubermatic/machine-controller/pkg/test" -) - -var update = flag.Bool("update", false, "update testdata files") - -func TestCloudConfigToString(t *testing.T) { - tests := []struct { - name string - config *CloudConfig - }{ - { - name: "simple-config", - config: &CloudConfig{ - Global: GlobalOpts{ - Zone: "some-zone", - VPC: "some-vpc", - SubnetID: "some-subnet", - KubernetesClusterID: "some-tag", - DisableSecurityGroupIngress: true, - DisableStrictZoneCheck: true, - ElbSecurityGroup: "some-sg", - KubernetesClusterTag: "some-tag", - RoleARN: "some-arn", - RouteTableID: "some-rt", - NodeIPFamilies: []string{"ipv4", "ipv6"}, - }, - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - s, err := CloudConfigToString(test.config) - if err != nil { - t.Fatal(err) - } - t.Logf("\n%s", s) - - nc := &CloudConfig{} - if err := gcfg.ReadStringInto(nc, s); err != nil { - t.Fatalf("failed to load string into config object: %v", err) - } - goldenName := test.name + ".golden" - testhelper.CompareOutput(t, goldenName, s, *update) - }) - } -} diff --git a/pkg/cloudprovider/provider/aws/types/testdata/simple-config.golden b/pkg/cloudprovider/provider/aws/types/testdata/simple-config.golden deleted file mode 100644 index 57bffe19e..000000000 --- a/pkg/cloudprovider/provider/aws/types/testdata/simple-config.golden +++ /dev/null @@ -1,12 +0,0 @@ -[global] -Zone="some-zone" -VPC="some-vpc" -SubnetID="some-subnet" -RouteTableID="some-rt" -RoleARN="some-arn" -KubernetesClusterID="some-tag" -DisableSecurityGroupIngress=true -ElbSecurityGroup="some-sg" -DisableStrictZoneCheck=true -NodeIPFamilies="ipv4" -NodeIPFamilies="ipv6" diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 581d5d526..f5ec24fa7 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -928,44 +928,6 @@ func (p *provider) get(ctx context.Context, log *zap.SugaredLogger, machine *clu return &azureVM{vm: vm, ipAddresses: ipAddresses, status: status}, nil } -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { - c, _, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return "", "", fmt.Errorf("failed to parse config: %w", err) - } - - var avSet string - if c.AssignAvailabilitySet == nil && c.AvailabilitySet != "" || - c.AssignAvailabilitySet != nil && *c.AssignAvailabilitySet && c.AvailabilitySet != "" { - avSet = c.AvailabilitySet - } - - cc := &azuretypes.CloudConfig{ - Cloud: "AZUREPUBLICCLOUD", - TenantID: c.TenantID, - SubscriptionID: c.SubscriptionID, - AADClientID: c.ClientID, - AADClientSecret: c.ClientSecret, - ResourceGroup: c.ResourceGroup, - VnetResourceGroup: c.VNetResourceGroup, - Location: c.Location, - VNetName: c.VNetName, - SubnetName: c.SubnetName, - LoadBalancerSku: c.LoadBalancerSku, - RouteTableName: c.RouteTableName, - PrimaryAvailabilitySetName: avSet, - SecurityGroupName: c.SecurityGroupName, - UseInstanceMetadata: true, - } - - s, err := azuretypes.CloudConfigToString(cc) - if err != nil { - return "", "", fmt.Errorf("failed to convert cloud-config to string: %w", err) - } - - return s, "azure", nil -} - func validateDiskSKUs(_ context.Context, c *config, sku compute.ResourceSku) error { if c.OSDiskSKU != nil || c.DataDiskSKU != nil { if c.OSDiskSKU != nil { diff --git a/pkg/cloudprovider/provider/azure/types/cloudconfig.go b/pkg/cloudprovider/provider/azure/types/cloudconfig.go deleted file mode 100644 index 6ddb8b5ca..000000000 --- a/pkg/cloudprovider/provider/azure/types/cloudconfig.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "encoding/json" - "fmt" -) - -type CloudConfig struct { - Cloud string `json:"cloud"` - TenantID string `json:"tenantId"` - SubscriptionID string `json:"subscriptionId"` - AADClientID string `json:"aadClientId"` - AADClientSecret string `json:"aadClientSecret"` - - ResourceGroup string `json:"resourceGroup"` - Location string `json:"location"` - VNetName string `json:"vnetName"` - SubnetName string `json:"subnetName"` - RouteTableName string `json:"routeTableName"` - SecurityGroupName string `json:"securityGroupName" yaml:"securityGroupName"` - PrimaryAvailabilitySetName string `json:"primaryAvailabilitySetName"` - VnetResourceGroup string `json:"vnetResourceGroup"` - UseInstanceMetadata bool `json:"useInstanceMetadata"` - LoadBalancerSku string `json:"loadBalancerSku"` -} - -func CloudConfigToString(c *CloudConfig) (string, error) { - b, err := json.Marshal(c) - if err != nil { - return "", fmt.Errorf("failed to unmarshal config: %w", err) - } - - return string(b), nil -} diff --git a/pkg/cloudprovider/provider/baremetal/provider.go b/pkg/cloudprovider/provider/baremetal/provider.go index 15bdae2bf..55afc2c15 100644 --- a/pkg/cloudprovider/provider/baremetal/provider.go +++ b/pkg/cloudprovider/provider/baremetal/provider.go @@ -209,10 +209,6 @@ func (p provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *cluste }, nil } -func (p provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return "", "", nil -} - func (p provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index 97d9c614a..d7e9c9cfd 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -473,10 +473,6 @@ func (p *provider) MigrateUID(ctx context.Context, _ *zap.SugaredLogger, machine return nil } -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return "", "", nil -} - func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) diff --git a/pkg/cloudprovider/provider/edge/provider.go b/pkg/cloudprovider/provider/edge/provider.go index f4456a7c5..9c4bd2ff9 100644 --- a/pkg/cloudprovider/provider/edge/provider.go +++ b/pkg/cloudprovider/provider/edge/provider.go @@ -74,10 +74,6 @@ func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alph return CloudProviderInstance{}, nil } -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string, error) { - return "", "", nil -} - // Create creates a cloud instance according to the given machine. func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, _ string) (instance.Instance, error) { return CloudProviderInstance{}, nil diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index 03f4c8d3e..1d197a76e 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -363,10 +363,6 @@ func (p *provider) MigrateUID(_ context.Context, log *zap.SugaredLogger, machine return nil } -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return "", "", nil -} - func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) diff --git a/pkg/cloudprovider/provider/fake/provider.go b/pkg/cloudprovider/provider/fake/provider.go index 2a01f5f92..67afd2993 100644 --- a/pkg/cloudprovider/provider/fake/provider.go +++ b/pkg/cloudprovider/provider/fake/provider.go @@ -95,10 +95,6 @@ func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alph return CloudProviderInstance{}, nil } -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string, error) { - return "", "", nil -} - // Create creates a cloud instance according to the given machine. func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, _ string) (instance.Instance, error) { return CloudProviderInstance{}, nil diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 4c81cd274..3e58ac482 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -37,7 +37,6 @@ import ( clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - gcetypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/gce/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/providerconfig" @@ -57,7 +56,6 @@ const ( errInvalidDiskType = "Disk type is missing or has wrong type, allowed are 'pd-standard' and 'pd-ssd'" errRetrieveInstance = "Failed to retrieve instance: %v" errGotTooManyInstances = "Got more than 1 instance matching the machine UID label" - errCloudConfig = "Failed to convert cloud-config to string: %v" errInsertInstance = "Failed to insert instance: %v" errDeleteInstance = "Failed to delete instance: %v" errSetLabels = "Failed to set the labels for the new machine UID: %v" @@ -184,32 +182,6 @@ func (p *Provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (* }, nil } -// GetCloudConfig returns the cloud provider specific cloud-config for the kubelet. -func (p *Provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { - // Read configuration. - cfg, err := newConfig(p.resolver, spec.ProviderSpec) - if err != nil { - return "", "", newError(common.InvalidConfigurationMachineError, errMachineSpec, err) - } - // Init cloud configuration. - cc := &gcetypes.CloudConfig{ - Global: gcetypes.GlobalOpts{ - ProjectID: cfg.projectID, - LocalZone: cfg.zone, - MultiZone: cfg.multizone, - Regional: cfg.regional, - NetworkName: cfg.network, - SubnetworkName: cfg.subnetwork, - NodeTags: cfg.tags, - }, - } - config, err = cc.AsString() - if err != nil { - return "", "", newError(common.InvalidConfigurationMachineError, errCloudConfig, err) - } - return config, "gce", nil -} - // Create inserts a cloud instance according to the given machine. func (p *Provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { // Read configuration. diff --git a/pkg/cloudprovider/provider/gce/types/cloudconfig.go b/pkg/cloudprovider/provider/gce/types/cloudconfig.go deleted file mode 100644 index 9c4201d4b..000000000 --- a/pkg/cloudprovider/provider/gce/types/cloudconfig.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// Google Cloud Provider for the Machine Controller -// - -package types - -import ( - "bytes" - "fmt" - "text/template" - - "github.com/Masterminds/sprig/v3" - - "github.com/kubermatic/machine-controller/pkg/ini" -) - -// cloudConfigTemplate renders the cloud-config in gcfg format. All -// fields are optional, that's why containing the ifs and the explicit newlines. -const cloudConfigTemplate = "[global]\n" + - "project-id = {{ .Global.ProjectID | iniEscape }}\n" + - "local-zone = {{ .Global.LocalZone | iniEscape }}\n" + - "network-name = {{ .Global.NetworkName | iniEscape }}\n" + - "subnetwork-name = {{ .Global.SubnetworkName | iniEscape }}\n" + - "token-url = {{ .Global.TokenURL | iniEscape }}\n" + - "multizone = {{ .Global.MultiZone }}\n" + - "regional = {{ .Global.Regional }}\n" + - "{{ range .Global.NodeTags }}node-tags = {{ . | iniEscape }}\n{{end}}" - -// GlobalOpts contains the values of the global section of the cloud configuration. -type GlobalOpts struct { - ProjectID string - LocalZone string - NetworkName string - SubnetworkName string - TokenURL string - MultiZone bool - Regional bool - NodeTags []string - RHSMOfflineToken string -} - -// CloudConfig contains only the section global. -type CloudConfig struct { - Global GlobalOpts -} - -// AsString renders the cloud configuration as string. -func (cc *CloudConfig) AsString() (string, error) { - funcMap := sprig.TxtFuncMap() - funcMap["iniEscape"] = ini.Escape - - tmpl, err := template.New("cloud-config").Funcs(funcMap).Parse(cloudConfigTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse the cloud config template: %w", err) - } - - buf := &bytes.Buffer{} - if err := tmpl.Execute(buf, cc); err != nil { - return "", fmt.Errorf("failed to execute cloud config template: %w", err) - } - - return buf.String(), nil -} diff --git a/pkg/cloudprovider/provider/gce/types/cloudconfig_test.go b/pkg/cloudprovider/provider/gce/types/cloudconfig_test.go deleted file mode 100644 index 6b91cefd5..000000000 --- a/pkg/cloudprovider/provider/gce/types/cloudconfig_test.go +++ /dev/null @@ -1,73 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// Google Cloud Provider for the Machine Controller -// -// Unit Tests -// - -package types - -import ( - "testing" -) - -func TestCloudConfigAsString(t *testing.T) { - tests := []struct { - name string - config *CloudConfig - contents string - }{ - { - name: "minimum test", - config: &CloudConfig{ - Global: GlobalOpts{ - ProjectID: "my-project-id", - LocalZone: "my-zone", - NetworkName: "my-cool-network", - SubnetworkName: "my-cool-subnetwork", - TokenURL: "nil", - MultiZone: true, - Regional: true, - NodeTags: []string{"tag1", "tag2"}, - }, - }, - contents: "[global]\n" + - "project-id = \"my-project-id\"\n" + - "local-zone = \"my-zone\"\n" + - "network-name = \"my-cool-network\"\n" + - "subnetwork-name = \"my-cool-subnetwork\"\n" + - "token-url = \"nil\"\n" + - "multizone = true\n" + - "regional = true\n" + - "node-tags = \"tag1\"\n" + - "node-tags = \"tag2\"\n", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - s, err := test.config.AsString() - if err != nil { - t.Fatalf("failed to convert to string: %v", err) - } - if s != test.contents { - t.Fatalf("output is not as expected: %s", s) - } - }) - } -} diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 6dde36495..e779b2b08 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -521,10 +521,6 @@ func (p *provider) MigrateUID(ctx context.Context, log *zap.SugaredLogger, machi return nil } -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return "", "", nil -} - func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index e7c9b0157..b3cd9e109 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -539,20 +539,6 @@ func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.Machin return spec, nil } -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { - c, _, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return "", "", fmt.Errorf("failed to parse config: %w", err) - } - - cc := kubevirttypes.CloudConfig{ - Namespace: c.Namespace, - } - ccs, err := cc.String() - - return ccs, string(providerconfigtypes.CloudProviderExternal), err -} - func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) diff --git a/pkg/cloudprovider/provider/kubevirt/types/cloudconfig.go b/pkg/cloudprovider/provider/kubevirt/types/cloudconfig.go deleted file mode 100644 index 8d41053e3..000000000 --- a/pkg/cloudprovider/provider/kubevirt/types/cloudconfig.go +++ /dev/null @@ -1,36 +0,0 @@ -/* -Copyright 2021 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "gopkg.in/yaml.v3" -) - -type CloudConfig struct { - // Kubeconfig used to connect to the cluster that runs KubeVirt - Kubeconfig string `yaml:"kubeconfig"` - // Namespace used in KubeVirt cloud-controller-manager as infra cluster namespace. - Namespace string `yaml:"namespace"` -} - -func (c *CloudConfig) String() (string, error) { - out, err := yaml.Marshal(c) - if err != nil { - return "", err - } - return string(out), nil -} diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index f25086850..628841414 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -372,10 +372,6 @@ func (p *provider) MigrateUID(ctx context.Context, _ *zap.SugaredLogger, machine return nil } -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return "", "", nil -} - func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 78b322fca..87f911220 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -422,11 +422,6 @@ func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *cluste return nil } -// GetCloudConfig returns an empty cloud configuration for Nutanix as no CCM exists. -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return "", "", nil -} - func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) diff --git a/pkg/cloudprovider/provider/opennebula/provider.go b/pkg/cloudprovider/provider/opennebula/provider.go index 35259d4e0..3a2f65da3 100644 --- a/pkg/cloudprovider/provider/opennebula/provider.go +++ b/pkg/cloudprovider/provider/opennebula/provider.go @@ -157,10 +157,6 @@ func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec cluste return nil } -func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string, error) { - return "", "", nil -} - func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 8cd021920..542bf1893 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -858,45 +858,6 @@ func (p *provider) MigrateUID(_ context.Context, log *zap.SugaredLogger, machine return nil } -func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { - c, _, _, err := p.getConfig(spec.ProviderSpec) - if err != nil { - return "", "", fmt.Errorf("failed to parse config: %w", err) - } - - cc := &openstacktypes.CloudConfig{ - Global: openstacktypes.GlobalOpts{ - AuthURL: c.IdentityEndpoint, - Username: c.Username, - Password: c.Password, - DomainName: c.DomainName, - ProjectName: c.ProjectName, - ProjectID: c.ProjectID, - Region: c.Region, - ApplicationCredentialSecret: c.ApplicationCredentialSecret, - ApplicationCredentialID: c.ApplicationCredentialID, - }, - LoadBalancer: openstacktypes.LoadBalancerOpts{ - ManageSecurityGroups: true, - }, - BlockStorage: openstacktypes.BlockStorageOpts{ - BSVersion: "auto", - TrustDevicePath: c.TrustDevicePath, - IgnoreVolumeAZ: true, - }, - Version: spec.Versions.Kubelet, - } - if c.NodeVolumeAttachLimit != nil { - cc.BlockStorage.NodeVolumeAttachLimit = *c.NodeVolumeAttachLimit - } - - s, err := openstacktypes.CloudConfigToString(cc) - if err != nil { - return "", "", fmt.Errorf("failed to convert the cloud-config to string: %w", err) - } - return s, "openstack", nil -} - func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { labels := make(map[string]string) diff --git a/pkg/cloudprovider/provider/openstack/types/cloudconfig.go b/pkg/cloudprovider/provider/openstack/types/cloudconfig.go deleted file mode 100644 index 8015fb2fa..000000000 --- a/pkg/cloudprovider/provider/openstack/types/cloudconfig.go +++ /dev/null @@ -1,144 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "bytes" - "fmt" - "strconv" - "text/template" - - "github.com/Masterminds/sprig/v3" - - "github.com/kubermatic/machine-controller/pkg/ini" -) - -// use-octavia is enabled by default in CCM since v1.17.0, and disabled by -// default with the in-tree cloud provider. -// https://v1-18.docs.kubernetes.io/docs/concepts/cluster-administration/cloud-providers/#load-balancer -const ( - cloudConfigTpl = `[Global] -auth-url = {{ .Global.AuthURL | iniEscape }} -{{- if .Global.ApplicationCredentialID }} -application-credential-id = {{ .Global.ApplicationCredentialID | iniEscape }} -application-credential-secret = {{ .Global.ApplicationCredentialSecret | iniEscape }} -{{- else }} -username = {{ .Global.Username | iniEscape }} -password = {{ .Global.Password | iniEscape }} -tenant-name = {{ .Global.ProjectName | iniEscape }} -tenant-id = {{ .Global.ProjectID | iniEscape }} -domain-name = {{ .Global.DomainName | iniEscape }} -{{- end }} -region = {{ .Global.Region | iniEscape }} - -[LoadBalancer] -lb-version = {{ default "v2" .LoadBalancer.LBVersion | iniEscape }} -subnet-id = {{ .LoadBalancer.SubnetID | iniEscape }} -floating-network-id = {{ .LoadBalancer.FloatingNetworkID | iniEscape }} -lb-method = {{ default "ROUND_ROBIN" .LoadBalancer.LBMethod | iniEscape }} -lb-provider = {{ .LoadBalancer.LBProvider | iniEscape }} -{{- if .LoadBalancer.UseOctavia }} -use-octavia = {{ .LoadBalancer.UseOctavia | boolPtr }} -{{- end }} - -{{- if .LoadBalancer.CreateMonitor }} -create-monitor = {{ .LoadBalancer.CreateMonitor }} -monitor-delay = {{ .LoadBalancer.MonitorDelay }} -monitor-timeout = {{ .LoadBalancer.MonitorTimeout }} -monitor-max-retries = {{ .LoadBalancer.MonitorMaxRetries }} -{{- end}} -{{- if semverCompare "~1.9.10 || ~1.10.6 || ~1.11.1 || >=1.12.*" .Version }} -manage-security-groups = {{ .LoadBalancer.ManageSecurityGroups }} -{{- end }} - -[BlockStorage] -{{- if semverCompare ">=1.9" .Version }} -ignore-volume-az = {{ .BlockStorage.IgnoreVolumeAZ }} -{{- end }} -trust-device-path = {{ .BlockStorage.TrustDevicePath }} -bs-version = {{ default "auto" .BlockStorage.BSVersion | iniEscape }} -{{- if .BlockStorage.NodeVolumeAttachLimit }} -node-volume-attach-limit = {{ .BlockStorage.NodeVolumeAttachLimit }} -{{- end }} -` -) - -type LoadBalancerOpts struct { - LBVersion string `gcfg:"lb-version"` - SubnetID string `gcfg:"subnet-id"` - FloatingNetworkID string `gcfg:"floating-network-id"` - LBMethod string `gcfg:"lb-method"` - LBProvider string `gcfg:"lb-provider"` - CreateMonitor bool `gcfg:"create-monitor"` - MonitorDelay ini.Duration `gcfg:"monitor-delay"` - MonitorTimeout ini.Duration `gcfg:"monitor-timeout"` - MonitorMaxRetries uint `gcfg:"monitor-max-retries"` - ManageSecurityGroups bool `gcfg:"manage-security-groups"` - UseOctavia *bool `gcfg:"use-octavia"` -} - -type BlockStorageOpts struct { - BSVersion string `gcfg:"bs-version"` - TrustDevicePath bool `gcfg:"trust-device-path"` - IgnoreVolumeAZ bool `gcfg:"ignore-volume-az"` - NodeVolumeAttachLimit uint `gcfg:"node-volume-attach-limit"` -} - -type GlobalOpts struct { - AuthURL string `gcfg:"auth-url"` - Username string - Password string - ApplicationCredentialID string `gcfg:"application-credential-id"` - ApplicationCredentialSecret string `gcfg:"application-credential-secret"` - - // project name formerly known as tenant name. - // it serialized as tenant-name because openstack CCM reads only tenant-name. In CCM, internally project and tenant - // are stored into tenant-name. - ProjectName string `gcfg:"tenant-name"` - - // project id formerly known as tenant id. - // serialized as tenant-id for same reason as ProjectName - ProjectID string `gcfg:"tenant-id"` - DomainName string `gcfg:"domain-name"` - Region string -} - -// CloudConfig is used to read and store information from the cloud configuration file. -type CloudConfig struct { - Global GlobalOpts - LoadBalancer LoadBalancerOpts - BlockStorage BlockStorageOpts - Version string -} - -func CloudConfigToString(c *CloudConfig) (string, error) { - funcMap := sprig.TxtFuncMap() - funcMap["iniEscape"] = ini.Escape - funcMap["boolPtr"] = func(b *bool) string { return strconv.FormatBool(*b) } - - tpl, err := template.New("cloud-config").Funcs(funcMap).Parse(cloudConfigTpl) - if err != nil { - return "", fmt.Errorf("failed to parse the cloud config template: %w", err) - } - - buf := &bytes.Buffer{} - if err := tpl.Execute(buf, c); err != nil { - return "", fmt.Errorf("failed to execute cloud config template: %w", err) - } - - return buf.String(), nil -} diff --git a/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go b/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go deleted file mode 100644 index 0fa109116..000000000 --- a/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go +++ /dev/null @@ -1,204 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "flag" - "testing" - "time" - - "gopkg.in/gcfg.v1" - - "github.com/kubermatic/machine-controller/pkg/ini" - testhelper "github.com/kubermatic/machine-controller/pkg/test" - - "k8s.io/utils/ptr" -) - -var update = flag.Bool("update", false, "update testdata files") - -func TestCloudConfigToString(t *testing.T) { - tests := []struct { - name string - config *CloudConfig - }{ - { - name: "simple-config", - config: &CloudConfig{ - Global: GlobalOpts{ - AuthURL: "https://127.0.0.1:8443/", - Username: "admin", - Password: "password", - DomainName: "Default", - ProjectName: "Test", - Region: "eu-central1", - }, - BlockStorage: BlockStorageOpts{ - BSVersion: "v2", - IgnoreVolumeAZ: true, - TrustDevicePath: true, - NodeVolumeAttachLimit: 25, - }, - LoadBalancer: LoadBalancerOpts{ - ManageSecurityGroups: true, - }, - Version: "1.10.0", - }, - }, - { - name: "use-octavia-explicitly-enabled", - config: &CloudConfig{ - Global: GlobalOpts{ - AuthURL: "https://127.0.0.1:8443/", - Username: "admin", - Password: "password", - DomainName: "Default", - ProjectName: "Test", - Region: "eu-central1", - }, - BlockStorage: BlockStorageOpts{ - BSVersion: "v2", - IgnoreVolumeAZ: true, - TrustDevicePath: true, - NodeVolumeAttachLimit: 25, - }, - LoadBalancer: LoadBalancerOpts{ - ManageSecurityGroups: true, - UseOctavia: ptr.To(true), - }, - Version: "1.10.0", - }, - }, - { - name: "use-octavia-explicitly-disabled", - config: &CloudConfig{ - Global: GlobalOpts{ - AuthURL: "https://127.0.0.1:8443/", - Username: "admin", - Password: "password", - DomainName: "Default", - ProjectName: "Test", - Region: "eu-central1", - }, - BlockStorage: BlockStorageOpts{ - BSVersion: "v2", - IgnoreVolumeAZ: true, - TrustDevicePath: true, - NodeVolumeAttachLimit: 25, - }, - LoadBalancer: LoadBalancerOpts{ - ManageSecurityGroups: true, - UseOctavia: ptr.To(false), - }, - Version: "1.10.0", - }, - }, - { - name: "config-with-special-chars", - config: &CloudConfig{ - Global: GlobalOpts{ - AuthURL: "https://127.0.0.1:8443/", - Username: "admin", - Password: `.)\^x[tt0L@};p Date: Thu, 6 Jun 2024 13:11:04 +0530 Subject: [PATCH 393/489] Bump GO version to 1.22.4 (#1806) Signed-off-by: archups --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 14 +++++++------- .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 10 +++++----- .prow/verify.yaml | 16 ++++++++-------- Dockerfile | 2 +- Makefile | 2 +- hack/ci/calico.yaml | 2 +- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- 23 files changed, 47 insertions(+), 47 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 68189470c..53350366a 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -119,7 +119,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index f90e89d3f..f368c9344 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index 05dfe490c..ccd97f5ae 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 58094e08d..fff0e0b0c 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 1811fc824..e96d3fe5d 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -61,7 +61,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -126,7 +126,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -157,7 +157,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -188,7 +188,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -219,7 +219,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index c70fd17f4..e7bfd42c3 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index d73e043f5..8defef915 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 75bbbc3ee..7ed2dd2b9 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 3b34bb8a1..197bbcd86 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index a16858989..43282a3f1 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 26e1986ab..94f9693a0 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index c82c40a3b..71ebc222e 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 0c2398c08..e6a21ea30 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 4402359f5..4de152975 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index c7d567d0e..3507c853c 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index e8cf27955..59e380f19 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index c5866d1cf..fb386b0a9 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -156,7 +156,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index e406805c5..8efda6f92 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-9 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-9 command: - make args: @@ -63,7 +63,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-9 command: - make args: @@ -83,7 +83,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-9 command: - make args: @@ -102,7 +102,7 @@ presubmits: clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-9 command: - "/usr/local/bin/shfmt" args: @@ -130,7 +130,7 @@ presubmits: clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-9 command: - "./hack/verify-boilerplate.sh" resources: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-9 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-3 + - image: quay.io/kubermatic/build:go-1.22-node-20-9 command: - make args: diff --git a/Dockerfile b/Dockerfile index 6c87ca442..a32f1ec50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.22.2 +ARG GO_VERSION=1.22.4 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index f5c592bea..dadb53b27 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.22.2 +GO_VERSION ?= 1.22.4 GOOS ?= $(shell go env GOOS) diff --git a/hack/ci/calico.yaml b/hack/ci/calico.yaml index f64820f0f..666a3a9f1 100644 --- a/hack/ci/calico.yaml +++ b/hack/ci/calico.yaml @@ -325,7 +325,7 @@ spec: numAllowedLocalASNumbers: description: Maximum number of local AS numbers that are allowed in the AS path for received routes. This removes BGP loop prevention - and should only be used if absolutely necesssary. + and should only be used if absolutely necessary. format: int32 type: integer password: diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index c90dbad7e..807d089b0 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.22.2 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.22.4 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index c90bd7108..b49f9f9c0 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-20-kind-0.22-3 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 containerize ./hack/verify-licenses.sh go mod vendor From d62ab03c71afd83c2d13549b06f85bd4621ef186 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 6 Jun 2024 13:52:04 +0500 Subject: [PATCH 394/489] Remove container runtime related code (#1812) * Remove provision readme Signed-off-by: Waleed Malik * Container runtime cleanup Signed-off-by: Waleed Malik * Drop unused ini pkg Signed-off-by: Waleed Malik * Refactored code Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- cmd/machine-controller/main.go | 127 ++++------ cmd/provision/README.md | 84 ------- cmd/webhook/main.go | 3 - go.mod | 10 +- go.sum | 31 --- pkg/containerruntime/config.go | 132 ---------- pkg/containerruntime/config_test.go | 134 ---------- pkg/containerruntime/containerd.go | 296 ----------------------- pkg/containerruntime/containerruntime.go | 113 --------- pkg/containerruntime/flags.go | 60 ----- pkg/controller/machine/controller.go | 3 - pkg/ini/duration.go | 41 ---- pkg/ini/escape.go | 32 --- pkg/ini/escape_test.go | 87 ------- pkg/node/flags.go | 1 - 15 files changed, 45 insertions(+), 1109 deletions(-) delete mode 100644 cmd/provision/README.md delete mode 100644 pkg/containerruntime/config.go delete mode 100644 pkg/containerruntime/config_test.go delete mode 100644 pkg/containerruntime/containerd.go delete mode 100644 pkg/containerruntime/containerruntime.go delete mode 100644 pkg/containerruntime/flags.go delete mode 100644 pkg/ini/duration.go delete mode 100644 pkg/ini/escape.go delete mode 100644 pkg/ini/escape_test.go diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index cf15f855a..a2e3c2937 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -21,7 +21,6 @@ import ( "flag" "fmt" "log" - "net" "net/http" "net/http/pprof" "strings" @@ -36,7 +35,6 @@ import ( cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" clusterinfo "github.com/kubermatic/machine-controller/pkg/clusterinfo" - "github.com/kubermatic/machine-controller/pkg/containerruntime" machinecontroller "github.com/kubermatic/machine-controller/pkg/controller/machine" machinedeploymentcontroller "github.com/kubermatic/machine-controller/pkg/controller/machinedeployment" machinesetcontroller "github.com/kubermatic/machine-controller/pkg/controller/machineset" @@ -77,24 +75,33 @@ var ( enableLeaderElection bool leaderElectionNamespace string - useOSM bool - useExternalBootstrap bool - + useExternalBootstrap bool + overrideBootstrapKubeletAPIServer string nodeCSRApprover bool - nodeHTTPProxy string - nodeNoProxy string - nodeInsecureRegistries string - nodeRegistryMirrors string - nodePauseImage string - nodeContainerRuntime string - podCIDR string nodePortRange string - nodeRegistryCredentialsSecret string - nodeContainerdVersion string - nodeContainerdRegistryMirrors = containerruntime.RegistryMirrorsFlags{} - overrideBootstrapKubeletAPIServer string + + nodeHTTPProxy string + nodeNoProxy string + nodeInsecureRegistries string + nodeRegistryMirrors string + nodePauseImage string + nodeContainerRuntime string + nodeRegistryCredentialsSecret string + nodeContainerdVersion string + nodeContainerdRegistryMirrors sliceVar ) +type sliceVar []string + +func (s *sliceVar) String() string { + return strings.Join(*s, ",") +} + +func (s *sliceVar) Set(value string) error { + *s = append(*s, value) + return nil +} + const ( defaultLeaderElectionNamespace = "kube-system" defaultLeaderElectionID = "machine-controller" @@ -170,23 +177,21 @@ func main() { flag.StringVar(&bootstrapTokenServiceAccountName, "bootstrap-token-service-account-name", "", "When set use the service account token from this SA as bootstrap token instead of creating a temporary one. Passed in namespace/name format") flag.BoolVar(&profiling, "enable-profiling", false, "when set, enables the endpoints on the http server under /debug/pprof/") flag.DurationVar(&skipEvictionAfter, "skip-eviction-after", 2*time.Hour, "Skips the eviction if a machine is not gone after the specified duration.") - flag.StringVar(&nodeHTTPProxy, "node-http-proxy", "", "If set, it configures the 'HTTP_PROXY' & 'HTTPS_PROXY' environment variable on the nodes.") - flag.StringVar(&nodeNoProxy, "node-no-proxy", ".svc,.cluster.local,localhost,127.0.0.1", "If set, it configures the 'NO_PROXY' environment variable on the nodes.") - flag.StringVar(&nodeInsecureRegistries, "node-insecure-registries", "", "Comma separated list of registries which should be configured as insecure on the container runtime") - flag.StringVar(&nodeRegistryMirrors, "node-registry-mirrors", "", "Comma separated list of Docker image mirrors") - flag.StringVar(&nodePauseImage, "node-pause-image", "", "Image for the pause container including tag. If not set, the kubelet default will be used: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/") - flag.String("node-kubelet-repository", "quay.io/kubermatic/kubelet", "[NO-OP] Repository for the kubelet container. Has no effects.") - flag.StringVar(&nodeContainerRuntime, "node-container-runtime", "containerd", "container-runtime to deploy") - flag.StringVar(&nodeContainerdVersion, "node-containerd-version", "", "version of containerd to deploy") - flag.Var(&nodeContainerdRegistryMirrors, "node-containerd-registry-mirrors", "Configure registry mirrors endpoints. Can be used multiple times to specify multiple mirrors") + flag.BoolVar(&useExternalBootstrap, "use-external-bootstrap", true, "DEPRECATED: This flag is no-op and will have no effect since machine-controller only supports external bootstrap mechanism. This flag is only kept for backwards compatibility and will be removed in the future") + flag.StringVar(&overrideBootstrapKubeletAPIServer, "override-bootstrap-kubelet-apiserver", "", "Override for the API server address used in worker nodes bootstrap-kubelet.conf") flag.StringVar(&caBundleFile, "ca-bundle", "", "path to a file containing all PEM-encoded CA certificates (will be used instead of the host's certificates if set)") flag.BoolVar(&nodeCSRApprover, "node-csr-approver", true, "Enable NodeCSRApprover controller to automatically approve node serving certificate requests") - flag.StringVar(&podCIDR, "pod-cidr", "172.25.0.0/16", "WARNING: flag is unused, kept only for backwards compatibility") flag.StringVar(&nodePortRange, "node-port-range", "30000-32767", "A port range to reserve for services with NodePort visibility") - flag.StringVar(&nodeRegistryCredentialsSecret, "node-registry-credentials-secret", "", "A Secret object reference, that contains auth info for image registry in namespace/secret-name form, example: kube-system/registry-credentials. See doc at https://github.com/kubermaric/machine-controller/blob/main/docs/registry-authentication.md") - flag.BoolVar(&useOSM, "use-osm", false, "DEPRECATED: use osm controller for node bootstrap [use use-external-bootstrap instead]") - flag.BoolVar(&useExternalBootstrap, "use-external-bootstrap", true, "DEPRECATED: This flag is no-op and will have no effect since machine-controller only supports external bootstrap mechanism. This flag is only kept for backwards compatibility and will be removed in the future") - flag.StringVar(&overrideBootstrapKubeletAPIServer, "override-bootstrap-kubelet-apiserver", "", "Override for the API server address used in worker nodes bootstrap-kubelet.conf") + + flag.StringVar(&nodeHTTPProxy, "node-http-proxy", "", "DEPRECATED: This flag is no-op and will have no effect. This value should be configured in the user-data provider, such as operating-system-manager.") + flag.StringVar(&nodeNoProxy, "node-no-proxy", "", "DEPRECATED: This flag is no-op and will have no effect. This value should be configured in the user-data provider, such as operating-system-manager.") + flag.StringVar(&nodeInsecureRegistries, "node-insecure-registries", "", "DEPRECATED: This flag is no-op and will have no effect. This value should be configured in the user-data provider, such as operating-system-manager.") + flag.StringVar(&nodeRegistryMirrors, "node-registry-mirrors", "", "DEPRECATED: This flag is no-op and will have no effect. This value should be configured in the user-data provider, such as operating-system-manager.") + flag.StringVar(&nodePauseImage, "node-pause-image", "", "DEPRECATED: This flag is no-op and will have no effect. This value should be configured in the user-data provider, such as operating-system-manager.") + flag.StringVar(&nodeContainerRuntime, "node-container-runtime", "", "DEPRECATED: This flag is no-op and will have no effect. This value should be configured in the user-data provider, such as operating-system-manager.") + flag.StringVar(&nodeContainerdVersion, "node-containerd-version", "", "DEPRECATED: This flag is no-op and will have no effect. This value should be configured in the user-data provider, such as operating-system-manager.") + flag.Var(&nodeContainerdRegistryMirrors, "node-containerd-registry-mirrors", "DEPRECATED: This flag is no-op and will have no effect. This value should be configured in the user-data provider, such as operating-system-manager.") + flag.StringVar(&nodeRegistryCredentialsSecret, "node-registry-credentials-secret", "", "DEPRECATED: This flag is no-op and will have no effect. This value should be configured in the user-data provider, such as operating-system-manager.") flag.Parse() @@ -194,10 +199,6 @@ func main() { log.Fatalf("Invalid options: %v", err) } - if nodeContainerRuntime != "containerd" { - log.Fatalf("%s not supported; containerd is the only supported container runtime", nodeContainerRuntime) - } - rawLog := machinecontrollerlog.New(logFlags.Debug, logFlags.Format) log := rawLog.Sugar() @@ -207,11 +208,6 @@ func main() { kubeconfig = flag.Lookup("kubeconfig").Value.(flag.Getter).Get().(string) masterURL = flag.Lookup("master").Value.(flag.Getter).Get().(string) - clusterDNSIPs, err := parseClusterDNSIPs(clusterDNSIPs) - if err != nil { - log.Fatalw("Invalid cluster dns specified", zap.Error(err)) - } - var parsedJoinClusterTimeout *time.Duration if joinClusterTimeout != "" { parsedJoinClusterTimeoutLiteral, err := time.ParseDuration(joinClusterTimeout) @@ -261,38 +257,16 @@ func main() { ctrlMetrics := machinecontroller.NewMachineControllerMetrics() ctrlMetrics.MustRegister(metrics.Registry) - containerRuntimeOpts := containerruntime.Opts{ - ContainerRuntime: nodeContainerRuntime, - ContainerdVersion: nodeContainerdVersion, - ContainerdRegistryMirrors: nodeContainerdRegistryMirrors, - InsecureRegistries: nodeInsecureRegistries, - PauseImage: nodePauseImage, - RegistryMirrors: nodeRegistryMirrors, - RegistryCredentialsSecret: nodeRegistryCredentialsSecret, - } - containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) - if err != nil { - log.Fatalw("Failed to generate container runtime config", zap.Error(err)) - } - runOptions := controllerRunOptions{ - log: log, - kubeClient: kubeClient, - kubeconfigProvider: kubeconfigProvider, - name: name, - cfg: machineCfg, - metrics: ctrlMetrics, - prometheusRegisterer: metrics.Registry, - skipEvictionAfter: skipEvictionAfter, - nodeCSRApprover: nodeCSRApprover, - node: machinecontroller.NodeSettings{ - ClusterDNSIPs: clusterDNSIPs, - HTTPProxy: nodeHTTPProxy, - NoProxy: nodeNoProxy, - PauseImage: nodePauseImage, - RegistryCredentialsSecretRef: nodeRegistryCredentialsSecret, - ContainerRuntime: containerRuntimeConfig, - }, + log: log, + kubeClient: kubeClient, + kubeconfigProvider: kubeconfigProvider, + name: name, + cfg: machineCfg, + metrics: ctrlMetrics, + prometheusRegisterer: metrics.Registry, + skipEvictionAfter: skipEvictionAfter, + nodeCSRApprover: nodeCSRApprover, nodePortRange: nodePortRange, overrideBootstrapKubeletAPIServer: overrideBootstrapKubeletAPIServer, } @@ -457,16 +431,3 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { return nil } - -func parseClusterDNSIPs(s string) ([]net.IP, error) { - var ips []net.IP - sips := strings.Split(s, ",") - for _, sip := range sips { - ip := net.ParseIP(strings.TrimSpace(sip)) - if ip == nil { - return nil, fmt.Errorf("failed to parse IP %q", sip) - } - ips = append(ips, ip) - } - return ips, nil -} diff --git a/cmd/provision/README.md b/cmd/provision/README.md deleted file mode 100644 index 4811de7dc..000000000 --- a/cmd/provision/README.md +++ /dev/null @@ -1,84 +0,0 @@ -# Provisioning - -This command offers all required functionality to provision an host to join a Kubernetes cluster. - -The following operating systems are supported - -- Ubuntu 18.04 -- CentOS 7 -- Flatcar - -## Requirements - -- The cluster needs to use the bootstrap token authentication - -## CLI - -```bash -./provision \ - --kubelet-version="v1.13.1" \ - --cloud-provider="openstack" \ - --cloud-config="/etc/kubernetes/cloud-config" \ - --token="AAAAAAAAAAAAAAAA" \ - --ca-cert="/etc/kubernetes/ca.crt" -``` - -## Process - -Nodes will boot with a cloud-init (Or Ignition) which writes required files & a shell script (called `setup.sh` here). - -### cloud-init (Or ignition) - -Parts which will be covered by cloud-init (or Ignition) - -- Install SSH keys -- Configure hostname -- `ca.crt` - The CA certificate which got used to issue the certificates of the API server serving certificates -- `cloud-config` - A optional cloud-config used by the kubelet to interact with the cloud provider. -- `setup.sh` - Is responsible for downloading the `provision` binary and to execute it. - The download of the binary might also be done using built-in `cloud-init` (or Ignition) features - -### Provision - -The `provision` binary will identify the operating system and execute a set of provisioning steps. - -The provisioning process gets separated into 2 phases: - -- Base provisioning - Install and configure all required dependencies -- Join - Write & start the kubelet systemd unit - -#### Base provisioning - -The following steps belong into the base provisioning: - -- Install required packages (apt & yum action) -- Configure required kernel parameter (Like ip forwarding, etc.) -- Configure required kernel modules -- Disable swap -- Download & install the CNI plugins -- Download & Install docker -- Download Kubelet -- Install health checks (Kubelet & Docker) - -#### Join - -This part will: - -- Write & start the kubelet systemd unit - -## Offline usage - -The `provision` binary should also be usable for "prebaking" images, which then can be used for offline usage. - -## Development process - -To make sure the local development version of the `provision` command gets used for new machines created by the local running machine controller, -a new flag `--provision-source` must be introduced. -This flag will instruct the machine controller to download the `provision` binary from the specified location. - -For simplicity the `/hack/run-machine-controller.sh` will be updated to include a step which will compile the `provoision` command & upload it to a gcs bucket. diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index cc48082ae..aecc45282 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -42,7 +42,6 @@ type options struct { admissionTLSCertPath string admissionTLSKeyPath string caBundleFile string - useOSM bool useExternalBootstrap bool namespace string workerClusterKubeconfig string @@ -70,8 +69,6 @@ func main() { flag.StringVar(&opt.workerClusterKubeconfig, "worker-cluster-kubeconfig", "", "Path to kubeconfig of worker/user cluster where machines and machinedeployments exist. If not specified, value from --kubeconfig or in-cluster config will be used") flag.StringVar(&opt.versionConstraint, "kubernetes-version-constraints", ">=0.0.0", "") - // OSM specific flags - flag.BoolVar(&opt.useOSM, "use-osm", false, "DEPRECATED: osm controller is enabled for node bootstrap [use use-external-bootstrap instead]") flag.BoolVar(&opt.useExternalBootstrap, "use-external-bootstrap", true, "DEPRECATED: This flag is no-op and will have no effect since machine-controller only supports external bootstrap mechanism. This flag is only kept for backwards compatibility and will be removed in the future") flag.Parse() diff --git a/go.mod b/go.mod index 37b7d00b9..4ae2bb6c7 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/kubermatic/machine-controller go 1.22.0 -toolchain go1.22.2 +toolchain go1.22.4 require ( cloud.google.com/go/logging v1.9.0 @@ -10,9 +10,7 @@ require ( github.com/Azure/azure-sdk-for-go v68.0.0+incompatible github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 github.com/Azure/go-autorest/autorest/to v0.4.0 - github.com/BurntSushi/toml v1.3.2 github.com/Masterminds/semver/v3 v3.2.1 - github.com/Masterminds/sprig/v3 v3.2.3 github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240311170802-57efabf2d590 github.com/aliyun/alibaba-cloud-sdk-go v1.62.695 github.com/aws/aws-sdk-go-v2 v1.25.3 @@ -39,7 +37,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_golang v1.19.0 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25 - github.com/sethvargo/go-password v0.2.0 github.com/spf13/pflag v1.0.5 github.com/tinkerbell/tink v0.8.0 github.com/vmware/go-vcloud-director/v2 v2.22.0 @@ -52,7 +49,6 @@ require ( gomodules.xyz/jsonpatch/v2 v2.4.0 google.golang.org/api v0.170.0 google.golang.org/grpc v1.62.1 - gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.30.0 k8s.io/apiextensions-apiserver v0.30.0 @@ -80,7 +76,6 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/Masterminds/goutils v1.1.1 // indirect github.com/PaesslerAG/gval v1.2.2 // indirect github.com/PaesslerAG/jsonpath v0.1.1 // indirect github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect @@ -123,7 +118,6 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.5 // indirect github.com/hashicorp/go-version v1.6.0 // indirect - github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -151,7 +145,6 @@ require ( github.com/prometheus/procfs v0.12.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect - github.com/spf13/cast v1.6.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect @@ -176,7 +169,6 @@ require ( gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect diff --git a/go.sum b/go.sum index 4d8f9a6e0..43b9b245e 100644 --- a/go.sum +++ b/go.sum @@ -84,17 +84,10 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= -github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= -github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= -github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= -github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240311170802-57efabf2d590 h1:wvNejQUL/d0Z2n4DZfAtAQv+/fUFrFSkLj3X49ioDiM= @@ -226,8 +219,6 @@ github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= -github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -390,7 +381,6 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -428,13 +418,9 @@ github.com/hetznercloud/hcloud-go v1.53.0 h1:xThhlJc6MdpvDAqVB7bAw+nAQuCpQMwsf3y github.com/hetznercloud/hcloud-go v1.53.0/go.mod h1:VzDWThl47lOnZXY0q5/LPFD+M62pfe/52TV+mOrpp9Q= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= -github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= @@ -514,14 +500,12 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= @@ -635,9 +619,6 @@ github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25 h1:/8rfZAdFfafRXOgz+ZpMZZWZ5p github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI= -github.com/sethvargo/go-password v0.2.0/go.mod h1:Ym4Mr9JXLBycr02MFuVQ/0JHidNetSgbzutTr3zsYXE= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -646,9 +627,6 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= -github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= @@ -774,7 +752,6 @@ golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= @@ -873,7 +850,6 @@ golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= @@ -980,7 +956,6 @@ golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -989,7 +964,6 @@ golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= @@ -1005,7 +979,6 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= @@ -1228,16 +1201,12 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= -gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/containerruntime/config.go b/pkg/containerruntime/config.go deleted file mode 100644 index 65b8815be..000000000 --- a/pkg/containerruntime/config.go +++ /dev/null @@ -1,132 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package containerruntime - -import ( - "context" - "encoding/json" - "fmt" - "net/url" - "regexp" - "strings" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" -) - -type Opts struct { - ContainerRuntime string - ContainerdVersion string - InsecureRegistries string - RegistryMirrors string - RegistryCredentialsSecret string - PauseImage string - ContainerdRegistryMirrors RegistryMirrorsFlags -} - -type DockerCfgJSON struct { - Auths map[string]AuthConfig `json:"auths,omitempty"` -} - -func BuildConfig(opts Opts) (Config, error) { - var insecureRegistries []string - for _, registry := range strings.Split(opts.InsecureRegistries, ",") { - if trimmedRegistry := strings.TrimSpace(registry); trimmedRegistry != "" { - insecureRegistries = append(insecureRegistries, trimmedRegistry) - } - } - - // we want to match e.g. docker.io=registry.docker-cn.com, having docker.io as the first - // match group and registry.docker-cn.com as the second one. - registryMirrorRegexp := regexp.MustCompile(`^([a-zA-Z0-9\.-]+)=(.*)`) - - if opts.ContainerdRegistryMirrors == nil { - opts.ContainerdRegistryMirrors = make(RegistryMirrorsFlags) - } - - for _, mirror := range strings.Split(opts.RegistryMirrors, ",") { - if trimmedMirror := strings.TrimSpace(mirror); trimmedMirror != "" { - registry := "docker.io" - - if matches := registryMirrorRegexp.FindStringSubmatch(trimmedMirror); matches != nil { - registry = matches[1] - trimmedMirror = matches[2] - } - - if !strings.HasPrefix(trimmedMirror, "http") { - trimmedMirror = "https://" + trimmedMirror - } - - _, err := url.Parse(trimmedMirror) - if err != nil { - return Config{}, fmt.Errorf("incorrect mirror provided: %w", err) - } - - if opts.ContainerdRegistryMirrors[registry] == nil { - opts.ContainerdRegistryMirrors[registry] = make([]string, 0, 1) - } - - opts.ContainerdRegistryMirrors[registry] = append(opts.ContainerdRegistryMirrors[registry], trimmedMirror) - } - } - - // Only validate registry credential here - if opts.RegistryCredentialsSecret != "" { - if secRef := strings.Split(opts.RegistryCredentialsSecret, "/"); len(secRef) != 2 { - return Config{}, fmt.Errorf("-node-registry-credentials-secret is in incorrect format %q, should be in 'namespace/secretname'", opts.RegistryCredentialsSecret) - } - } - - return get( - opts.ContainerRuntime, - withInsecureRegistries(insecureRegistries), - withRegistryMirrors(opts.ContainerdRegistryMirrors), - withSandboxImage(opts.PauseImage), - withContainerdVersion(opts.ContainerdVersion), - ), nil -} - -func GetContainerdAuthConfig(ctx context.Context, client ctrlruntimeclient.Client, registryCredentialsSecret string) (map[string]AuthConfig, error) { - registryCredentials := map[string]AuthConfig{} - - if secRef := strings.SplitN(registryCredentialsSecret, "/", 2); len(secRef) == 2 { - var credsSecret corev1.Secret - err := client.Get(ctx, types.NamespacedName{Namespace: secRef[0], Name: secRef[1]}, &credsSecret) - if err != nil { - return nil, fmt.Errorf("failed to retrieve registry credentials secret object: %w", err) - } - - switch credsSecret.Type { - case corev1.SecretTypeDockerConfigJson: - var regCred DockerCfgJSON - if err := json.Unmarshal(credsSecret.Data[".dockerconfigjson"], ®Cred); err != nil { - return nil, fmt.Errorf("failed to unmarshal registry credentials: %w", err) - } - registryCredentials = regCred.Auths - default: - for registry, data := range credsSecret.Data { - var regCred AuthConfig - if err := json.Unmarshal(data, ®Cred); err != nil { - return nil, fmt.Errorf("failed to unmarshal registry credentials: %w", err) - } - registryCredentials[registry] = regCred - } - } - } - return registryCredentials, nil -} diff --git a/pkg/containerruntime/config_test.go b/pkg/containerruntime/config_test.go deleted file mode 100644 index 4ee6ecd79..000000000 --- a/pkg/containerruntime/config_test.go +++ /dev/null @@ -1,134 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package containerruntime - -import ( - "errors" - "fmt" - "testing" -) - -func TestContainerdRegistryMirror(t *testing.T) { - type testCase struct { - desc string - flag string - expectedMirrors map[string][]string - expectedError error - } - - testCases := []testCase{ - { - desc: "no registry mirrors set", - flag: "", - expectedMirrors: map[string][]string{}, - expectedError: nil, - }, - - { - desc: "registry mirror without name and protocol", - flag: "registry-v1.docker.io", - expectedMirrors: map[string][]string{ - "docker.io": {"https://registry-v1.docker.io/"}, - }, - expectedError: nil, - }, - { - desc: "multiple registry mirrors without name, with and without protocol", - flag: "registry-v1.docker.io,http://registry.docker-cn.com", - expectedMirrors: map[string][]string{ - "docker.io": { - "https://registry-v1.docker.io/", - "http://registry.docker-cn.com/", - }, - }, - expectedError: nil, - }, - - { - desc: "registry mirror with name and without protocol", - flag: "quay.io=my-quay-io-mirror.example.com", - expectedMirrors: map[string][]string{ - "quay.io": {"https://my-quay-io-mirror.example.com/"}, - }, - expectedError: nil, - }, - { - desc: "registry mirror with name and protocol", - flag: "quay.io=http://my-quay-io-mirror.example.com", - expectedMirrors: map[string][]string{ - "quay.io": {"http://my-quay-io-mirror.example.com/"}, - }, - expectedError: nil, - }, - { - desc: "multiple registry mirrors with same name", - flag: "quay.io=http://my-quay-io-mirror.example.com,quay.io=example.net", - expectedMirrors: map[string][]string{ - "quay.io": { - "http://my-quay-io-mirror.example.com/", - "https://example.net/", - }, - }, - expectedError: nil, - }, - - { - desc: "complex example", - flag: "quay.io=http://my-quay-io-mirror.example.com,quay.io=example.net," + - "registry-v1.docker.io,http://registry.docker-cn.com," + - "ghcr.io=http://foo/bar", - expectedMirrors: map[string][]string{ - "quay.io": { - "http://my-quay-io-mirror.example.com/", - "https://example.net/", - }, - "docker.io": { - "https://registry-v1.docker.io/", - "http://registry.docker-cn.com/", - }, - "ghcr.io": { - "http://foo/bar", - }, - }, - expectedError: nil, - }, - } - - for _, tc := range testCases { - t.Run(tc.desc, func(t *testing.T) { - opts := Opts{ - ContainerRuntime: containerdName, - RegistryMirrors: tc.flag, - } - - config, err := BuildConfig(opts) - if tc.expectedError != nil { - if !errors.Is(err, tc.expectedError) { - t.Errorf("expected error %q but got %q", tc.expectedError, err) - } - } - - if err != nil { - t.Errorf("expected success but got error: %q", err) - } - - if fmt.Sprint(config.RegistryMirrors) != fmt.Sprint(tc.expectedMirrors) { - t.Errorf("expected to get %v instead got: %v", tc.expectedMirrors, config.RegistryMirrors) - } - }) - } -} diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go deleted file mode 100644 index 2b5250658..000000000 --- a/pkg/containerruntime/containerd.go +++ /dev/null @@ -1,296 +0,0 @@ -/* -Copyright 2020 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package containerruntime - -import ( - "fmt" - "strings" - "text/template" - - "github.com/BurntSushi/toml" - - "github.com/kubermatic/machine-controller/pkg/providerconfig/types" -) - -const ( - DefaultContainerdVersion = "1.6*" -) - -type Containerd struct { - insecureRegistries []string - registryMirrors map[string][]string - sandboxImage string - registryCredentials map[string]AuthConfig - version string -} - -func (eng *Containerd) ConfigFileName() string { - return "/etc/containerd/config.toml" -} - -func (eng *Containerd) AuthConfig() (string, error) { - return "", nil -} - -func (eng *Containerd) AuthConfigFileName() string { - return "" -} - -func (eng *Containerd) KubeletFlags() []string { - return []string{ - "--container-runtime-endpoint=unix:///run/containerd/containerd.sock", - } -} - -func (eng *Containerd) ScriptFor(os types.OperatingSystem) (string, error) { - var buf strings.Builder - - args := struct { - ContainerdVersion string - }{ - ContainerdVersion: DefaultContainerdVersion, - } - - if eng.version != "" { - args.ContainerdVersion = eng.version - } - - switch os { - case types.OperatingSystemAmazonLinux2: - err := containerdAmzn2Template.Execute(&buf, args) - return buf.String(), err - case types.OperatingSystemCentOS, types.OperatingSystemRHEL, types.OperatingSystemRockyLinux: - err := containerdYumTemplate.Execute(&buf, args) - return buf.String(), err - case types.OperatingSystemUbuntu: - err := containerdAptTemplate.Execute(&buf, args) - return buf.String(), err - case types.OperatingSystemFlatcar: - err := containedFlatcarTemplate.Execute(&buf, args) - return buf.String(), err - } - - return "", fmt.Errorf("unknown OS: %s", os) -} - -var ( - containedFlatcarTemplate = template.Must(template.New("containerd-flatcar").Parse(` -mkdir -p /etc/systemd/system/containerd.service.d - -cat < Date: Thu, 6 Jun 2024 15:24:04 +0500 Subject: [PATCH 395/489] Use cluster-exposer from build image (#1813) Signed-off-by: Waleed Malik --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 14 +++++++------- .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 10 +++++----- .prow/verify.yaml | 16 ++++++++-------- hack/ci/setup-kind-cluster.sh | 17 ++--------------- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- 21 files changed, 46 insertions(+), 59 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 53350366a..d79c4c156 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -119,7 +119,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index f368c9344..505b1930a 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index ccd97f5ae..fe85be652 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index fff0e0b0c..a0509c80b 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index e96d3fe5d..92e0bba49 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -61,7 +61,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -126,7 +126,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -157,7 +157,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -188,7 +188,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -219,7 +219,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index e7bfd42c3..2965f708c 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 8defef915..97a8e66e3 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 7ed2dd2b9..f331ad08c 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 197bbcd86..d94ddd6bb 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 43282a3f1..3b39a7457 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 94f9693a0..d0ae3e087 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 71ebc222e..25b306a43 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index e6a21ea30..71b713648 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 4de152975..d9c66be28 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 3507c853c..a1f6cfb47 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 59e380f19..4dd09ca07 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index fb386b0a9..f082e4193 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -156,7 +156,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 8efda6f92..3b6b4babe 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-10 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-10 command: - make args: @@ -63,7 +63,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-10 command: - make args: @@ -83,7 +83,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-10 command: - make args: @@ -102,7 +102,7 @@ presubmits: clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-10 command: - "/usr/local/bin/shfmt" args: @@ -130,7 +130,7 @@ presubmits: clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-10 command: - "./hack/verify-boilerplate.sh" resources: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-10 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-9 + - image: quay.io/kubermatic/build:go-1.22-node-20-10 command: - make args: diff --git a/hack/ci/setup-kind-cluster.sh b/hack/ci/setup-kind-cluster.sh index c68467511..14b29aed0 100755 --- a/hack/ci/setup-kind-cluster.sh +++ b/hack/ci/setup-kind-cluster.sh @@ -150,22 +150,9 @@ if [ -z "${DISABLE_CLUSTER_EXPOSER:-}" ]; then # Start cluster exposer, which will expose services from within kind as # a NodePort service on the host - echodate "Starting cluster exposer" - ( - # Clone kubermatic repo to build clusterexposer - mkdir -p /tmp/kubermatic - cd /tmp/kubermatic - echodate "Cloning cluster exposer" - KKP_REPO_URL="${KKP_REPO_URL:-https://github.com/kubermatic/kubermatic.git}" - KKP_REPO_TAG="${KKP_REPO_BRANCH:-main}" - git clone --depth 1 --branch "${KKP_REPO_TAG}" "${KKP_REPO_URL}" . - - echodate "Building cluster exposer" - CGO_ENABLED=0 go build --tags ce -v -o /tmp/clusterexposer ./pkg/test/clusterexposer/cmd - ) - export KUBECONFIG=~/.kube/config - /tmp/clusterexposer \ + echodate "Starting cluster exposer" + clusterexposer \ --kubeconfig-inner "$KUBECONFIG" \ --kubeconfig-outer "/etc/kubeconfig/kubeconfig" \ --build-id "$PROW_JOB_ID" &> /var/log/clusterexposer.log & diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 807d089b0..75b66ead0 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.22.4 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-20-10 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index b49f9f9c0..bad984802 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-9 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-20-10 containerize ./hack/verify-licenses.sh go mod vendor From 300e7d6778a8c32b1283fc2eb88ed1a90914c97f Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 6 Jun 2024 18:41:04 +0500 Subject: [PATCH 396/489] Deprecate cluster-dns flag (#1814) Signed-off-by: Waleed Malik --- cmd/machine-controller/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index a2e3c2937..8926b61c8 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -165,7 +165,7 @@ func main() { if flag.Lookup("master") == nil { flag.StringVar(&masterURL, "master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.") } - flag.StringVar(&clusterDNSIPs, "cluster-dns", "10.10.10.10", "Comma-separated list of DNS server IP address.") + flag.StringVar(&clusterDNSIPs, "cluster-dns", "", "DEPRECATED: This flag is no-op and will have no effect. This value should be configured in the user-data provider, such as operating-system-manager.") flag.IntVar(&workerCount, "worker-count", 1, "Number of workers to process machines. Using a high number with a lot of machines might cause getting rate-limited from your cloud provider.") flag.StringVar(&healthProbeAddress, "health-probe-address", "127.0.0.1:8085", "The address on which the liveness check on /healthz and readiness check on /readyz will be available") flag.StringVar(&metricsAddress, "metrics-address", "127.0.0.1:8080", "The address on which Prometheus metrics will be available under /metrics") From 635188ca82fc5f214995256baaa4c78bde9bd8e8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Jun 2024 10:11:20 +0300 Subject: [PATCH 397/489] Bump github.com/hashicorp/go-retryablehttp from 0.7.5 to 0.7.7 (#1819) Bumps [github.com/hashicorp/go-retryablehttp](https://github.com/hashicorp/go-retryablehttp) from 0.7.5 to 0.7.7. - [Changelog](https://github.com/hashicorp/go-retryablehttp/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/go-retryablehttp/compare/v0.7.5...v0.7.7) --- updated-dependencies: - dependency-name: github.com/hashicorp/go-retryablehttp dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 4ae2bb6c7..0677b7c9e 100644 --- a/go.mod +++ b/go.mod @@ -116,7 +116,7 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.2 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-retryablehttp v0.7.5 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -156,7 +156,7 @@ require ( golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/sys v0.20.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect diff --git a/go.sum b/go.sum index 43b9b245e..205d8dd90 100644 --- a/go.sum +++ b/go.sum @@ -214,6 +214,8 @@ github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= @@ -404,10 +406,10 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= -github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -494,10 +496,14 @@ github.com/matryer/moq v0.2.3/go.mod h1:9RtPYjTnH1bSBIkpvtHkFN7nbWAnO7oRpdJkEIn6 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= @@ -959,8 +965,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From 9db708c6da83e8b49800ca6cf41a18adc3dac696 Mon Sep 17 00:00:00 2001 From: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> Date: Tue, 25 Jun 2024 10:13:19 +0200 Subject: [PATCH 398/489] Hetzner: Skip operating system validation if custom image is provided (#1818) This should make it possible to use operating systems on Hetzner Cloud, that are not natively supported, by using custom images (snapshots). A side-effect of this commit is, that images returned by `getNameForOs()` are now also validated against the Hetzner API. For reference: #1817 Requirements to use custom operating systems: 1. Create a snapshot of the operating system you want to use on Hetzner Cloud, e.g. using https://github.com/apricote/hcloud-upload-image/ 2. Add a new `OperatingSystemProfile` 3. Set the image in `MachineDeployment` to the name of the snapshot created in step 1 Signed-off-by: Pascal Sthamer <10992664+P4sca1@users.noreply.github.com> --- .../provider/hetzner/provider.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index e779b2b08..229b93a3c 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -200,11 +200,6 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus return errors.New("token is missing") } - _, err = getNameForOS(pc.OperatingSystem) - if err != nil { - return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, err) - } - client := getClient(c.Token) if c.Location != "" && c.Datacenter != "" { @@ -223,13 +218,19 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus } } - if c.Image != "" { - //nolint:staticcheck // We do not have the architecture available here. - if _, _, err = client.Image.Get(ctx, c.Image); err != nil { - return fmt.Errorf("failed to get image: %w", err) + image := c.Image + if image == "" { + image, err = getNameForOS(pc.OperatingSystem) + if err != nil { + return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, err) } } + //nolint:staticcheck // We do not have the architecture available here. + if _, _, err = client.Image.Get(ctx, image); err != nil { + return fmt.Errorf("failed to get image: %w", err) + } + for _, network := range c.Networks { if _, _, err = client.Network.Get(ctx, network); err != nil { return fmt.Errorf("failed to get network %q: %w", network, err) From 8f3166ae03b0a555bd518382ff668518a694ff44 Mon Sep 17 00:00:00 2001 From: 7oku <8048380+7oku@users.noreply.github.com> Date: Thu, 27 Jun 2024 08:01:20 +0200 Subject: [PATCH 399/489] Support for Hetzner arm64 architecture instances (#1816) * Support for Hetzner arm64 architecture instances Signed-off-by: 7oku <8048380+7oku@users.noreply.github.com> * fix conflict with #1818 Signed-off-by: 7oku <8048380+7oku@users.noreply.github.com> * remove leftovers from merge Signed-off-by: 7oku <8048380+7oku@users.noreply.github.com> --------- Signed-off-by: 7oku <8048380+7oku@users.noreply.github.com> --- go.mod | 18 +++++----- go.sum | 36 +++++++++---------- .../provider/hetzner/provider.go | 35 +++++++++--------- 3 files changed, 44 insertions(+), 45 deletions(-) diff --git a/go.mod b/go.mod index 0677b7c9e..4cef2d7c9 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/google/uuid v1.6.0 github.com/gophercloud/gophercloud v1.11.0 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb - github.com/hetznercloud/hcloud-go v1.53.0 + github.com/hetznercloud/hcloud-go/v2 v2.10.1 github.com/linode/linodego v1.30.0 github.com/nutanix-cloud-native/prism-go-client v0.3.4 github.com/packethost/packngo v0.31.0 @@ -35,7 +35,7 @@ require ( github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 - github.com/prometheus/client_golang v1.19.0 + github.com/prometheus/client_golang v1.19.1 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25 github.com/spf13/pflag v1.0.5 github.com/tinkerbell/tink v0.8.0 @@ -44,7 +44,7 @@ require ( github.com/vultr/govultr/v3 v3.6.4 go.anx.io/go-anxcloud v0.6.4 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.21.0 + golang.org/x/crypto v0.24.0 golang.org/x/oauth2 v0.18.0 gomodules.xyz/jsonpatch/v2 v2.4.0 google.golang.org/api v0.170.0 @@ -154,13 +154,13 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.18.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect diff --git a/go.sum b/go.sum index 205d8dd90..702f98a8c 100644 --- a/go.sum +++ b/go.sum @@ -416,8 +416,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb h1:tsEKRC3PU9rMw18w/uAptoijhgG4EvlA5kfJPtwrMDk= github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= -github.com/hetznercloud/hcloud-go v1.53.0 h1:xThhlJc6MdpvDAqVB7bAw+nAQuCpQMwsf3yanCis4rM= -github.com/hetznercloud/hcloud-go v1.53.0/go.mod h1:VzDWThl47lOnZXY0q5/LPFD+M62pfe/52TV+mOrpp9Q= +github.com/hetznercloud/hcloud-go/v2 v2.10.1 h1:MLfIJJvl7qo79Q+52YmrJ4BOTL5oICJWHjzb0sCtgQI= +github.com/hetznercloud/hcloud-go/v2 v2.10.1/go.mod h1:xQ+8KhIS62W0D78Dpi57jsufWh844gUw1az5OUvaeq8= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -585,8 +585,8 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= -github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -760,8 +760,8 @@ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -859,8 +859,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -889,8 +889,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -965,16 +965,16 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -988,8 +988,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1061,8 +1061,8 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= -golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 229b93a3c..61fdb2284 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -24,7 +24,7 @@ import ( "strconv" "strings" - "github.com/hetznercloud/hcloud-go/hcloud" + "github.com/hetznercloud/hcloud-go/v2/hcloud" "go.uber.org/zap" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" @@ -218,6 +218,11 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus } } + serverType, _, err := client.ServerType.Get(ctx, c.ServerType) + if err != nil { + return fmt.Errorf("failed to get server type: %w", err) + } + image := c.Image if image == "" { image, err = getNameForOS(pc.OperatingSystem) @@ -226,8 +231,7 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus } } - //nolint:staticcheck // We do not have the architecture available here. - if _, _, err = client.Image.Get(ctx, image); err != nil { + if _, _, err = client.Image.GetForArchitecture(ctx, image, serverType.Architecture); err != nil { return fmt.Errorf("failed to get image: %w", err) } @@ -251,10 +255,6 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus return errors.New("server should have either a public ipv4, ipv6 or dedicated network") } - if _, _, err = client.ServerType.Get(ctx, c.ServerType); err != nil { - return fmt.Errorf("failed to get server type: %w", err) - } - return nil } @@ -348,16 +348,6 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * serverCreateOpts.Firewalls = append(serverCreateOpts.Firewalls, &hcloud.ServerCreateFirewall{Firewall: *n}) } - //nolint:staticcheck // We do not have the architecture available here. - image, _, err := client.Image.Get(ctx, c.Image) - if err != nil { - return nil, hzErrorToTerminalError(err, "failed to get image") - } - if image == nil { - return nil, fmt.Errorf("image %q does not exist", c.Image) - } - serverCreateOpts.Image = image - serverType, _, err := client.ServerType.Get(ctx, c.ServerType) if err != nil { return nil, hzErrorToTerminalError(err, "failed to get server type") @@ -367,6 +357,15 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * } serverCreateOpts.ServerType = serverType + image, _, err := client.Image.GetForArchitecture(ctx, c.Image, serverType.Architecture) + if err != nil { + return nil, hzErrorToTerminalError(err, "failed to get image") + } + if image == nil { + return nil, fmt.Errorf("image %q does not exist", c.Image) + } + serverCreateOpts.Image = image + // We generate a temporary SSH key here, because otherwise Hetzner creates // a password and sends it via E-Mail to the account owner, which can be quite // spammy. No one will ever get access to the private key. @@ -544,7 +543,7 @@ func (s *hetznerServer) Name() string { } func (s *hetznerServer) ID() string { - return strconv.Itoa(s.server.ID) + return strconv.FormatInt(s.server.ID, 10) } func (s *hetznerServer) ProviderID() string { From 3dffe174449ee5d4408721e545e38a5965ea1ac6 Mon Sep 17 00:00:00 2001 From: Archana Sawant Date: Mon, 8 Jul 2024 12:49:11 +0530 Subject: [PATCH 400/489] Bump Go version to 1.22.5 (#1822) Signed-off-by: archups --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 14 +++++++------- .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 10 +++++----- .prow/verify.yaml | 16 ++++++++-------- Dockerfile | 2 +- Makefile | 2 +- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- 22 files changed, 46 insertions(+), 46 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index d79c4c156..335a7e498 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -119,7 +119,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 505b1930a..3e0d190bd 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index fe85be652..8a19efd95 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index a0509c80b..60ab55363 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 92e0bba49..56933396c 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -61,7 +61,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -126,7 +126,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -157,7 +157,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -188,7 +188,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -219,7 +219,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 2965f708c..5775b0724 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 97a8e66e3..41bc48629 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index f331ad08c..8bcb58d1c 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index d94ddd6bb..43b17c324 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 3b39a7457..27601cde5 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index d0ae3e087..da5f331ac 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 25b306a43..752e61e71 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 71b713648..0669747b2 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index d9c66be28..2d10d1236 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index a1f6cfb47..8a1cc2a4c 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 4dd09ca07..3e9f887f6 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index f082e4193..21c27cbad 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -156,7 +156,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 3b6b4babe..003609082 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-11 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-11 command: - make args: @@ -63,7 +63,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-11 command: - make args: @@ -83,7 +83,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-11 command: - make args: @@ -102,7 +102,7 @@ presubmits: clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-11 command: - "/usr/local/bin/shfmt" args: @@ -130,7 +130,7 @@ presubmits: clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-11 command: - "./hack/verify-boilerplate.sh" resources: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-11 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-10 + - image: quay.io/kubermatic/build:go-1.22-node-20-11 command: - make args: diff --git a/Dockerfile b/Dockerfile index a32f1ec50..3428f4fc6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.22.4 +ARG GO_VERSION=1.22.5 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index dadb53b27..ef58a6c26 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.22.4 +GO_VERSION ?= 1.22.5 GOOS ?= $(shell go env GOOS) diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 75b66ead0..3bb93987d 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-20-10 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-20-11 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index bad984802..cef2e0291 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-20-10 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-20-11 containerize ./hack/verify-licenses.sh go mod vendor From 25971d5e8ea6f9330a26f63b5b65f872fac5a0e0 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 8 Jul 2024 17:46:11 +0500 Subject: [PATCH 401/489] Update to latest k8s patches for e2e tests (#1824) Signed-off-by: Waleed Malik --- .prow/provider-vmware-cloud-director.yaml | 2 ++ examples/alibaba-machinedeployment.yaml | 2 +- examples/anexia-machinedeployment.yaml | 2 +- examples/aws-machinedeployment.yaml | 2 +- examples/azure-machinedeployment.yaml | 2 +- examples/digitalocean-machinedeployment.yaml | 2 +- examples/equinixmetal-machinedeployment.yaml | 2 +- examples/gce-machinedeployment.yaml | 2 +- examples/hetzner-machinedeployment.yaml | 2 +- examples/kubevirt-machinedeployment.yaml | 2 +- examples/linode-machinedeployment.yaml | 2 +- examples/nutanix-machinedeployment.yaml | 2 +- examples/opennebula-machinedeployment.yaml | 2 +- examples/openstack-machinedeployment.yaml | 2 +- examples/scaleway-machinedeployment.yaml | 2 +- ...ware-cloud-director-machinedeployment.yaml | 2 +- ...e-datastore-cluster-machinedeployment.yaml | 2 +- examples/vsphere-machinedeployment.yaml | 2 +- examples/vultr-machinedeployment.yaml | 2 +- test/e2e/provisioning/all_e2e_test.go | 20 +++++++++---------- test/e2e/provisioning/helper.go | 6 +++--- ...hinedeployment-azure-redhat-satellite.yaml | 2 +- 22 files changed, 34 insertions(+), 32 deletions(-) diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 3e9f887f6..27494b3fa 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -16,6 +16,8 @@ presubmits: - name: pull-machine-controller-e2e-vmware-cloud-director always_run: false decorate: true + # Please check: https://github.com/kubermatic/machine-controller/issues/1619 + optional: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" run_if_changed: "(pkg/cloudprovider/provider/vmwareclouddirector/)" labels: diff --git a/examples/alibaba-machinedeployment.yaml b/examples/alibaba-machinedeployment.yaml index c34377b9b..123a2279c 100644 --- a/examples/alibaba-machinedeployment.yaml +++ b/examples/alibaba-machinedeployment.yaml @@ -62,4 +62,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index f1366e358..85b5f986f 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -59,4 +59,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/aws-machinedeployment.yaml b/examples/aws-machinedeployment.yaml index 76443a95a..5fc670744 100644 --- a/examples/aws-machinedeployment.yaml +++ b/examples/aws-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/azure-machinedeployment.yaml b/examples/azure-machinedeployment.yaml index ddfb72bbe..81372e487 100644 --- a/examples/azure-machinedeployment.yaml +++ b/examples/azure-machinedeployment.yaml @@ -91,4 +91,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/digitalocean-machinedeployment.yaml b/examples/digitalocean-machinedeployment.yaml index 02a2810a4..30855854e 100644 --- a/examples/digitalocean-machinedeployment.yaml +++ b/examples/digitalocean-machinedeployment.yaml @@ -57,4 +57,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/equinixmetal-machinedeployment.yaml b/examples/equinixmetal-machinedeployment.yaml index 94ec553ad..c553209c6 100644 --- a/examples/equinixmetal-machinedeployment.yaml +++ b/examples/equinixmetal-machinedeployment.yaml @@ -51,4 +51,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index aa220f166..277e8a2d1 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -90,4 +90,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/hetzner-machinedeployment.yaml b/examples/hetzner-machinedeployment.yaml index 3384e5dbb..80b9184d7 100644 --- a/examples/hetzner-machinedeployment.yaml +++ b/examples/hetzner-machinedeployment.yaml @@ -67,4 +67,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index a58fabb2b..377654ee5 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -69,4 +69,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/linode-machinedeployment.yaml b/examples/linode-machinedeployment.yaml index 3757be954..e13b2f91d 100644 --- a/examples/linode-machinedeployment.yaml +++ b/examples/linode-machinedeployment.yaml @@ -53,4 +53,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/nutanix-machinedeployment.yaml b/examples/nutanix-machinedeployment.yaml index bc20126c1..21e0b9958 100644 --- a/examples/nutanix-machinedeployment.yaml +++ b/examples/nutanix-machinedeployment.yaml @@ -82,4 +82,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/opennebula-machinedeployment.yaml b/examples/opennebula-machinedeployment.yaml index b706270ce..68a32cab0 100644 --- a/examples/opennebula-machinedeployment.yaml +++ b/examples/opennebula-machinedeployment.yaml @@ -67,4 +67,4 @@ spec: # use cloud-init for flatcar as ignition doesn't know anything about OpenNebula yet provisioningUtility: "cloud-init" versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index 1de28e02c..2c9255f9f 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -166,4 +166,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/scaleway-machinedeployment.yaml b/examples/scaleway-machinedeployment.yaml index 7858f9763..db09377d5 100644 --- a/examples/scaleway-machinedeployment.yaml +++ b/examples/scaleway-machinedeployment.yaml @@ -61,4 +61,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/vmware-cloud-director-machinedeployment.yaml b/examples/vmware-cloud-director-machinedeployment.yaml index ae7983ecd..ccf89ae24 100644 --- a/examples/vmware-cloud-director-machinedeployment.yaml +++ b/examples/vmware-cloud-director-machinedeployment.yaml @@ -89,4 +89,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml index f2e7df996..a914c9808 100644 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ b/examples/vsphere-datastore-cluster-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index 0387105cf..47376de65 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/examples/vultr-machinedeployment.yaml b/examples/vultr-machinedeployment.yaml index 6b6635a44..2a3a22a84 100644 --- a/examples/vultr-machinedeployment.yaml +++ b/examples/vultr-machinedeployment.yaml @@ -70,4 +70,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.29.4 + kubelet: 1.29.6 diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index f1eeaa32c..c43faff55 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -85,7 +85,7 @@ const ( ) const ( - defaultKubernetesVersion = "1.29.4" + defaultKubernetesVersion = "1.29.6" awsDefaultKubernetesVersion = "1.26.12" defaultContainerRuntime = "containerd" ) @@ -348,7 +348,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.26. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.27.13", "1.28.9", "1.29.4", "1.30.0"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.27.13", "1.28.11", "1.29.6", "1.30.2"))) runScenarios(context.Background(), t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -424,7 +424,7 @@ func TestAWSProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := Not(VersionSelector("1.27.13", "1.28.9", "1.29.4", "1.30.0")) + selector := Not(VersionSelector("1.27.13", "1.28.11", "1.29.6", "1.30.2")) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -478,7 +478,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.13", "1.28.9", "1.29.4", "1.30.0"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.13", "1.28.11", "1.29.6", "1.30.2"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -500,7 +500,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.13", "1.28.9", "1.29.4", "1.30.0"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.13", "1.28.11", "1.29.6", "1.30.2"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -596,7 +596,7 @@ func TestAzureProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.30.0"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.30.2"))) // act params := []string{ @@ -625,7 +625,7 @@ func TestAzureCustomImageReferenceProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.0"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.2"))) // act params := []string{ fmt.Sprintf("<< AZURE_TENANT_ID >>=%s", azureTenantID), @@ -841,7 +841,7 @@ func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(Not(OsSelector("amzn2", "centos")), Not(VersionSelector("1.30.0"))) + selector := And(Not(OsSelector("amzn2", "centos")), Not(VersionSelector("1.30.2"))) params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) @@ -853,7 +853,7 @@ func TestVsphereMultipleNICProvisioningE2E(t *testing.T) { t.Parallel() // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.0"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.2"))) params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereMultipleNICManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) @@ -882,7 +882,7 @@ func TestVsphereDatastoreClusterProvisioningE2E(t *testing.T) { t.Parallel() // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(OsSelector("ubuntu", "centos", "rhel", "flatcar"), Not(VersionSelector("1.30.0"))) + selector := And(OsSelector("ubuntu", "centos", "rhel", "flatcar"), Not(VersionSelector("1.30.2"))) params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereDSCManifest, fmt.Sprintf("vs-dsc-%s", *testRunIdentifier)) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 3389f8bf4..cfe5d1d40 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -35,9 +35,9 @@ var ( versions = []*semver.Version{ semver.MustParse("v1.27.13"), - semver.MustParse("v1.28.9"), - semver.MustParse("v1.29.4"), - semver.MustParse("v1.30.0"), + semver.MustParse("v1.28.11"), + semver.MustParse("v1.29.6"), + semver.MustParse("v1.30.2"), } operatingSystems = []providerconfigtypes.OperatingSystem{ diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml index 71485b5f2..3cf509b0a 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml @@ -55,4 +55,4 @@ spec: rhelOrganizationName: "" rhelActivationKey: "" versions: - kubelet: 1.29.4 + kubelet: 1.29.6 From 843aa90644203505c63a24d560e057ff9d9bb81d Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 8 Jul 2024 21:57:12 +0500 Subject: [PATCH 402/489] Update dependencies (#1825) Signed-off-by: Waleed Malik --- .golangci.yml | 9 +- .wwhrd.yml | 4 - go.mod | 164 ++--- go.sum | 606 +++++++----------- .../provider/scaleway/provider.go | 9 +- 5 files changed, 315 insertions(+), 477 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 01ec59870..6c14b4d17 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -51,20 +51,13 @@ linters-settings: issues: exclude: - - should have comment or be unexported - - should have comment \\(or a comment on this block\\) or be unexported - func Convert_MachinesV1alpha1Machine_To_ClusterV1alpha1Machine should be ConvertMachinesV1alpha1MachineToClusterV1alpha1Machine - func Convert_MachineDeployment_ProviderConfig_To_ProviderSpec should be ConvertMachineDeploymentProviderConfigToProviderSpec - func Convert_MachineSet_ProviderConfig_To_ProviderSpec should be ConvertMachineSetProviderConfigToProviderSpec - func Convert_Machine_ProviderConfig_To_ProviderSpec should be ConvertMachineProviderConfigToProviderSpec - - 'counter\.Set is deprecated: Use NewConstMetric' - - 'eviction\.go:221:4: the surrounding loop is unconditionally terminated' - - "cyclomatic complexity 31 of func `verifyMigrateUID` is high" - - "cyclomatic complexity 31 of func `main` is high" - - 'cyclomatic complexity 34 of func `\(\*provider\)\.getConfig` is high' - - 'cyclomatic complexity 31 of func `\(\*provider\)\.Validate` is high' - 'cyclomatic complexity 33 of func `\(\*provider\)\.Create` is high' - 'cyclomatic complexity 32 of func `\(\*Reconciler\)\.ensureInstanceExistsForMachine` is high' + - "SA1019: s.server.IPv6 is deprecated" exclude-dirs: - pkg/client - pkg/machines diff --git a/.wwhrd.yml b/.wwhrd.yml index 677ba2b2f..43eb38e7a 100644 --- a/.wwhrd.yml +++ b/.wwhrd.yml @@ -27,10 +27,6 @@ allowlist: exceptions: - github.com/hashicorp/golang-lru # MPL-2.0 - github.com/hashicorp/golang-lru/simplelru # MPL-2.0 - - github.com/embik/nutanix-client-go/pkg/client # MPL-2.0 - - github.com/embik/nutanix-client-go/pkg/client/v3 # MPL-2.0 - - github.com/embik/nutanix-client-go/internal/utils # MPL-2.0 - - github.com/ajeddeloh/go-json # Since it's a fork, https://github.com/golang/go/blob/master/LICENSE - github.com/hashicorp/go-version # MPL-2.0 - github.com/hashicorp/go-cleanhttp # MPL-2.0 - github.com/hashicorp/go-retryablehttp # MPL-2.0 diff --git a/go.mod b/go.mod index 4cef2d7c9..ea112db63 100644 --- a/go.mod +++ b/go.mod @@ -5,72 +5,73 @@ go 1.22.0 toolchain go1.22.4 require ( - cloud.google.com/go/logging v1.9.0 - cloud.google.com/go/monitoring v1.18.1 + cloud.google.com/go/logging v1.10.0 + cloud.google.com/go/monitoring v1.20.1 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible - github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 + github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/Masterminds/semver/v3 v3.2.1 - github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240311170802-57efabf2d590 - github.com/aliyun/alibaba-cloud-sdk-go v1.62.695 - github.com/aws/aws-sdk-go-v2 v1.25.3 - github.com/aws/aws-sdk-go-v2/config v1.27.7 - github.com/aws/aws-sdk-go-v2/credentials v1.17.7 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.151.0 - github.com/aws/aws-sdk-go-v2/service/sts v1.28.4 - github.com/aws/smithy-go v1.20.1 + github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240703112053-c8500c47bced + github.com/aliyun/alibaba-cloud-sdk-go v1.62.784 + github.com/aws/aws-sdk-go-v2 v1.30.1 + github.com/aws/aws-sdk-go-v2/config v1.27.24 + github.com/aws/aws-sdk-go-v2/credentials v1.17.24 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.168.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 + github.com/aws/smithy-go v1.20.3 github.com/davecgh/go-spew v1.1.1 - github.com/digitalocean/godo v1.110.0 - github.com/go-logr/logr v1.4.1 + github.com/digitalocean/godo v1.118.0 + github.com/go-logr/logr v1.4.2 github.com/go-logr/zapr v1.3.0 github.com/go-test/deep v1.1.0 github.com/google/uuid v1.6.0 - github.com/gophercloud/gophercloud v1.11.0 + github.com/gophercloud/gophercloud v1.12.0 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb - github.com/hetznercloud/hcloud-go/v2 v2.10.1 - github.com/linode/linodego v1.30.0 - github.com/nutanix-cloud-native/prism-go-client v0.3.4 + github.com/hetznercloud/hcloud-go/v2 v2.10.2 + github.com/linode/linodego v1.36.1 + github.com/nutanix-cloud-native/prism-go-client v0.4.0 github.com/packethost/packngo v0.31.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 github.com/prometheus/client_golang v1.19.1 - github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25 + github.com/scaleway/scaleway-sdk-go v1.0.0-beta.28 github.com/spf13/pflag v1.0.5 github.com/tinkerbell/tink v0.8.0 - github.com/vmware/go-vcloud-director/v2 v2.22.0 - github.com/vmware/govmomi v0.36.1 - github.com/vultr/govultr/v3 v3.6.4 - go.anx.io/go-anxcloud v0.6.4 + github.com/vmware/go-vcloud-director/v2 v2.25.0 + github.com/vmware/govmomi v0.38.0 + github.com/vultr/govultr/v3 v3.9.0 + go.anx.io/go-anxcloud v0.7.2 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.24.0 - golang.org/x/oauth2 v0.18.0 + golang.org/x/crypto v0.25.0 + golang.org/x/oauth2 v0.21.0 gomodules.xyz/jsonpatch/v2 v2.4.0 - google.golang.org/api v0.170.0 - google.golang.org/grpc v1.62.1 + google.golang.org/api v0.187.0 + google.golang.org/grpc v1.65.0 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.30.0 - k8s.io/apiextensions-apiserver v0.30.0 - k8s.io/apimachinery v0.30.0 - k8s.io/client-go v0.30.0 - k8s.io/cloud-provider v0.30.0 + k8s.io/api v0.30.2 + k8s.io/apiextensions-apiserver v0.30.2 + k8s.io/apimachinery v0.30.2 + k8s.io/client-go v0.30.2 + k8s.io/cloud-provider v0.30.2 k8s.io/klog v1.0.0 - k8s.io/utils v0.0.0-20240310230437-4693a0247e57 - kubevirt.io/api v1.2.0 - kubevirt.io/containerized-data-importer-api v1.58.1 - sigs.k8s.io/controller-runtime v0.18.2 + k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 + kubevirt.io/api v1.2.2 + kubevirt.io/containerized-data-importer-api v1.59.0 + sigs.k8s.io/controller-runtime v0.18.4 sigs.k8s.io/yaml v1.4.0 ) require ( - cloud.google.com/go v0.112.1 // indirect - cloud.google.com/go/compute v1.24.0 // indirect - cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/longrunning v0.5.5 // indirect + cloud.google.com/go v0.115.0 // indirect + cloud.google.com/go/auth v0.6.1 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect + cloud.google.com/go/compute/metadata v0.4.0 // indirect + cloud.google.com/go/longrunning v0.5.9 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.29 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.23 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.24 // indirect github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect @@ -79,30 +80,30 @@ require ( github.com/PaesslerAG/gval v1.2.2 // indirect github.com/PaesslerAG/jsonpath v0.1.1 // indirect github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/distribution/reference v0.5.0 // indirect github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/emicklei/go-restful/v3 v3.11.1 // indirect - github.com/evanphx/json-patch v5.7.0+incompatible // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.20.2 // indirect - github.com/go-openapi/jsonreference v0.20.4 // indirect - github.com/go-openapi/swag v0.22.7 // indirect - github.com/go-resty/resty/v2 v2.11.0 // indirect - github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-resty/resty/v2 v2.13.1 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect @@ -111,13 +112,13 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect + github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.5 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect - github.com/hashicorp/go-version v1.6.0 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -132,46 +133,45 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/ginkgo/v2 v2.17.1 // indirect - github.com/onsi/gomega v1.32.0 // indirect + github.com/onsi/ginkgo/v2 v2.19.0 // indirect + github.com/onsi/gomega v1.33.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f // indirect + github.com/openshift/api v0.0.0-20240708071937-c9a91940bf0f // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046 // indirect github.com/peterhellberg/link v1.2.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.48.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect - github.com/shopspring/decimal v1.3.1 // indirect + github.com/shopspring/decimal v1.4.0 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect - go.opentelemetry.io/otel v1.24.0 // indirect - go.opentelemetry.io/otel/metric v1.24.0 // indirect - go.opentelemetry.io/otel/trace v1.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect + go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect - golang.org/x/net v0.26.0 // indirect + golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect + golang.org/x/net v0.27.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/term v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/term v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 // indirect - google.golang.org/protobuf v1.33.0 // indirect + golang.org/x/tools v0.23.0 // indirect + google.golang.org/genproto v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/klog/v2 v2.120.1 // indirect - k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240703190633-0aa61b46e8c2 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index 702f98a8c..2f7af65ad 100644 --- a/go.sum +++ b/go.sum @@ -13,33 +13,30 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= -cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= +cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= +cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= +cloud.google.com/go/auth v0.6.1 h1:T0Zw1XM5c1GlpN2HYr2s+m3vr1p2wy+8VN+Z1FKxW38= +cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4= +cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= +cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= -cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute/metadata v0.4.0 h1:vHzJCWaM4g8XIcm8kopr3XmDA4Gy/lblD3EhhSux05c= +cloud.google.com/go/compute/metadata v0.4.0/go.mod h1:SIQh1Kkb4ZJ8zJ874fqVkslA29PRXuleyj6vOzlbK7M= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc= -cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI= -cloud.google.com/go/logging v1.9.0 h1:iEIOXFO9EmSiTjDmfpbRjOxECO7R8C7b8IXUGOj7xZw= -cloud.google.com/go/logging v1.9.0/go.mod h1:1Io0vnZv4onoUnsVUQY3HZ3Igb1nBchky0A0y7BBBhE= -cloud.google.com/go/longrunning v0.5.5 h1:GOE6pZFdSrTb4KAiKnXsJBtlE6mEyaW44oKyMILWnOg= -cloud.google.com/go/longrunning v0.5.5/go.mod h1:WV2LAxD8/rg5Z1cNW6FJ/ZpX4E4VnDnoTk0yawPBB7s= -cloud.google.com/go/monitoring v1.18.1 h1:0yvFXK+xQd95VKo6thndjwnJMno7c7Xw1CwMByg0B+8= -cloud.google.com/go/monitoring v1.18.1/go.mod h1:52hTzJ5XOUMRm7jYi7928aEdVxBEmGwA0EjNJXIBvt8= +cloud.google.com/go/iam v1.1.9 h1:oSkYLVtVme29uGYrOcKcvJRht7cHJpYD09GM9JaR0TE= +cloud.google.com/go/iam v1.1.9/go.mod h1:Nt1eDWNYH9nGQg3d/mY7U1hvfGmsaG9o/kLGoLoLXjQ= +cloud.google.com/go/logging v1.10.0 h1:f+ZXMqyrSJ5vZ5pE/zr0xC8y/M9BLNzQeLBwfeZ+wY4= +cloud.google.com/go/logging v1.10.0/go.mod h1:EHOwcxlltJrYGqMGfghSet736KR3hX1MAj614mrMk9I= +cloud.google.com/go/longrunning v0.5.9 h1:haH9pAuXdPAMqHvzX0zlWQigXT7B0+CL4/2nXXdBo5k= +cloud.google.com/go/longrunning v0.5.9/go.mod h1:HD+0l9/OOW0za6UWdKJtXoFAX/BGg/3Wj8p10NeWF7c= +cloud.google.com/go/monitoring v1.20.1 h1:XmM6uk4+mI2ZhWdI2n/2GNhJdpeQN+1VdG2UWEDhX48= +cloud.google.com/go/monitoring v1.20.1/go.mod h1:FYSe/brgfuaXiEzOQFhTjsEsJv+WePyK71X7Y8qo6uQ= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -56,18 +53,15 @@ github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= +github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= github.com/Azure/go-autorest/autorest v0.11.29 h1:I4+HL/JDvErx2LjyzaVxllw2lRDB5/BT2Bm4g20iqYw= github.com/Azure/go-autorest/autorest v0.11.29/go.mod h1:ZtEzC4Jy2JDrZLxvWs8LrBWEBycl1hbT1eknI8MtfAs= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/adal v0.9.22/go.mod h1:XuAbAEUv2Tta//+voMI038TrJBqjKam0me7qR+L8Cmk= -github.com/Azure/go-autorest/autorest/adal v0.9.23 h1:Yepx8CvFxwNKpH6ja7RZ+sKX+DWYNldbLiALMC3BTz8= -github.com/Azure/go-autorest/autorest/adal v0.9.23/go.mod h1:5pcMqFkdPhviJdlEy3kC/v1ZLnQl0MH6XA5YCcMhy4c= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.12 h1:wkAZRgT/pn8HhFyzfe9UnqOjJYqlembgCTi72Bm/xKk= -github.com/Azure/go-autorest/autorest/azure/auth v0.5.12/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= +github.com/Azure/go-autorest/autorest/adal v0.9.24 h1:BHZfgGsGwdkHDyZdtQRQk1WeUdW0m2WPAwuHZwUi5i4= +github.com/Azure/go-autorest/autorest/adal v0.9.24/go.mod h1:7T1+g0PYFmACYW5LlG2fcoPiPlFHjClyRGL7dRlP5c8= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 h1:Ov8avRZi2vmrE2JcXw+tu5K/yB41r7xK9GZDiBF7NdM= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.13/go.mod h1:5BAVfWLWXihP47vYrPuBKKf4cS0bXI+KM9Qx6ETDJYo= github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc= github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= @@ -90,8 +84,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240311170802-57efabf2d590 h1:wvNejQUL/d0Z2n4DZfAtAQv+/fUFrFSkLj3X49ioDiM= -github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240311170802-57efabf2d590/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= +github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240703112053-c8500c47bced h1:yLjIFnPG7e6IVIDxFWp+dUkxlOgOkx8ttesy9dGAEms= +github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240703112053-c8500c47bced/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= github.com/PaesslerAG/gval v1.2.2 h1:Y7iBzhgE09IGTt5QgGQ2IdaYYYOU134YGHBThD+wm9E= github.com/PaesslerAG/gval v1.2.2/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac= @@ -100,92 +94,77 @@ github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEs github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= -github.com/agnivade/levenshtein v1.1.0 h1:n6qGwyHG61v3ABce1rPVZklEYRT8NFpCMrpZdBUbYGM= -github.com/agnivade/levenshtein v1.1.0/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= +github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= +github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.695 h1:Lk9qjMhhkzZaD4eyx23v0E2+4nAIfwreJ/ecKdaTU6E= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.695/go.mod h1:CJJYa1ZMxjlN/NbXEwmejEnBkhi0DV+Yb3B2lxf+74o= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.784 h1:wtou656eZpKB1QFJ26ChcmpIZvAdtVZvxuhD7BJpZtA= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.784/go.mod h1:SOSDHfe1kX91v3W5QiBsWSLqeLxImobbMX1mxrFHsVQ= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= -github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= -github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.42.23/go.mod h1:gyRszuZ/icHmHAVE4gc/r+cfCmhA1AD+vqfWbgI+eHs= -github.com/aws/aws-sdk-go-v2 v1.25.3 h1:xYiLpZTQs1mzvz5PaI6uR0Wh57ippuEthxS4iK5v0n0= -github.com/aws/aws-sdk-go-v2 v1.25.3/go.mod h1:35hUlJVYd+M++iLI3ALmVwMOyRYMmRqUXpTtRGW+K9I= -github.com/aws/aws-sdk-go-v2/config v1.27.7 h1:JSfb5nOQF01iOgxFI5OIKWwDiEXWTyTgg1Mm1mHi0A4= -github.com/aws/aws-sdk-go-v2/config v1.27.7/go.mod h1:PH0/cNpoMO+B04qET699o5W92Ca79fVtbUnvMIZro4I= -github.com/aws/aws-sdk-go-v2/credentials v1.17.7 h1:WJd+ubWKoBeRh7A5iNMnxEOs982SyVKOJD+K8HIezu4= -github.com/aws/aws-sdk-go-v2/credentials v1.17.7/go.mod h1:UQi7LMR0Vhvs+44w5ec8Q+VS+cd10cjwgHwiVkE0YGU= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3 h1:p+y7FvkK2dxS+FEwRIDHDe//ZX+jDhP8HHE50ppj4iI= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.3/go.mod h1:/fYB+FZbDlwlAiynK9KDXlzZl3ANI9JkD0Uhz5FjNT4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3 h1:ifbIbHZyGl1alsAhPIYsHOg5MuApgqOvVeI8wIugXfs= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.3/go.mod h1:oQZXg3c6SNeY6OZrDY+xHcF4VGIEoNotX2B4PrDeoJI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3 h1:Qvodo9gHG9F3E8SfYOspPeBt0bjSbsevK8WhRAUHcoY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.3/go.mod h1:vCKrdLXtybdf/uQd/YfVR2r5pcbNuEYKzMQpcxmeSJw= +github.com/aws/aws-sdk-go-v2 v1.30.1 h1:4y/5Dvfrhd1MxRDD77SrfsDaj8kUkkljU7XE83NPV+o= +github.com/aws/aws-sdk-go-v2 v1.30.1/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= +github.com/aws/aws-sdk-go-v2/config v1.27.24 h1:NM9XicZ5o1CBU/MZaHwFtimRpWx9ohAUAqkG6AqSqPo= +github.com/aws/aws-sdk-go-v2/config v1.27.24/go.mod h1:aXzi6QJTuQRVVusAO8/NxpdTeTyr/wRcybdDtfUwJSs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.24 h1:YclAsrnb1/GTQNt2nzv+756Iw4mF8AOzcDfweWwwm/M= +github.com/aws/aws-sdk-go-v2/credentials v1.17.24/go.mod h1:Hld7tmnAkoBQdTMNYZGzztzKRdA4fCdn9L83LOoigac= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 h1:Aznqksmd6Rfv2HQN9cpqIV/lQRMaIpJkLLaJ1ZI76no= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9/go.mod h1:WQr3MY7AxGNxaqAtsDWn+fBxmd4XvLkzeqQ8P1VM0/w= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 h1:5SAoZ4jYpGH4721ZNoS1znQrhOfZinOhc4XuTXx/nVc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13/go.mod h1:+rdA6ZLpaSeM7tSg/B0IEDinCIBJGmW8rKDFkYpP04g= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 h1:WIijqeaAO7TYFLbhsZmi2rgLEAtWOC1LhxCAVTJlSKw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13/go.mod h1:i+kbfa76PQbWw/ULoWnp51EYVWH4ENln76fLQE3lXT8= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.151.0 h1:gH571JR1hMfIER4zK457aNjCfi1FCuVwriKx0bAyw/I= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.151.0/go.mod h1:KNJMjsbzK97hci9ev2Vl/27GgUt3ZciRP4RGujAPF2I= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5 h1:K/NXvIftOlX+oGgWGIa3jDyYLDNsdVhsjHmsBH2GLAQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.5/go.mod h1:cl9HGLV66EnCmMNzq4sYOti+/xo8w34CsgzVtm2GgsY= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.2 h1:XOPfar83RIRPEzfihnp+U6udOveKZJvPQ76SKWrLRHc= -github.com/aws/aws-sdk-go-v2/service/sso v1.20.2/go.mod h1:Vv9Xyk1KMHXrR3vNQe8W5LMFdTjSeWk0gBZBzvf3Qa0= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2 h1:pi0Skl6mNl2w8qWZXcdOyg197Zsf4G97U7Sso9JXGZE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.2/go.mod h1:JYzLoEVeLXk+L4tn1+rrkfhkxl6mLDEVaDSvGq9og90= -github.com/aws/aws-sdk-go-v2/service/sts v1.28.4 h1:Ppup1nVNAOWbBOrcoOxaxPeEnSFB2RnnQdguhXpmeQk= -github.com/aws/aws-sdk-go-v2/service/sts v1.28.4/go.mod h1:+K1rNPVyGxkRuv9NNiaZ4YhBFuyw2MMA9SlIJ1Zlpz8= -github.com/aws/smithy-go v1.20.1 h1:4SZlSlMr36UEqC7XOyRVb27XMeZubNcBNN+9IgEPIQw= -github.com/aws/smithy-go v1.20.1/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.168.0 h1:xOPq0agGC1WMZvFpSZCKEjDVAQnLPZJZGvjuPVF2t9M= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.168.0/go.mod h1:CtLD6CPq9z9dyMxV+H6/M5d9+/ea3dO80um029GXqV0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 h1:I9zMeF107l0rJrpnHpjEiiTSCKYAIw8mALiXcPsGBiA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15/go.mod h1:9xWJ3Q/S6Ojusz1UIkfycgD1mGirJfLLKqq3LPT7WN8= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 h1:p1GahKIjyMDZtiKoIn0/jAj/TkMzfzndDv5+zi2Mhgc= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.1/go.mod h1:/vWdhoIoYA5hYoPZ6fm7Sv4d8701PiG5VKe8/pPJL60= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 h1:ORnrOK0C4WmYV/uYt3koHEWBLYsRDwk2Np+eEoyV4Z0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2/go.mod h1:xyFHA4zGxgYkdD73VeezHt3vSKEG9EmFnGwoKlP00u4= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 h1:+woJ607dllHJQtsnJLi52ycuqHMwlW+Wqm2Ppsfp4nQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.1/go.mod h1:jiNR3JqT15Dm+QWq2SRgh0x0bCNSRP2L25+CqPNpJlQ= +github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= +github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bnkamalesh/webgo/v4 v4.1.11/go.mod h1:taIAonQTzao8G5rnB22WgKmQuIOWHpQ0n/YLAidBXlM= -github.com/bnkamalesh/webgo/v6 v6.2.2/go.mod h1:2Y+dEdTp1xC/ra+3PAVZV6hh4sCI+iPK7mcHt+t9bfM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creasty/defaults v1.5.2 h1:/VfB6uxpyp6h0fr7SPp7n8WJBoV8jfxQXPCnkVSjyls= github.com/creasty/defaults v1.5.2/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= -github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= -github.com/digitalocean/godo v1.110.0 h1:EY+rewWCYrUNOPbk9wI2Ytf0TBSRTJcZ6BINCb5dfmQ= -github.com/digitalocean/godo v1.110.0/go.mod h1:R6EmmWI8CT1+fCtjWY9UCB+L5uufuZH13wk3YhxycCs= +github.com/digitalocean/godo v1.118.0 h1:lkzGFQmACrVCp7UqH1sAi4JK/PWwlc5aaxubgorKmC4= +github.com/digitalocean/godo v1.118.0/go.mod h1:Vk0vpCot2HOAJwc5WE8wljZGtJ3ZtWIc8MQ8rF38sdo= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= @@ -199,19 +178,18 @@ github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkg github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.11.1 h1:S+9bSbua1z3FgCnV0KKOSSZ3mDthb5NyEPL5gEpCvyk= -github.com/emicklei/go-restful/v3 v3.11.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= -github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= +github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= @@ -219,19 +197,14 @@ github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4Nij github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs= github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg= -github.com/go-chi/cors v1.2.0/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8= github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -248,53 +221,46 @@ github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= -github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= -github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.7 h1:JWrc1uc/P9cSomxfnsFSVWoE1FW6bNbrVPmpQYpCcR8= -github.com/go-openapi/swag v0.22.7/go.mod h1:Gl91UqO+btAM0plGGxHqJcQZ1ZTy6jbmridBTsDy8A0= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= -github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-playground/validator/v10 v10.10.1 h1:uA0+amWMiglNZKZ9FJRKUAe9U3RX91eVn1JYXMWt7ig= github.com/go-playground/validator/v10 v10.10.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= -github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= -github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= +github.com/go-resty/resty/v2 v2.13.1 h1:x+LHXBI2nMB1vqndymf26quycC4aggYJ7DECYbiz03g= +github.com/go-resty/resty/v2 v2.13.1/go.mod h1:GznXlLxkq6Nh4sU59rPmUw3VtgpO3aS96ORAI6Q7d+0= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/goccy/go-json v0.9.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= @@ -313,7 +279,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -329,15 +294,13 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -352,7 +315,6 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -364,7 +326,6 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -372,34 +333,30 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUhuHF+DA= -github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= +github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= +github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gophercloud/gophercloud v1.11.0 h1:ls0O747DIq1D8SUHc7r2vI8BFbMLeLFuENaAIfEx7OM= -github.com/gophercloud/gophercloud v1.11.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= +github.com/gophercloud/gophercloud v1.12.0 h1:Jrz16vPAL93l80q16fp8NplrTCp93y7rZh2P3Q4Yq7g= +github.com/gophercloud/gophercloud v1.12.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -410,24 +367,24 @@ github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB1 github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= -github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= -github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb h1:tsEKRC3PU9rMw18w/uAptoijhgG4EvlA5kfJPtwrMDk= github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= -github.com/hetznercloud/hcloud-go/v2 v2.10.1 h1:MLfIJJvl7qo79Q+52YmrJ4BOTL5oICJWHjzb0sCtgQI= -github.com/hetznercloud/hcloud-go/v2 v2.10.1/go.mod h1:xQ+8KhIS62W0D78Dpi57jsufWh844gUw1az5OUvaeq8= +github.com/hetznercloud/hcloud-go/v2 v2.10.2 h1:9gyTUPhfNbfbS40Spgij5mV5k37bOZgt8iHKCbfGs5I= +github.com/hetznercloud/hcloud-go/v2 v2.10.2/go.mod h1:xQ+8KhIS62W0D78Dpi57jsufWh844gUw1az5OUvaeq8= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -435,9 +392,7 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -447,16 +402,15 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/k0kubun/pp/v3 v3.1.0 h1:ifxtqJkRZhw3h554/z/8zm6AAbyO4LLKDlA5eV+9O8Q= +github.com/k0kubun/pp/v3 v3.1.0/go.mod h1:vIrP5CF0n78pKHm2Ku6GVerpZBJvscg48WepUYEk2gw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= -github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/keploy/go-sdk v0.4.3 h1:dCsmfANlZH94It+JKWx8/JEEC6dn8W7KIRRKRZwCPZQ= -github.com/keploy/go-sdk v0.4.3/go.mod h1:tn62gQ8a/AD7mY51DvQfhudiBPTlD+w3XtXemDcbON4= -github.com/kevinmbeaulieu/eq-go v1.0.0/go.mod h1:G3S8ajA56gKBZm4UB9AOyoOS37JO3roToPzKNM8dtdM= +github.com/keploy/go-sdk v0.7.2 h1:mvvjDRciMSFTgOF/KIGz38ElJZKkM1WlniaHseaPhpo= +github.com/keploy/go-sdk v0.7.2/go.mod h1:TtJIM+Gkq76FzfkD8W9u1F8NDkC9sVY8nYvmbRo1nhg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/kolo/xmlrpc v0.0.0-20190717152603-07c4ee3fd181/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00= github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= @@ -465,43 +419,25 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/labstack/echo/v4 v4.6.1/go.mod h1:RnjgMWNDB9g/HucVWhQYNQP9PvbYf6adqftqryo7s9k= -github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y= -github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ= -github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++0Gf8MBnAvE= -github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= -github.com/lestrrat-go/jwx v1.2.20/go.mod h1:tLE1XszaFgd7zaS5wHe4NxA+XVhu7xgdRvDpNyi3kNM= -github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/linode/linodego v1.30.0 h1:6HJli+LX7NGu+Sne2G+ux790EkVOWOV/SR4mK3jcs6k= -github.com/linode/linodego v1.30.0/go.mod h1:/46h/XpmWi//oSA92GX2p3FIxb8HbX7grslPPQalR2o= -github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= +github.com/linode/linodego v1.36.1 h1:lxYBKWJCk6m9p/OdHQlgteyj4S0eglq3glmK16QxUHY= +github.com/linode/linodego v1.36.1/go.mod h1:KyV4OO/9/tAxaLSjyjFyOQBcS9bYUdei1hwk3nl0UjI= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matryer/moq v0.2.3/go.mod h1:9RtPYjTnH1bSBIkpvtHkFN7nbWAnO7oRpdJkEIn6UtE= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/matryer/moq v0.2.5 h1:BGQISyhl7Gc9W/gMYmAJONh9mT6AYeyeTjNupNPknMs= +github.com/matryer/moq v0.2.5/go.mod h1:9RtPYjTnH1bSBIkpvtHkFN7nbWAnO7oRpdJkEIn6UtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= @@ -511,7 +447,8 @@ github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HK github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= @@ -523,7 +460,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -531,8 +467,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nutanix-cloud-native/prism-go-client v0.3.4 h1:bHY3VPrHHYnbRtkpGaKK+2ZmvUjNVRC55CYZbXIfnOk= -github.com/nutanix-cloud-native/prism-go-client v0.3.4/go.mod h1:tTIH02E6o6AWSShr98QChoxuZl+jBhkXFixom9+fd1Y= +github.com/nutanix-cloud-native/prism-go-client v0.4.0 h1:P9mLW6eyKMUXVQBzuVL5k7WjV1YwVu8XNpu2XAsRgGo= +github.com/nutanix-cloud-native/prism-go-client v0.4.0/go.mod h1:bHxgYigeclzjuaMEdjpsIEO4k7sjzP4Gr7ooF6nWXcI= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -540,24 +476,20 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= -github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= -github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f h1:3BMVfQpz1xe8MmJprp1+NL8hrpl9I04JVP9EczdCOqE= -github.com/openshift/api v0.0.0-20240104110125-c7a2d3b41e1f/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4= +github.com/openshift/api v0.0.0-20240708071937-c9a91940bf0f h1:NmJAlN2fPnL86aq5BbEQJ62v/D16LzIaaQ0Qn72s87E= +github.com/openshift/api v0.0.0-20240708071937-c9a91940bf0f/go.mod h1:OOh6Qopf21pSzqNVCB5gomomBXb8o5sGKZxG2KNpaXM= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -571,7 +503,6 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterhellberg/link v1.2.0 h1:UA5pg3Gp/E0F2WdX7GERiNrPQrM1K6CVJUUWfHa4t6c= github.com/peterhellberg/link v1.2.0/go.mod h1:gYfAh+oJgQu2SrZHg5hROVRQe1ICoK0/HHJTcE0edxc= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -591,43 +522,40 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.31.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= -github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rollbar/rollbar-go v1.4.2/go.mod h1:kLQ9gP3WCRGrvJmF0ueO3wK9xWocej8GRX98D8sa39w= github.com/rollbar/rollbar-go/errors v0.0.0-20210929193720-32947096267e/go.mod h1:Ie0xEc1Cyj+T4XMO8s0Vf7pMfvSAAy1sb4AYc8aJsao= -github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25 h1:/8rfZAdFfafRXOgz+ZpMZZWZ5pYggCY9t7e/BvjaBHM= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.25/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.28 h1:2vT+ryIQGfF21HN/W5yn/CBPpsTJULuuepWfUq/geV4= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.28/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -647,13 +575,17 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= +github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tinkerbell/lint-install v0.0.0-20211012174934-5ee5ab01db76/go.mod h1:0h2KsALaQLNkoVeV+G+HjBWWCnp0COFYhJdRd5WCQPM= github.com/tinkerbell/tink v0.8.0 h1:qgl/rglpO5Rvq6UKZd29O6X9mDgZZYgf841+Y0IYWak= github.com/tinkerbell/tink v0.8.0/go.mod h1:bfAkSH7J/QQYIyqZRR6IQp8w78aac6l8Z2Lws5uXz6A= @@ -661,26 +593,26 @@ github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaO github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.35.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/vektah/gqlparser/v2 v2.2.0 h1:bAc3slekAAJW6sZTi07aGq0OrfaCjj4jxARAaC7g2EM= github.com/vektah/gqlparser/v2 v2.2.0/go.mod h1:i3mQIGIrbK2PD1RrCeMTlVbkF2FJ6WkU1KJlJlC+3F4= -github.com/vmware/go-vcloud-director/v2 v2.22.0 h1:i1yFCoQZl/mTKViWLpT8mC9tlOAbupip703K0q1gQT0= -github.com/vmware/go-vcloud-director/v2 v2.22.0/go.mod h1:QPxGFgrUcSyzy9IlpwDE4UNT3tsOy2047tJOPEJ4nlw= -github.com/vmware/govmomi v0.36.1 h1:+E/nlfteQ8JvC0xhuKAfpnMsuIeGeGj7rJwqENUcWm8= -github.com/vmware/govmomi v0.36.1/go.mod h1:mtGWtM+YhTADHlCgJBiskSRPOZRsN9MSjPzaZLte/oQ= -github.com/vultr/govultr/v3 v3.6.4 h1:unvY9eXlBw667ECQZDbBDOIaWB8wkk6Bx+yB0IMKXJ4= -github.com/vultr/govultr/v3 v3.6.4/go.mod h1:rt9v2x114jZmmLAE/h5N5jnxTmsK9ewwS2oQZ0UBQzM= +github.com/vmware/go-vcloud-director/v2 v2.25.0 h1:RcJ5FQRku3FvQktTi8YOZsRfvhfLm315Cme50M9x9MQ= +github.com/vmware/go-vcloud-director/v2 v2.25.0/go.mod h1:7Of1qJja+LLNKVegjZG7uuhhy6xgGg3q7Fkw2CEP+Tw= +github.com/vmware/govmomi v0.38.0 h1:UvQpLAOjDpO0JUxoPCXnEzOlEa/9kejO6K58qOFr6cM= +github.com/vmware/govmomi v0.38.0/go.mod h1:mtGWtM+YhTADHlCgJBiskSRPOZRsN9MSjPzaZLte/oQ= +github.com/vultr/govultr/v3 v3.9.0 h1:63V/22mpfquRA5DenJ9EF0VozHg0k+X4dhUWcDXHPyc= +github.com/vultr/govultr/v3 v3.9.0/go.mod h1:Rd8ebpXm7jxH3MDmhnEs+zrlYW212ouhx+HeUMfHm2o= +github.com/wI2L/jsondiff v0.2.0 h1:dE00WemBa1uCjrzQUUTE/17I6m5qAaN0EMFOg2Ynr/k= +github.com/wI2L/jsondiff v0.2.0/go.mod h1:axTcwtBkY4TsKuV+RgoMhHyHKKFRI6nnjRLi8LLYQnA= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/scram v1.1.0 h1:d70R37I0HrDLsafRrMBXyrD4lmQbCHE873t00Vr0gm0= +github.com/xdg-go/scram v1.1.0/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= +github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -689,45 +621,41 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.anx.io/go-anxcloud v0.6.4 h1:SaFqYHFZC96PNt0cp7bX+4khAWg1u1hUdSt11R++fn8= -go.anx.io/go-anxcloud v0.6.4/go.mod h1:aattNBzzaDFtPRU/eTsNK1lDdTFa8QUXal+w1SQPCF0= -go.keploy.io/server v0.1.8 h1:b50vAt1+WKMscYVP5Bm8gx/iSaR7mpHox8VpaxjrQ88= -go.keploy.io/server v0.1.8/go.mod h1:ZqhwTZOBb+dzx5t30Wt6eUGI6kO5QizvPg6coNPtbow= -go.mongodb.org/mongo-driver v1.8.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= -go.mongodb.org/mongo-driver v1.8.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= +go.anx.io/go-anxcloud v0.7.2 h1:Y7Rs9jI9G+QDWz7NHao+Prff/JNeG5ST4bNpO5d2qcM= +go.anx.io/go-anxcloud v0.7.2/go.mod h1:AfsjodrtGIEnzzVQ6cHTZxkoNoi3UmeNmHaF0oG9RjE= +go.keploy.io/server v0.7.12 h1:DKDSO6T9Q4d4A8MKL+sk7U26KRcvZ+ZG0mbFhYIJJyk= +go.keploy.io/server v0.7.12/go.mod h1:ch4rD1NCgtxozDHD9yVk+sLHWz5HgefOqrgEdEIgfBQ= +go.mongodb.org/mongo-driver v1.8.3 h1:TDKlTkGDKm9kkJVUOAXDK5/fkqKHJVwYQSpoRfB43R4= +go.mongodb.org/mongo-driver v1.8.3/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= -go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= +go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= +go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= +go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= +go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= +go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -736,9 +664,7 @@ go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95a go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -748,20 +674,15 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -775,8 +696,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= -golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w= +golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -790,7 +711,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -800,12 +720,12 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -838,44 +758,32 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -896,7 +804,6 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -905,17 +812,14 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -931,77 +835,62 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211015200801-69063c4bb744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1010,14 +899,12 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1044,25 +931,17 @@ golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200815165600-90abf76919f3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1089,22 +968,14 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.170.0 h1:zMaruDePM88zxZBG+NG8+reALO2rfLhe/JShitLyT48= -google.golang.org/api v0.170.0/go.mod h1:/xql9M2btF85xac/VAm4PsLMTLVGUOpq4BE9R8jyNy8= +google.golang.org/api v0.187.0 h1:Mxs7VATVC2v7CY+7Xwm4ndkX71hpElcvx0D1Ji/p1eo= +google.golang.org/api v0.187.0/go.mod h1:KIHlTc4x7N7gKKuVsdmfBXN13yEEWXWFURWY6SBp2gk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= -google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1137,24 +1008,14 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= -google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= -google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ= -google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 h1:9IZDv+/GcI6u+a4jRFRLxQs0RUCfavGfoOgEW6jpkI0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= +google.golang.org/genproto v0.0.0-20240701130421-f6361c86f094 h1:6whtk83KtD3FkGrVb2hFXuQ+ZMbCNdakARIn/aHMmG8= +google.golang.org/genproto v0.0.0-20240701130421-f6361c86f094/go.mod h1:Zs4wYw8z1zr6RNF4cwYb31mvN/EGaKAdQjNCF3DW6K4= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= +google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1167,18 +1028,13 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= -google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/grpc/examples v0.0.0-20210728214646-ad0a2a847cdf/go.mod h1:bF8wuZSAZTcbF7ZPKrDI/qY52toTP/yxLpRRY4Eu9Js= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -1193,9 +1049,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1209,7 +1064,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -1236,21 +1090,17 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= -k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= -k8s.io/api v0.30.0 h1:siWhRq7cNjy2iHssOB9SCGNCl2spiF1dO3dABqZ8niA= -k8s.io/api v0.30.0/go.mod h1:OPlaYhoHs8EQ1ql0R/TsUgaRPhpKNxIMrKQfWUp8QSE= -k8s.io/apiextensions-apiserver v0.30.0 h1:jcZFKMqnICJfRxTgnC4E+Hpcq8UEhT8B2lhBcQ+6uAs= -k8s.io/apiextensions-apiserver v0.30.0/go.mod h1:N9ogQFGcrbWqAY9p2mUAL5mGxsLqwgtUce127VtRX5Y= +k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= +k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= +k8s.io/apiextensions-apiserver v0.30.2 h1:l7Eue2t6QiLHErfn2vwK4KgF4NeDgjQkCXtEbOocKIE= +k8s.io/apiextensions-apiserver v0.30.2/go.mod h1:lsJFLYyK40iguuinsb3nt+Sj6CmodSI4ACDLep1rgjw= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apimachinery v0.30.0 h1:qxVPsyDM5XS96NIh9Oj6LavoVFYff/Pon9cZeDIkHHA= -k8s.io/apimachinery v0.30.0/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= -k8s.io/client-go v0.30.0 h1:sB1AGGlhY/o7KCyCEQ0bPWzYDL0pwOZO4vAtTSh/gJQ= -k8s.io/client-go v0.30.0/go.mod h1:g7li5O5256qe6TYdAMyX/otJqMhIiGgTapdLchhmOaY= -k8s.io/cloud-provider v0.30.0 h1:hz1MXkFjsyO167sRZVchXEi2YYMQ6kolBi79nuICjzw= -k8s.io/cloud-provider v0.30.0/go.mod h1:iyVcGvDfmZ7m5cliI9TTHj0VTjYDNpc/K71Gp6hukjU= +k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= +k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50= +k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs= +k8s.io/cloud-provider v0.30.2 h1:yov6r02v7sMUNNvzEz51LtL2krn2c1wsC+dy/8BxKQI= +k8s.io/cloud-provider v0.30.2/go.mod h1:w69t2dSjDtI9BYK6SEqj6HmMKIojEk08fXRoUzjFN2I= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= @@ -1261,23 +1111,20 @@ k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= -k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= +k8s.io/kube-openapi v0.0.0-20240703190633-0aa61b46e8c2 h1:T5TEV4a+pEjc+j9Xui3MGGeoDLIN6uzZrx8NYotFMgQ= +k8s.io/kube-openapi v0.0.0-20240703190633-0aa61b46e8c2/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0gQBEuevE/AaBsHY= -k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -kubevirt.io/api v1.2.0 h1:1f8XQLPl4BuHPsc6SHTPnYSYeDxucKCQGa8CdrGJSRc= -kubevirt.io/api v1.2.0/go.mod h1:SbeR9ma4EwnaOZEUkh/lNz0kzYm5LPpEDE30vKXC5Zg= -kubevirt.io/containerized-data-importer-api v1.58.1 h1:Zbf0pCvxb4fBvtMR6uI2OIJZ4UfwFxripzOLMO4HPbI= -kubevirt.io/containerized-data-importer-api v1.58.1/go.mod h1:Y/8ETgHS1GjO89bl682DPtQOYEU/1ctPFBz6Sjxm4DM= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= +k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +kubevirt.io/api v1.2.2 h1:PeA937vsZawmKAsiiDQZJ/BbGH4OhEWsIzWrCNfmYXk= +kubevirt.io/api v1.2.2/go.mod h1:SbeR9ma4EwnaOZEUkh/lNz0kzYm5LPpEDE30vKXC5Zg= +kubevirt.io/containerized-data-importer-api v1.59.0 h1:GdDt9BlR0qHejpMaPfASbsG8JWDmBf1s7xZBj5W9qn0= +kubevirt.io/containerized-data-importer-api v1.59.0/go.mod h1:4yOGtCE7HvgKp7wftZZ3TBvDJ0x9d6N6KaRjRYcUFpE= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 h1:fZYvD3/Vnitfkx6IJxjLAk8ugnZQ7CXVYcRfkSKmuZY= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= @@ -1285,10 +1132,9 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.18.2 h1:RqVW6Kpeaji67CY5nPEfRz6ZfFMk0lWQlNrLqlNpx+Q= -sigs.k8s.io/controller-runtime v0.18.2/go.mod h1:tuAt1+wbVsXIT8lPtk5RURxqAnq7xkpv2Mhttslg7Hw= +sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= +sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= -sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go index ab3e4559a..ddd682704 100644 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ b/pkg/cloudprovider/provider/scaleway/provider.go @@ -39,6 +39,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/utils/ptr" ) type provider struct { @@ -194,7 +195,7 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * Name: machine.Spec.Name, CommercialType: c.CommercialType, Tags: append(c.Tags, string(machine.UID)), - EnableIPv6: c.IPv6, + EnableIPv6: ptr.To(c.IPv6), } serverResp, err := api.CreateServer(createServerRequest, scw.WithContext(ctx)) @@ -382,8 +383,10 @@ func (s *scwServer) Addresses() map[string]corev1.NodeAddressType { addresses[*s.server.PrivateIP] = corev1.NodeInternalIP } - if s.server.PublicIP != nil { - addresses[s.server.PublicIP.Address.String()] = corev1.NodeExternalIP + if s.server.PublicIPs != nil { + for _, publicIP := range s.server.PublicIPs { + addresses[publicIP.Address.String()] = corev1.NodeExternalIP + } } if s.server.IPv6 != nil { From b0bc220889f9bcd03182ca0f3af9031f2c5e71d7 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 9 Jul 2024 14:31:12 +0500 Subject: [PATCH 403/489] Remove kubernetes 1.27 support (#1823) Signed-off-by: Waleed Malik --- README.md | 1 - pkg/controller/machine/controller.go | 12 ++---------- pkg/providerconfig/types/types.go | 24 ++++-------------------- test/e2e/provisioning/all_e2e_test.go | 8 ++++---- test/e2e/provisioning/helper.go | 1 - 5 files changed, 10 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 0803f7d4d..201b49511 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,6 @@ Currently supported K8S versions are: - 1.30 - 1.29 - 1.28 -- 1.27 ### Community Providers diff --git a/pkg/controller/machine/controller.go b/pkg/controller/machine/controller.go index 10b83f677..0a179c175 100644 --- a/pkg/controller/machine/controller.go +++ b/pkg/controller/machine/controller.go @@ -457,11 +457,7 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, mach } // case 3.2: if the node exists and both external and internal CCM are not available. Then set the provider-id for the node. - inTree, err := providerconfigtypes.IntreeCloudProviderImplementationSupported(providerConfig.CloudProvider, machine.Spec.Versions.Kubelet) - if err != nil { - return nil, fmt.Errorf("failed to check if cloud provider %q has in-tree implementation: %w", providerConfig.CloudProvider, err) - } - + inTree := providerconfigtypes.IntreeCloudProviderImplementationSupported(providerConfig.CloudProvider) if !inTree && !r.nodeSettings.ExternalCloudProvider && node.Spec.ProviderID == "" { providerID := fmt.Sprintf(ProviderIDPattern, providerConfig.CloudProvider, machine.UID) if err := r.updateNode(ctx, node, func(n *corev1.Node) { @@ -897,11 +893,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( var providerID string if machine.Spec.ProviderID == nil { - inTree, err := providerconfigtypes.IntreeCloudProviderImplementationSupported(providerConfig.CloudProvider, machine.Spec.Versions.Kubelet) - if err != nil { - return nil, fmt.Errorf("failed to check if cloud provider %q has in-tree implementation: %w", providerConfig.CloudProvider, err) - } - + inTree := providerconfigtypes.IntreeCloudProviderImplementationSupported(providerConfig.CloudProvider) // If both external and internal CCM are not available. We set provider-id for the machine explicitly. if !inTree && !r.nodeSettings.ExternalCloudProvider { providerID = fmt.Sprintf(ProviderIDPattern, providerConfig.CloudProvider, machine.UID) diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 72583bd86..1a6a89640 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -23,8 +23,6 @@ import ( "fmt" "strconv" - "github.com/Masterminds/semver/v3" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/jsonutil" @@ -120,25 +118,11 @@ var ( } ) -func IntreeCloudProviderImplementationSupported(cloudProvider CloudProvider, version string) (inTree bool, err error) { - kubeletVer, err := semver.NewVersion(version) - if err != nil { - return false, fmt.Errorf("failed to parse kubelet version: %w", err) - } - - switch cloudProvider { - case CloudProviderAzure, CloudProviderVsphere, CloudProviderGoogle: - return true, nil - case CloudProviderAWS: - // In-tree AWS support was removed in Kubernetes 1.27. - ltKube127Condition, _ := semver.NewConstraint("< 1.27") - if ltKube127Condition.Check(kubeletVer) { - return true, nil - } - return false, nil - default: - return false, nil +func IntreeCloudProviderImplementationSupported(cloudProvider CloudProvider) (inTree bool) { + if cloudProvider == CloudProviderAzure || cloudProvider == CloudProviderVsphere || cloudProvider == CloudProviderGoogle { + return true } + return false } // DNSConfig contains a machine's DNS configuration. diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index c43faff55..61339d252 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -348,7 +348,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.26. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.27.13", "1.28.11", "1.29.6", "1.30.2"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.28.11", "1.29.6", "1.30.2"))) runScenarios(context.Background(), t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -424,7 +424,7 @@ func TestAWSProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := Not(VersionSelector("1.27.13", "1.28.11", "1.29.6", "1.30.2")) + selector := Not(VersionSelector("1.28.11", "1.29.6", "1.30.2")) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -478,7 +478,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.13", "1.28.11", "1.29.6", "1.30.2"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.28.11", "1.29.6", "1.30.2"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -500,7 +500,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.27.13", "1.28.11", "1.29.6", "1.30.2"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.28.11", "1.29.6", "1.30.2"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index cfe5d1d40..d5739465b 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -34,7 +34,6 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.27.13"), semver.MustParse("v1.28.11"), semver.MustParse("v1.29.6"), semver.MustParse("v1.30.2"), From 7a3746acdc2c3c3bc7481a1f6a11b7c9fa7da538 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 12 Jul 2024 15:11:32 +0500 Subject: [PATCH 404/489] Cleanup for golangci config (#1828) Signed-off-by: Waleed Malik --- .golangci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 6c14b4d17..d8a5e1e44 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -56,8 +56,6 @@ issues: - func Convert_MachineSet_ProviderConfig_To_ProviderSpec should be ConvertMachineSetProviderConfigToProviderSpec - func Convert_Machine_ProviderConfig_To_ProviderSpec should be ConvertMachineProviderConfigToProviderSpec - 'cyclomatic complexity 33 of func `\(\*provider\)\.Create` is high' - - 'cyclomatic complexity 32 of func `\(\*Reconciler\)\.ensureInstanceExistsForMachine` is high' - "SA1019: s.server.IPv6 is deprecated" exclude-dirs: - - pkg/client - pkg/machines From b030806dc31d1b7efad3249b201c374b3faee1e4 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Tue, 23 Jul 2024 19:06:07 +0500 Subject: [PATCH 405/489] OpenStack: Support enabling config drive (#1829) Signed-off-by: Waleed Malik --- docs/cloud-provider.md | 2 ++ pkg/cloudprovider/provider/openstack/provider.go | 7 +++++++ pkg/cloudprovider/provider/openstack/provider_test.go | 4 ++++ pkg/cloudprovider/provider/openstack/types/types.go | 1 + 4 files changed, 14 insertions(+) diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index 74edb7a4a..24979afd7 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -135,6 +135,8 @@ network: "" computeAPIVersion: "" # set trust-device-path flag for kubelet trustDevicePath: false +# set to true to store metadata on a configuration drive instead of the metadata service +configDrive: false # set root disk size rootDiskSizeGB: 50 # set root disk volume type diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 542bf1893..37d81d02c 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -103,6 +103,7 @@ type Config struct { FloatingIPPool string AvailabilityZone string TrustDevicePath bool + ConfigDrive bool RootDiskSizeGB *int RootDiskVolumeType string NodeVolumeAttachLimit *uint @@ -267,6 +268,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, nil, err } + cfg.ConfigDrive, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.ConfigDrive) + if err != nil { + return nil, nil, nil, err + } + cfg.ComputeAPIVersion, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ComputeAPIVersion) if err != nil { return nil, nil, nil, err @@ -611,6 +617,7 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * Name: machine.Spec.Name, FlavorRef: flavor.ID, UserData: []byte(userdata), + ConfigDrive: &cfg.ConfigDrive, SecurityGroups: securityGroups, AvailabilityZone: cfg.AvailabilityZone, Networks: []osservers.Network{{UUID: network.ID}}, diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index 8222076f1..b7757b7e8 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -45,6 +45,7 @@ import ( const expectedServerRequest = `{ "server": { "availability_zone": "eu-de-01", + "config_drive": false, "flavorRef": "1", "imageRef": "1bea47ed-f6a9-463b-b423-14b9cca9ad27", "metadata": { @@ -72,6 +73,7 @@ const expectedServerRequest = `{ const expectedBlockDeviceBootRequest = `{ "server": { "availability_zone": "eu-de-01", + "config_drive": false, "block_device_mapping_v2": [ { "boot_index": 0, @@ -108,6 +110,7 @@ const expectedBlockDeviceBootRequest = `{ const expectedBlockDeviceBootVolumeTypeRequest = `{ "server": { "availability_zone": "eu-de-01", + "config_drive": false, "block_device_mapping_v2": [ { "boot_index": 0, @@ -152,6 +155,7 @@ type openstackProviderSpecConf struct { ProjectID string TenantID string TenantName string + ConfigDrive bool ComputeAPIVersion string } diff --git a/pkg/cloudprovider/provider/openstack/types/types.go b/pkg/cloudprovider/provider/openstack/types/types.go index b6d33369a..17aed61ee 100644 --- a/pkg/cloudprovider/provider/openstack/types/types.go +++ b/pkg/cloudprovider/provider/openstack/types/types.go @@ -52,6 +52,7 @@ type RawConfig struct { RootDiskVolumeType providerconfigtypes.ConfigVarString `json:"rootDiskVolumeType,omitempty"` NodeVolumeAttachLimit *uint `json:"nodeVolumeAttachLimit"` ServerGroup providerconfigtypes.ConfigVarString `json:"serverGroup"` + ConfigDrive providerconfigtypes.ConfigVarBool `json:"configDrive,omitempty"` // This tag is related to server metadata, not compute server's tag Tags map[string]string `json:"tags,omitempty"` } From d89c4fb1189d5301ce43ca378b8c820061067350 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Thu, 25 Jul 2024 22:29:09 +0200 Subject: [PATCH 406/489] Refactor Tinkerbell provider (#1830) * update baremetal provider * fix lint issues * refactor tinkerbell clients Signed-off-by: moadqassem * add netmask to cidr utility Signed-off-by: moadqassem * refactor tinkerbell bare metal plugin * add bare metal machine states * refactor the bare metal provider * remove hegelurl * addressing PR review Signed-off-by: moadqassem * linting Signed-off-by: moadqassem * remove the metadata client as it is not needed Signed-off-by: moadqassem * linting Signed-off-by: moadqassem --------- Signed-off-by: moadqassem Co-authored-by: Mohamed Rafraf --- ...aremetal-tinkerbell-machinedeployment.yaml | 89 +--- go.mod | 19 +- go.sum | 403 +----------------- .../provider/baremetal/plugins/driver.go | 9 +- .../plugins/tinkerbell/client/hardware.go | 116 +++-- .../plugins/tinkerbell/client/interface.go | 39 -- .../plugins/tinkerbell/client/template.go | 260 +++++++++-- .../tinkerbell/client/{common.go => utils.go} | 29 +- .../plugins/tinkerbell/client/workflow.go | 235 +++------- .../baremetal/plugins/tinkerbell/driver.go | 275 ++++++------ .../plugins/tinkerbell/driver_test.go | 346 --------------- .../baremetal/plugins/tinkerbell/hardware.go | 64 --- .../plugins/tinkerbell/metadata/client.go | 123 ------ .../baremetal/plugins/tinkerbell/template.go | 93 ---- .../plugins/tinkerbell/types/hardware.go | 68 +++ .../plugins/tinkerbell/types/types.go | 63 +++ .../provider/baremetal/provider.go | 79 +--- .../provider/baremetal/types/types.go | 13 +- 18 files changed, 673 insertions(+), 1650 deletions(-) delete mode 100644 pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/interface.go rename pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/{common.go => utils.go} (51%) delete mode 100644 pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver_test.go delete mode 100644 pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/hardware.go delete mode 100644 pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata/client.go delete mode 100644 pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/template.go create mode 100644 pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/hardware.go create mode 100644 pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/types.go diff --git a/examples/baremetal-tinkerbell-machinedeployment.yaml b/examples/baremetal-tinkerbell-machinedeployment.yaml index ff3ed3c6e..d7154e590 100644 --- a/examples/baremetal-tinkerbell-machinedeployment.yaml +++ b/examples/baremetal-tinkerbell-machinedeployment.yaml @@ -25,88 +25,15 @@ spec: cloudProvider: "baremetal" cloudProviderSpec: driver: "tinkerbell" - metadataClientConfig: - endpoint: - secretKeyRef: - namespace: kube-system - name: machine-controller-baremetal-tb - key: endpoint - authMethod: - secretKeyRef: - namespace: kube-system - name: machine-controller-baremetal-tb - key: authMethod - username: - secretKeyRef: - namespace: kube-system - name: machine-controller-baremetal-tb - key: username - password: - secretKeyRef: - namespace: kube-system - name: machine-controller-baremetal-tb - key: password - token: - secretKeyRef: - namespace: kube-system - name: machine-controller-baremetal-tb - key: token driverSpec: - provisionerIPAddress: << PROVISIONER_IP_ADDRESS >> - mirrorHost: << MIRROR_HOST >> - hardware: - id: << MACHINE_NAME >> - metadata: - facility: - facilitycode: << FACILITY_CODE >> - planslug: << PLAN_SLUG >> - state: "" - instance: - operatingsystemversion: - distro: << OS_NAME >> - imagetag: << IMAGE_TAG >> - osslug: << OS_NAME >> - slug: << OS_NAME >> - version: << OS_VERSION >> - storage: - disks: - - device: /dev/sda - wipetable: true - partitions: - - size: 4096 - label: BIOS - number: 1 - - size: 3993600 - label: SWAP - number: 2 - - size: 0 - label: ROOT - number: 3 - filesystems: - - mount: - point: / - create: - options: - - -L - - ROOT - device: /dev/sda3 - format: ext4 - - mount: - point: none - create: - options: - - -L - - SWAP - device: /dev/sda2 - format: swap - network: - interfaces: - - dhcp: - arch: x86_64 - uefi: false - netboot: - allowpxe: false - allowworkflow: false + clusterName: "<< CLUSTER_NAME >>" + osImageUrl: "<< OS_IMAGE_URL >>" + auth: + kubeconfig: + value: "<< KUBECONFIG_BASE64 >>" + hardwareRef: + name: hardware-1 + namespace: "default" operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/go.mod b/go.mod index ea112db63..64672b4ba 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ec2 v1.168.0 github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 github.com/aws/smithy-go v1.20.3 - github.com/davecgh/go-spew v1.1.1 + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/digitalocean/godo v1.118.0 github.com/go-logr/logr v1.4.2 github.com/go-logr/zapr v1.3.0 @@ -34,11 +34,11 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 - github.com/pmezard/go-difflib v1.0.0 + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 github.com/prometheus/client_golang v1.19.1 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.28 github.com/spf13/pflag v1.0.5 - github.com/tinkerbell/tink v0.8.0 + github.com/tinkerbell/tink v0.10.0 github.com/vmware/go-vcloud-director/v2 v2.25.0 github.com/vmware/govmomi v0.38.0 github.com/vultr/govultr/v3 v3.9.0 @@ -48,14 +48,14 @@ require ( golang.org/x/oauth2 v0.21.0 gomodules.xyz/jsonpatch/v2 v2.4.0 google.golang.org/api v0.187.0 - google.golang.org/grpc v1.65.0 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.30.2 + k8s.io/api v0.30.3 k8s.io/apiextensions-apiserver v0.30.2 - k8s.io/apimachinery v0.30.2 - k8s.io/client-go v0.30.2 + k8s.io/apimachinery v0.30.3 + k8s.io/client-go v0.30.3 k8s.io/cloud-provider v0.30.2 k8s.io/klog v1.0.0 + k8s.io/kubectl v0.30.3 k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 kubevirt.io/api v1.2.2 kubevirt.io/containerized-data-importer-api v1.59.0 @@ -91,8 +91,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/distribution/reference v0.5.0 // indirect - github.com/docker/distribution v2.8.3+incompatible // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect @@ -135,11 +133,9 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/onsi/ginkgo/v2 v2.19.0 // indirect github.com/onsi/gomega v1.33.1 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect github.com/openshift/api v0.0.0-20240708071937-c9a91940bf0f // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect - github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046 // indirect github.com/peterhellberg/link v1.2.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect @@ -165,6 +161,7 @@ require ( google.golang.org/genproto v0.0.0-20240701130421-f6361c86f094 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + google.golang.org/grpc v1.65.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 2f7af65ad..e512e355a 100644 --- a/go.sum +++ b/go.sum @@ -1,34 +1,12 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= cloud.google.com/go/auth v0.6.1 h1:T0Zw1XM5c1GlpN2HYr2s+m3vr1p2wy+8VN+Z1FKxW38= cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4= cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/compute/metadata v0.4.0 h1:vHzJCWaM4g8XIcm8kopr3XmDA4Gy/lblD3EhhSux05c= cloud.google.com/go/compute/metadata v0.4.0/go.mod h1:SIQh1Kkb4ZJ8zJ874fqVkslA29PRXuleyj6vOzlbK7M= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/iam v1.1.9 h1:oSkYLVtVme29uGYrOcKcvJRht7cHJpYD09GM9JaR0TE= cloud.google.com/go/iam v1.1.9/go.mod h1:Nt1eDWNYH9nGQg3d/mY7U1hvfGmsaG9o/kLGoLoLXjQ= cloud.google.com/go/logging v1.10.0 h1:f+ZXMqyrSJ5vZ5pE/zr0xC8y/M9BLNzQeLBwfeZ+wY4= @@ -37,15 +15,6 @@ cloud.google.com/go/longrunning v0.5.9 h1:haH9pAuXdPAMqHvzX0zlWQigXT7B0+CL4/2nXX cloud.google.com/go/longrunning v0.5.9/go.mod h1:HD+0l9/OOW0za6UWdKJtXoFAX/BGg/3Wj8p10NeWF7c= cloud.google.com/go/monitoring v1.20.1 h1:XmM6uk4+mI2ZhWdI2n/2GNhJdpeQN+1VdG2UWEDhX48= cloud.google.com/go/monitoring v1.20.1/go.mod h1:FYSe/brgfuaXiEzOQFhTjsEsJv+WePyK71X7Y8qo6uQ= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/99designs/gqlgen v0.15.1 h1:48bRXecwlCNTa/n2bMSp2rQsXNxwZ54QHbiULNf78ec= github.com/99designs/gqlgen v0.15.1/go.mod h1:nbeSjFkqphIqpZsYe1ULVz0yfH8hjpJdJIQoX/e0G2I= @@ -83,7 +52,6 @@ github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXY github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240703112053-c8500c47bced h1:yLjIFnPG7e6IVIDxFWp+dUkxlOgOkx8ttesy9dGAEms= github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240703112053-c8500c47bced/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= @@ -97,14 +65,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/aliyun/alibaba-cloud-sdk-go v1.62.784 h1:wtou656eZpKB1QFJ26ChcmpIZvAdtVZvxuhD7BJpZtA= github.com/aliyun/alibaba-cloud-sdk-go v1.62.784/go.mod h1:SOSDHfe1kX91v3W5QiBsWSLqeLxImobbMX1mxrFHsVQ= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= @@ -136,15 +98,9 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 h1:+woJ607dllHJQtsnJLi52ycuqHMw github.com/aws/aws-sdk-go-v2/service/sts v1.30.1/go.mod h1:jiNR3JqT15Dm+QWq2SRgh0x0bCNSRP2L25+CqPNpJlQ= github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -152,27 +108,21 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creasty/defaults v1.5.2 h1:/VfB6uxpyp6h0fr7SPp7n8WJBoV8jfxQXPCnkVSjyls= github.com/creasty/defaults v1.5.2/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/digitalocean/godo v1.118.0 h1:lkzGFQmACrVCp7UqH1sAi4JK/PWwlc5aaxubgorKmC4= github.com/digitalocean/godo v1.118.0/go.mod h1:Vk0vpCot2HOAJwc5WE8wljZGtJ3ZtWIc8MQ8rF38sdo= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= -github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= -github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= -github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -183,9 +133,6 @@ github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRr github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= @@ -208,17 +155,8 @@ github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIu github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8= github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -249,7 +187,6 @@ github.com/go-playground/validator/v10 v10.10.1 h1:uA0+amWMiglNZKZ9FJRKUAe9U3RX9 github.com/go-playground/validator/v10 v10.10.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= github.com/go-resty/resty/v2 v2.13.1 h1:x+LHXBI2nMB1vqndymf26quycC4aggYJ7DECYbiz03g= github.com/go-resty/resty/v2 v2.13.1/go.mod h1:GznXlLxkq6Nh4sU59rPmUw3VtgpO3aS96ORAI6Q7d+0= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= @@ -257,7 +194,6 @@ github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1v github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= @@ -267,24 +203,12 @@ github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOW github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -299,17 +223,13 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -324,19 +244,9 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -345,8 +255,6 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= @@ -357,9 +265,6 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= @@ -369,17 +274,13 @@ github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISH github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb h1:tsEKRC3PU9rMw18w/uAptoijhgG4EvlA5kfJPtwrMDk= github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= github.com/hetznercloud/hcloud-go/v2 v2.10.2 h1:9gyTUPhfNbfbS40Spgij5mV5k37bOZgt8iHKCbfGs5I= github.com/hetznercloud/hcloud-go/v2 v2.10.2/go.mod h1:xQ+8KhIS62W0D78Dpi57jsufWh844gUw1az5OUvaeq8= -github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= @@ -391,32 +292,21 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/k0kubun/pp/v3 v3.1.0 h1:ifxtqJkRZhw3h554/z/8zm6AAbyO4LLKDlA5eV+9O8Q= github.com/k0kubun/pp/v3 v3.1.0/go.mod h1:vIrP5CF0n78pKHm2Ku6GVerpZBJvscg48WepUYEk2gw= -github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/keploy/go-sdk v0.7.2 h1:mvvjDRciMSFTgOF/KIGz38ElJZKkM1WlniaHseaPhpo= github.com/keploy/go-sdk v0.7.2/go.mod h1:TtJIM+Gkq76FzfkD8W9u1F8NDkC9sVY8nYvmbRo1nhg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= +github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/kolo/xmlrpc v0.0.0-20190717152603-07c4ee3fd181/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00= github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -441,21 +331,19 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= @@ -463,8 +351,6 @@ github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3P github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nutanix-cloud-native/prism-go-client v0.4.0 h1:P9mLW6eyKMUXVQBzuVL5k7WjV1YwVu8XNpu2XAsRgGo= @@ -486,19 +372,14 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/openshift/api v0.0.0-20240708071937-c9a91940bf0f h1:NmJAlN2fPnL86aq5BbEQJ62v/D16LzIaaQ0Qn72s87E= github.com/openshift/api v0.0.0-20240708071937-c9a91940bf0f/go.mod h1:OOh6Qopf21pSzqNVCB5gomomBXb8o5sGKZxG2KNpaXM= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= github.com/packethost/packngo v0.31.0 h1:LLH90ardhULWbagBIc3I3nl2uU75io0a7AwY6hyi0S4= github.com/packethost/packngo v0.31.0/go.mod h1:Io6VJqzkiqmIEQbpOjeIw9v8q9PfcTEq8TEY/tMQsfw= -github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046 h1:zF+CUhv8LMpqTFFpECX6WF+yUWS2Bd1Nc1W+AczzqbY= -github.com/packethost/pkg v0.0.0-20230710142318-f8a288cd3046/go.mod h1:W/xTaqgJ2kJCwayvm3BF3bOj9ku0F5DjjYnZaioxnOk= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= @@ -506,46 +387,24 @@ github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/peterhellberg/link v1.2.0 h1:UA5pg3Gp/E0F2WdX7GERiNrPQrM1K6CVJUUWfHa4t6c= github.com/peterhellberg/link v1.2.0/go.mod h1:gYfAh+oJgQu2SrZHg5hROVRQe1ICoK0/HHJTcE0edxc= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.31.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/rollbar/rollbar-go v1.4.2/go.mod h1:kLQ9gP3WCRGrvJmF0ueO3wK9xWocej8GRX98D8sa39w= -github.com/rollbar/rollbar-go/errors v0.0.0-20210929193720-32947096267e/go.mod h1:Ie0xEc1Cyj+T4XMO8s0Vf7pMfvSAAy1sb4AYc8aJsao= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= @@ -556,21 +415,14 @@ github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -586,9 +438,8 @@ github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tinkerbell/lint-install v0.0.0-20211012174934-5ee5ab01db76/go.mod h1:0h2KsALaQLNkoVeV+G+HjBWWCnp0COFYhJdRd5WCQPM= -github.com/tinkerbell/tink v0.8.0 h1:qgl/rglpO5Rvq6UKZd29O6X9mDgZZYgf841+Y0IYWak= -github.com/tinkerbell/tink v0.8.0/go.mod h1:bfAkSH7J/QQYIyqZRR6IQp8w78aac6l8Z2Lws5uXz6A= +github.com/tinkerbell/tink v0.10.0 h1:W34Psx6Yn5o0jILSgyjF8C7Cg6ivDcccUqJ6zG9zGzA= +github.com/tinkerbell/tink v0.10.0/go.mod h1:Dc8PpbvrC8Kfhr8yGhUsWBN6Dq6opRIX655KDTxLUJY= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= @@ -613,9 +464,7 @@ github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyh github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -627,50 +476,31 @@ go.keploy.io/server v0.7.12 h1:DKDSO6T9Q4d4A8MKL+sk7U26KRcvZ+ZG0mbFhYIJJyk= go.keploy.io/server v0.7.12/go.mod h1:ch4rD1NCgtxozDHD9yVk+sLHWz5HgefOqrgEdEIgfBQ= go.mongodb.org/mongo-driver v1.8.3 h1:TDKlTkGDKm9kkJVUOAXDK5/fkqKHJVwYQSpoRfB43R4= go.mongodb.org/mongo-driver v1.8.3/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= -go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= -go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -688,14 +518,7 @@ golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w= golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= @@ -703,23 +526,11 @@ golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMx golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -729,41 +540,19 @@ golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -777,77 +566,39 @@ golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211015200801-69063c4bb744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -869,9 +620,7 @@ golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -883,9 +632,6 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -894,46 +640,12 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= @@ -952,64 +664,14 @@ gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJ gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.187.0 h1:Mxs7VATVC2v7CY+7Xwm4ndkX71hpElcvx0D1Ji/p1eo= google.golang.org/api v0.187.0/go.mod h1:KIHlTc4x7N7gKKuVsdmfBXN13yEEWXWFURWY6SBp2gk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20240701130421-f6361c86f094 h1:6whtk83KtD3FkGrVb2hFXuQ+ZMbCNdakARIn/aHMmG8= google.golang.org/genproto v0.0.0-20240701130421-f6361c86f094/go.mod h1:Zs4wYw8z1zr6RNF4cwYb31mvN/EGaKAdQjNCF3DW6K4= google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= @@ -1017,25 +679,12 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= -google.golang.org/grpc/examples v0.0.0-20210728214646-ad0a2a847cdf/go.mod h1:bF8wuZSAZTcbF7ZPKrDI/qY52toTP/yxLpRRY4Eu9Js= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1053,14 +702,12 @@ google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6h google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= @@ -1069,9 +716,7 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -1083,22 +728,17 @@ gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= -k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= -k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= +k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ= +k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04= k8s.io/apiextensions-apiserver v0.30.2 h1:l7Eue2t6QiLHErfn2vwK4KgF4NeDgjQkCXtEbOocKIE= k8s.io/apiextensions-apiserver v0.30.2/go.mod h1:lsJFLYyK40iguuinsb3nt+Sj6CmodSI4ACDLep1rgjw= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.30.2 h1:fEMcnBj6qkzzPGSVsAZtQThU62SmQ4ZymlXRC5yFSCg= -k8s.io/apimachinery v0.30.2/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/client-go v0.30.2 h1:sBIVJdojUNPDU/jObC+18tXWcTJVcwyqS9diGdWHk50= -k8s.io/client-go v0.30.2/go.mod h1:JglKSWULm9xlJLx4KCkfLLQ7XwtlbflV6uFFSHTMgVs= +k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= +k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= +k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= k8s.io/cloud-provider v0.30.2 h1:yov6r02v7sMUNNvzEz51LtL2krn2c1wsC+dy/8BxKQI= k8s.io/cloud-provider v0.30.2/go.mod h1:w69t2dSjDtI9BYK6SEqj6HmMKIojEk08fXRoUzjFN2I= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= @@ -1108,7 +748,6 @@ k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= -k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= @@ -1117,6 +756,8 @@ k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lV k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20240703190633-0aa61b46e8c2 h1:T5TEV4a+pEjc+j9Xui3MGGeoDLIN6uzZrx8NYotFMgQ= k8s.io/kube-openapi v0.0.0-20240703190633-0aa61b46e8c2/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= +k8s.io/kubectl v0.30.3 h1:YIBBvMdTW0xcDpmrOBzcpUVsn+zOgjMYIu7kAq+yqiI= +k8s.io/kubectl v0.30.3/go.mod h1:IcR0I9RN2+zzTRUa1BzZCm4oM0NLOawE6RzlDvd1Fpo= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= @@ -1127,11 +768,7 @@ kubevirt.io/containerized-data-importer-api v1.59.0 h1:GdDt9BlR0qHejpMaPfASbsG8J kubevirt.io/containerized-data-importer-api v1.59.0/go.mod h1:4yOGtCE7HvgKp7wftZZ3TBvDJ0x9d6N6KaRjRYcUFpE= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 h1:fZYvD3/Vnitfkx6IJxjLAk8ugnZQ7CXVYcRfkSKmuZY= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= -mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= diff --git a/pkg/cloudprovider/provider/baremetal/plugins/driver.go b/pkg/cloudprovider/provider/baremetal/plugins/driver.go index 0147e1907..79f9d5d36 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/driver.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/driver.go @@ -18,9 +18,10 @@ package plugins import ( "context" + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" ) type Driver string @@ -36,10 +37,10 @@ type CloudConfigSettings struct { // PluginDriver manages the communications between the machine controller cloud provider and the bare metal env. type PluginDriver interface { - GetServer(context.Context, types.UID, runtime.RawExtension) (Server, error) + GetServer(context.Context) (Server, error) Validate(runtime.RawExtension) error - ProvisionServer(context.Context, types.UID, *CloudConfigSettings, runtime.RawExtension) (Server, error) - DeprovisionServer(context.Context, types.UID) error + ProvisionServer(context.Context, *zap.SugaredLogger, metav1.ObjectMeta, runtime.RawExtension, string) (Server, error) + DeprovisionServer(context.Context) error } // Server represents the server/instance which exists in the bare metal env. diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/hardware.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/hardware.go index c8ccf6c98..2d3b1178c 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/hardware.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/hardware.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Machine Controller Authors. +Copyright 2024 The Machine Controller Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,94 +18,78 @@ package client import ( "context" - "errors" "fmt" - "strings" - "github.com/google/uuid" - "github.com/tinkerbell/tink/protos/hardware" - "google.golang.org/grpc" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" + tbtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" + tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) -// Hardware client for Tinkerbell. -type Hardware struct { - client hardware.HardwareServiceClient +// HardwareClient manages Tinkerbell hardware resources across two clusters. +type HardwareClient struct { + TinkerbellClient client.Client } -// NewHardwareClient returns a Hardware client. -func NewHardwareClient(client hardware.HardwareServiceClient) *Hardware { - return &Hardware{client: client} +// NewHardwareClient creates a new instance of HardwareClient. +func NewHardwareClient(tinkerbellClient client.Client) *HardwareClient { + return &HardwareClient{ + TinkerbellClient: tinkerbellClient, + } } -// Create Tinkerbell Hardware. -func (t *Hardware) Create(ctx context.Context, h *hardware.Hardware) error { - if h == nil { - return errors.New("hardware should not be nil") +// GetHardware fetches a hardware object from the Tinkerbell cluster based on the hardware reference in the machine +// deployment object. +func (h *HardwareClient) GetHardware(ctx context.Context, hardwareRef types.NamespacedName) (*tinkv1alpha1.Hardware, error) { + hardware := &tinkv1alpha1.Hardware{} + if err := h.TinkerbellClient.Get(ctx, client.ObjectKey{Namespace: hardwareRef.Namespace, Name: hardwareRef.Name}, hardware); err != nil { + return nil, fmt.Errorf("failed to get hardware '%s' in namespace '%s': %w", hardwareRef.Name, hardwareRef.Namespace, err) } - if h.GetId() == "" { - h.Id = uuid.New().String() + return hardware, nil +} + +// SetHardwareID sets the ID of a specified Hardware object. +func (h *HardwareClient) SetHardwareID(ctx context.Context, hardware *tinkv1alpha1.Hardware, newID string) error { + if hardware.Spec.Metadata == nil { + hardware.Spec.Metadata = &tinkv1alpha1.HardwareMetadata{} } - if _, err := t.client.Push(ctx, &hardware.PushRequest{Data: h}); err != nil { - return fmt.Errorf("creating hardware in Tinkerbell: %w", err) + if hardware.Spec.Metadata.Instance == nil { + hardware.Spec.Metadata.Instance = &tinkv1alpha1.MetadataInstance{} } - return nil -} + hardware.Spec.Metadata.Instance.ID = newID + // Set the new ID + hardware.Spec.Metadata.State = tbtypes.Staged + if newID == "" { + // Machine has been deprovisioned + hardware.Spec.Metadata.State = tbtypes.Decommissioned + } -// Update Tinkerbell Hardware. -func (t *Hardware) Update(ctx context.Context, h *hardware.Hardware) error { - if _, err := t.client.Push(ctx, &hardware.PushRequest{Data: h}); err != nil { - return fmt.Errorf("updating template in Tinkerbell: %w", err) + // Update the hardware object in the cluster + if err := h.TinkerbellClient.Update(ctx, hardware); err != nil { + return fmt.Errorf("failed to update hardware ID for '%s': %w", hardware.Name, err) } return nil } -// Get returns a Tinkerbell Hardware. -func (t *Hardware) Get(ctx context.Context, id, ip, mac string) (*hardware.Hardware, error) { - var method func(context.Context, *hardware.GetRequest, ...grpc.CallOption) (*hardware.Hardware, error) - - req := &hardware.GetRequest{} - - switch { - case id != "": - req.Id = id - method = t.client.ByID - case mac != "": - req.Mac = mac - method = t.client.ByMAC - case ip != "": - req.Ip = ip - method = t.client.ByIP - default: - return nil, errors.New("need to specify either id, ip, or mac") +func (h *HardwareClient) GetHardwareWithID(ctx context.Context, uid string) (*tinkv1alpha1.Hardware, error) { + // List all hardware in the cluster + var hardwares tinkv1alpha1.HardwareList + if err := h.TinkerbellClient.List(ctx, &hardwares); err != nil { + return nil, fmt.Errorf("failed to list hardware: %w", err) } - tinkHardware, err := method(ctx, req) - if err != nil { - if err.Error() == sqlErrorString || err.Error() == sqlErrorStringAlt { - return nil, fmt.Errorf("hardware %w", ErrNotFound) + // Find the Hardware with the given ID + for _, hw := range hardwares.Items { + if hw.Spec.Metadata.Instance.ID == uid { + return &hw, nil } - - return nil, fmt.Errorf("getting hardware from Tinkerbell: %w", err) } - return tinkHardware, nil -} - -// Delete a Tinkerbell Hardware. -func (t *Hardware) Delete(ctx context.Context, id string) error { - if _, err := t.client.Delete(ctx, &hardware.DeleteRequest{Id: id}); err != nil { - if err.Error() == sqlErrorString || - err.Error() == sqlErrorStringAlt || - strings.Contains(err.Error(), sqlErrorNotFound) { - return fmt.Errorf("hardware %w", ErrNotFound) - } - - return fmt.Errorf("deleting hardware from Tinkerbell: %w", err) - } - - return nil + return nil, errors.ErrInstanceNotFound } diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/interface.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/interface.go deleted file mode 100644 index 00b5c6410..000000000 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/interface.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2021 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package client - -import ( - "context" - - "github.com/tinkerbell/tink/protos/hardware" - "github.com/tinkerbell/tink/protos/template" -) - -type HardwareClient interface { - Get(context.Context, string, string, string) (*hardware.Hardware, error) - Delete(context.Context, string) error - Create(context.Context, *hardware.Hardware) error -} - -type TemplateClient interface { - Get(context.Context, string, string) (*template.WorkflowTemplate, error) - Create(context.Context, *template.WorkflowTemplate) error -} - -type WorkflowClient interface { - Create(context.Context, string, string) (string, error) -} diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go index 79e9f00be..cc1078ef6 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Machine Controller Authors. +Copyright 2024 The Machine Controller Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -20,73 +20,251 @@ import ( "context" "fmt" - "github.com/tinkerbell/tink/protos/template" + tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" + "gopkg.in/yaml.v3" + + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) -// Template client for Tinkerbell. +type Task struct { + Name string `json:"name"` + WorkerAddr string `json:"worker" yaml:"worker"` + Actions []Action `json:"actions"` + Volumes []string `json:"volumes,omitempty"` + Environment map[string]string `json:"environment,omitempty"` +} + +// Action represents a workflow action. +type Action struct { + Name string `json:"name,omitempty"` + Image string `json:"image,omitempty"` + Timeout int64 `json:"timeout,omitempty"` + Volumes []string `json:"volumes,omitempty"` + Pid string `json:"pid,omitempty"` + Environment map[string]string `json:"environment,omitempty"` + Command []string `json:"command,omitempty"` +} type Template struct { - client template.TemplateServiceClient + Version string `yaml:"version"` + Name string `yaml:"name"` + GlobalTimeout int64 `yaml:"global_timeout"` + Tasks []Task `yaml:"tasks"` } -// NewTemplateClient returns a Template client. -func NewTemplateClient(client template.TemplateServiceClient) *Template { - return &Template{client: client} +const ( + fsType = "ext4" + defaultInterpreter = "/bin/sh -c" + hardwareDisk1 = "{{ index .Hardware.Disks 0 }}" + + ProvisionWorkerNodeTemplate = "provision-worker-node" +) + +// TemplateClient handles interactions with the Tinkerbell Templates in the Tinkerbell cluster. +type TemplateClient struct { + tinkclient client.Client } -// Get returns a Tinkerbell Template. -func (t *Template) Get(ctx context.Context, id, name string) (*template.WorkflowTemplate, error) { - req := &template.GetRequest{} - if id != "" { - req.GetBy = &template.GetRequest_Id{Id: id} - } else { - req.GetBy = &template.GetRequest_Name{Name: name} +// NewTemplateClient creates a new client for managing Tinkerbell Templates. +func NewTemplateClient(k8sClient client.Client) *TemplateClient { + return &TemplateClient{ + tinkclient: k8sClient, } +} - tinkTemplate, err := t.client.GetTemplate(ctx, req) - if err != nil { - if err.Error() == sqlErrorString || err.Error() == sqlErrorStringAlt { - return nil, fmt.Errorf("template %w", ErrNotFound) - } +func (t *TemplateClient) Delete(ctx context.Context, namespacedName types.NamespacedName) error { + template := &tinkv1alpha1.Template{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespacedName.Name, + Namespace: namespacedName.Namespace, + }, + } - return nil, fmt.Errorf("getting template from Tinkerbell: %w", err) + if err := t.tinkclient.Delete(ctx, template); err != nil { + return fmt.Errorf("failed to delete Template in Tinkerbell cluster: %w", err) } - return tinkTemplate, nil + return nil } -// Update a Tinkerbell Template. -func (t *Template) Update(ctx context.Context, template *template.WorkflowTemplate) error { - if _, err := t.client.UpdateTemplate(ctx, template); err != nil { - return fmt.Errorf("updating template in Tinkerbefll: %w", err) +// CreateTemplate creates a Tinkerbell Template in the Kubernetes cluster. +func (t *TemplateClient) CreateTemplate(ctx context.Context, hardware *tinkv1alpha1.Hardware, namespace, osImageURL string) error { + template := &tinkv1alpha1.Template{} + if err := t.tinkclient.Get(ctx, types.NamespacedName{ + Name: ProvisionWorkerNodeTemplate, + Namespace: namespace, + }, template); err != nil { + if kerrors.IsNotFound(err) { + data, err := getTemplate(hardware, osImageURL) + if err != nil { + return err + } + + template.Name = ProvisionWorkerNodeTemplate + template.Namespace = namespace + template.Spec = tinkv1alpha1.TemplateSpec{ + Data: &data, // templateData is a string containing the YAML definition. + } + + // Create the Template object in the Tinkerbell cluster + if err := t.tinkclient.Create(ctx, template); err != nil { + return fmt.Errorf("failed to create Template in Tinkerbell cluster: %w", err) + } + + return nil + } + + return fmt.Errorf("failed to get template %s: %w", ProvisionWorkerNodeTemplate, err) } return nil } -// Create a Tinkerbell Template. -func (t *Template) Create(ctx context.Context, template *template.WorkflowTemplate) error { - resp, err := t.client.CreateTemplate(ctx, template) +func getTemplate(hardware *tinkv1alpha1.Hardware, osImageURL string) (string, error) { + actions := []Action{ + createWipeDiskAction(), + createStreamUbuntuImageAction(hardwareDisk1, osImageURL), + createGrowPartitionAction(hardwareDisk1), + createNetworkConfigAction(), + createCloudInitConfigAction(), + decodeCloudInitFile(hardware.Name), + } + + task := Task{ + Name: "os-installation", + WorkerAddr: "{{.device_1}}", + Volumes: []string{"/dev:/dev", "/dev/console:/dev/console", "/lib/firmware:/lib/firmware:ro"}, + Actions: actions, + } + + template := Template{ + Name: "ubuntu", + Version: "0.1", + GlobalTimeout: 1800, + Tasks: []Task{task}, + } + yamlData, err := yaml.Marshal(template) if err != nil { - return fmt.Errorf("creating template in Tinkerbell: %w", err) + return "", fmt.Errorf("error marshaling the template to YAML: %w", err) } - template.Id = resp.GetId() + return string(yamlData), nil +} - return nil +func createWipeDiskAction() Action { + wipeScript := `apk add --no-cache util-linux +disks="{{ .Hardware.Disks }}" +disks=${disks:1:-1} +for disk in $disks; do + for partition in $(ls ${disk}* 2>/dev/null); do + if [ -b "${partition}" ]; then + echo "Wiping ${partition}..." + wipefs -af "${partition}" + fi + done +done +echo "All partitions on ${disks} have been wiped." +` + return Action{ + Name: "wipe-disk", + Image: "alpine:3.18", + Timeout: 600, + Command: []string{"/bin/sh", "-c", wipeScript}, + } } -// Delete a Tinkerbell Template. -func (t *Template) Delete(ctx context.Context, id string) error { - req := &template.GetRequest{ - GetBy: &template.GetRequest_Id{Id: id}, +func createStreamUbuntuImageAction(destDisk, osImageURL string) Action { + return Action{ + Name: "stream-ubuntu-image", + Image: "quay.io/tinkerbell-actions/image2disk:v1.0.0", + Timeout: 600, + Environment: map[string]string{ + "DEST_DISK": destDisk, + "IMG_URL": osImageURL, + "COMPRESSED": "true", + }, } - if _, err := t.client.DeleteTemplate(ctx, req); err != nil { - if err.Error() == sqlErrorString || err.Error() == sqlErrorStringAlt { - return fmt.Errorf("template %w", ErrNotFound) - } +} - return fmt.Errorf("deleting template from Tinkerbell: %w", err) +func createGrowPartitionAction(destDisk string) Action { + return Action{ + Name: "grow-partition", + Image: "quay.io/tinkerbell/actions/cexec:c5bde803d9f6c90f1a9d5e06930d856d1481854c", + Timeout: 90, + Environment: map[string]string{ + "BLOCK_DEVICE": "{{ index .Hardware.Disks 0 }}3", + "FS_TYPE": fsType, + "CHROOT": "y", + "DEFAULT_INTERPRETER": defaultInterpreter, + "CMD_LINE": fmt.Sprintf("growpart %s 3 && resize2fs %s3", destDisk, destDisk), + }, } +} - return nil +func createNetworkConfigAction() Action { + netplaneConfig := ` +network: + version: 2 + renderer: networkd + ethernets: + {{.interface_name}}: + dhcp4: no + addresses: + - {{.cidr}} + nameservers: + addresses: + - {{.ns}} + routes: + - to: default + via: {{.default_route}}` + return Action{ + Name: "add-netplan-config", + Image: "quay.io/tinkerbell-actions/writefile:v1.0.0", + Timeout: 90, + Environment: map[string]string{ + "DEST_DISK": "{{ index .Hardware.Disks 0 }}3", + "FS_TYPE": fsType, + "DEST_PATH": "/etc/netplan/config.yaml", + "CONTENTS": netplaneConfig, + "UID": "0", + "GID": "0", + "MODE": "0644", + "DIRMODE": "0755", + }, + } +} + +func createCloudInitConfigAction() Action { + return Action{ + Name: "add-cloud-init-config", + Image: "quay.io/tinkerbell-actions/writefile:v1.0.0", + Timeout: 90, + Environment: map[string]string{ + "DEST_DISK": "{{ index .Hardware.Disks 0 }}3", + "FS_TYPE": fsType, + "DEST_PATH": "{{.dst_path}}", + "CONTENTS": "{{.cloud_init_script}}", + "UID": "0", + "GID": "0", + "MODE": "0644", + "DIRMODE": "0755", + }, + } +} + +func decodeCloudInitFile(hardwareName string) Action { + return Action{ + Name: "decode-cloud-init-file", + Image: "quay.io/tinkerbell/actions/cexec:latest", + Timeout: 90, + Environment: map[string]string{ + "BLOCK_DEVICE": "{{ index .Hardware.Disks 0 }}3", + "FS_TYPE": fsType, + "CHROOT": "y", + "DEFAULT_INTERPRETER": "/bin/sh -c", + "CMD_LINE": fmt.Sprintf("cat /tmp/%s-bootstrap-config | base64 -d > /etc/cloud/cloud.cfg.d/%s-cloud-init.cfg", hardwareName, hardwareName), + }, + } } diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/common.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/utils.go similarity index 51% rename from pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/common.go rename to pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/utils.go index 524168e09..f73e817db 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/common.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/utils.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Machine Controller Authors. +Copyright 2024 The Machine Controller Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,19 +14,26 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Package client contains a client wrapper for Tinkerbell. package client import ( - "errors" + "fmt" + "net" + "strings" + + tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" ) -// ErrNotFound is returned if a requested resource is not found. -var ErrNotFound = errors.New("resource not found") +func convertNetmaskToCIDR(ip *tinkv1alpha1.IP) string { + mask := net.IPMask(net.ParseIP(ip.Netmask).To4()) + length, _ := mask.Size() -// than parsing for these specific error message. -const ( - sqlErrorString = "rpc error: code = Unknown desc = sql: no rows in result set" - sqlErrorStringAlt = "rpc error: code = Unknown desc = SELECT: sql: no rows in result set" - sqlErrorNotFound = "rpc error: code = NotFound desc = not found" -) + cidr := "" + parts := strings.Split(ip.Address, ".") + for i := 0; i < len(parts); i++ { + cidr += parts[i] + "." + } + cidr = strings.TrimSuffix(cidr, ".") + + return fmt.Sprintf("%s/%v", cidr, length) +} diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go index 0c6e682eb..313609c9f 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Machine Controller Authors. +Copyright 2024 The Machine Controller Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,203 +18,90 @@ package client import ( "context" - "encoding/json" - "errors" + "encoding/base64" "fmt" - "io" + kerrors "k8s.io/apimachinery/pkg/api/errors" - "github.com/tinkerbell/tink/protos/hardware" - "github.com/tinkerbell/tink/protos/workflow" -) - -// Workflow client for Tinkerbell. -type Workflow struct { - client workflow.WorkflowServiceClient - hardwareClient *Hardware -} - -// NewWorkflowClient returns a Workflow client. -func NewWorkflowClient(client workflow.WorkflowServiceClient, hClient *Hardware) *Workflow { - return &Workflow{client: client, hardwareClient: hClient} -} - -// Get returns a Tinkerbell Workflow. -func (t *Workflow) Get(ctx context.Context, id string) (*workflow.Workflow, error) { - tinkWorkflow, err := t.client.GetWorkflow(ctx, &workflow.GetRequest{Id: id}) - if err != nil { - if err.Error() == sqlErrorString || err.Error() == sqlErrorStringAlt { - return nil, fmt.Errorf("workflow %w", ErrNotFound) - } - - return nil, fmt.Errorf("getting workflow from Tinkerbell: %w", err) - } - - return tinkWorkflow, nil -} - -// GetMetadata returns the metadata for a given Tinkerbell Workflow. -func (t *Workflow) GetMetadata(ctx context.Context, id string) ([]byte, error) { - verReq := &workflow.GetWorkflowDataRequest{WorkflowId: id} - - verResp, err := t.client.GetWorkflowDataVersion(ctx, verReq) - if err != nil { - if err.Error() == sqlErrorString || err.Error() == sqlErrorStringAlt { - return nil, fmt.Errorf("workflow %w", ErrNotFound) - } + tink "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" - return nil, fmt.Errorf("getting workflow version from Tinkerbell: %w", err) - } - - req := &workflow.GetWorkflowDataRequest{WorkflowId: id, Version: verResp.GetVersion()} - - resp, err := t.client.GetWorkflowMetadata(ctx, req) - if err != nil { - if err.Error() == sqlErrorString || err.Error() == sqlErrorStringAlt { - return nil, fmt.Errorf("workflow %w", ErrNotFound) - } - - return nil, fmt.Errorf("getting workflow metadata from Tinkerbell: %w", err) - } + tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) - return resp.GetData(), nil +// WorkflowClient handles interactions with the Tinkerbell Workflows. +type WorkflowClient struct { + tinkclient client.Client } -// GetActions returns the actions for a given Tinkerbell Workflow. -func (t *Workflow) GetActions(ctx context.Context, id string) ([]*workflow.WorkflowAction, error) { - req := &workflow.WorkflowActionsRequest{WorkflowId: id} - - resp, err := t.client.GetWorkflowActions(ctx, req) - if err != nil { - if err.Error() == sqlErrorString || err.Error() == sqlErrorStringAlt { - return nil, fmt.Errorf("workflow %w", ErrNotFound) - } - - return nil, fmt.Errorf("getting workflow actions from Tinkerbell: %w", err) +// NewWorkflowClient creates a new client for managing Tinkerbell workflows. +func NewWorkflowClient(k8sClient client.Client) *WorkflowClient { + return &WorkflowClient{ + tinkclient: k8sClient, } - - return resp.GetActionList(), nil } -// GetEvents returns the events for a given Tinkerbell Workflow. -func (t *Workflow) GetEvents(ctx context.Context, id string) ([]*workflow.WorkflowActionStatus, error) { - req := &workflow.GetRequest{Id: id} - - resp, err := t.client.ShowWorkflowEvents(ctx, req) - if err != nil { - if err.Error() == sqlErrorString || err.Error() == sqlErrorStringAlt { - return nil, fmt.Errorf("workflow %w", ErrNotFound) - } +// CreateWorkflow creates a new Tinkerbell Workflow resource in the cluster. +func (w *WorkflowClient) CreateWorkflow(ctx context.Context, userData, workflowName, templateRef string, hardware tink.Hardware) error { + // Construct the Workflow object + ifaceConfig := hardware.Spec.Interfaces[0].DHCP + dnsNameservers := "1.1.1.1" - return nil, fmt.Errorf("getting workflow events from Tinkerbell: %w", err) + for _, ns := range ifaceConfig.NameServers { + dnsNameservers = ns } - result := []*workflow.WorkflowActionStatus{} - - for { - e, err := resp.Recv() - if errors.Is(err, io.EOF) { - break - } - - if err != nil { - return nil, fmt.Errorf("getting workflow event from Tinkerbell: %w", err) - } - - result = append(result, e) + workflow := &tinkv1alpha1.Workflow{ + ObjectMeta: metav1.ObjectMeta{ + // TODO(MQ): generalize the naming of the workflow and implement a function that can be used across the provider. + Name: workflowName + "-workflow", + Namespace: hardware.Namespace, + }, + Spec: tinkv1alpha1.WorkflowSpec{ + TemplateRef: templateRef, + HardwareRef: hardware.GetName(), + HardwareMap: map[string]string{ + "device_1": hardware.GetMACAddress(), + "dst_path": fmt.Sprintf("/tmp/%s-bootstrap-config", hardware.Name), + "cloud_init_script": base64.StdEncoding.EncodeToString([]byte(userData)), + "interface_name": ifaceConfig.IfaceName, + "cidr": convertNetmaskToCIDR(ifaceConfig.IP), + "ns": dnsNameservers, + "default_route": ifaceConfig.IP.Gateway, + }, + }, } - return result, nil -} - -// GetState returns the state for a given Tinkerbell Workflow. -func (t *Workflow) GetState(ctx context.Context, id string) (workflow.State, error) { - req := &workflow.GetRequest{Id: id} - - resp, err := t.client.GetWorkflowContext(ctx, req) - if err != nil { - if err.Error() == sqlErrorString || err.Error() == sqlErrorStringAlt { - return 0, fmt.Errorf("workflow %w", ErrNotFound) - } - - return 0, fmt.Errorf("getting workflow state from Tinkerbell: %w", err) + // Create the Workflow in the cluster + if err := w.tinkclient.Create(ctx, workflow); err != nil { + return fmt.Errorf("failed to create the workflow: %w", err) } - currIndex := resp.GetCurrentActionIndex() - total := resp.GetTotalNumberOfActions() - currState := resp.GetCurrentActionState() - - switch { - case total == 0: - // If there are no actions, let's just call it pending - return workflow.State_STATE_PENDING, nil - case currIndex+1 == total: - // If we are on the last action, just report it's state - return currState, nil - case currState != workflow.State_STATE_SUCCESS: - // If the state of the last action is anything other than - // success, just report it's state. - return currState, nil - default: - // We are not on the last action, and the last action - // was successful, we should report pending - return workflow.State_STATE_PENDING, nil - } + return nil } -// Create a Tinkerbell Workflow. -func (t *Workflow) Create(ctx context.Context, templateID, hardwareID string) (string, error) { - h, err := t.hardwareClient.Get(ctx, hardwareID, "", "") - if err != nil { - return "", err - } - - hardwareString, err := HardwareToJSON(h) - if err != nil { - return "", err +// DeleteWorkflow deletes an existing Tinkerbell Workflow resource from the cluster. +func (w *WorkflowClient) DeleteWorkflow(ctx context.Context, name string, namespace string) error { + workflow := &tinkv1alpha1.Workflow{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, } - - req := &workflow.CreateRequest{ - Template: templateID, - Hardware: hardwareString, - } - - resp, err := t.client.CreateWorkflow(ctx, req) - if err != nil { - return "", fmt.Errorf("creating workflow in Tinkerbell: %w", err) - } - - return resp.GetId(), nil -} - -// Delete a Tinkerbell Workflow. -func (t *Workflow) Delete(ctx context.Context, id string) error { - if _, err := t.client.DeleteWorkflow(ctx, &workflow.GetRequest{Id: id}); err != nil { - if err.Error() == sqlErrorString || err.Error() == sqlErrorStringAlt { - return fmt.Errorf("workflow %w", ErrNotFound) + if err := w.tinkclient.Delete(ctx, workflow); err != nil { + if !kerrors.IsNotFound(err) { + return fmt.Errorf("failed to delete workflow: %w", err) } - - return fmt.Errorf("deleting workflow from Tinkerbell: %w", err) } return nil } -// HardwareToJSON converts Hardware to a string suitable for use in a -// Workflow Request for the raw Tinkerbell client. -func HardwareToJSON(h *hardware.Hardware) (string, error) { - hardwareInterfaces := h.GetNetwork().GetInterfaces() - hardwareInfo := make(map[string]string, len(hardwareInterfaces)) - - for i, hi := range hardwareInterfaces { - if mac := hi.GetDhcp().GetMac(); mac != "" { - hardwareInfo[fmt.Sprintf("device_%d", i+1)] = mac - } - } - - hardwareJSON, err := json.Marshal(hardwareInfo) - if err != nil { - return "", fmt.Errorf("marshaling hardware info into json: %w", err) +// GetWorkflow retrieves a Tinkerbell Workflow resource from the cluster. +func (w *WorkflowClient) GetWorkflow(ctx context.Context, name string, namespace string) (*tinkv1alpha1.Workflow, error) { + workflow := &tinkv1alpha1.Workflow{} + if err := w.tinkclient.Get(ctx, client.ObjectKey{Name: name, Namespace: namespace}, workflow); err != nil { + return nil, fmt.Errorf("failed to get workflow: %w", err) } - - return string(hardwareJSON), nil + return workflow, nil } diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go index 8c08ff8c4..787cad8a0 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go @@ -18,205 +18,188 @@ package tinkerbell import ( "context" - "encoding/json" - "errors" + "encoding/base64" "fmt" + "github.com/aws/smithy-go/ptr" + cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" + "go.uber.org/zap" - tinkclient "github.com/tinkerbell/tink/client" - tinkpkg "github.com/tinkerbell/tink/pkg" - "github.com/tinkerbell/tink/protos/hardware" - tinktmpl "github.com/tinkerbell/tink/protos/template" - "gopkg.in/yaml.v3" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins" - tinkerbellclient "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client" - metadataclient "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - + "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client" + tinktypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" + tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/kubectl/pkg/scheme" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) -type ClientFactory func() (metadataclient.Client, tinkerbellclient.HardwareClient, tinkerbellclient.TemplateClient, tinkerbellclient.WorkflowClient) - type driver struct { - TinkServerAddress string - ImageRepoAddress string + ClusterName string + OSImageURL string + + HardwareRef types.NamespacedName - metadataClient metadataclient.Client - hardwareClient tinkerbellclient.HardwareClient - templateClient tinkerbellclient.TemplateClient - workflowClient tinkerbellclient.WorkflowClient + TinkClient ctrlruntimeclient.Client + HardwareClient client.HardwareClient + WorkflowClient client.WorkflowClient + TemplateClient client.TemplateClient +} + +func init() { + // Ensure the Tinkerbell API types are registered with the global scheme. + if err := tinkv1alpha1.SchemeBuilder.AddToScheme(scheme.Scheme); err != nil { + panic(fmt.Sprintf("failed to add tinkv1alpha1 to scheme: %v", err)) + } } // NewTinkerbellDriver returns a new TinkerBell driver with a configured tinkserver address and a client timeout. -func NewTinkerbellDriver(mdConfig *metadataclient.Config, factory ClientFactory, tinkServerAddress, imageRepoAddress string) (plugins.PluginDriver, error) { - if tinkServerAddress == "" || imageRepoAddress == "" { - return nil, errors.New("tink-server address, ImageRepoAddress cannot be empty") +func NewTinkerbellDriver(tinkConfig tinktypes.Config, tinkSpec *tinktypes.TinkerbellPluginSpec) (plugins.PluginDriver, error) { + tinkClient, err := ctrlruntimeclient.New(tinkConfig.RestConfig, ctrlruntimeclient.Options{Scheme: scheme.Scheme}) + if err != nil { + return nil, fmt.Errorf("failed to create k8s client: %w", err) } + hwClient := client.NewHardwareClient(tinkClient) - var ( - mdClient metadataclient.Client - hwClient tinkerbellclient.HardwareClient - tmplClient tinkerbellclient.TemplateClient - wflClient tinkerbellclient.WorkflowClient - err error - ) + wkClient := client.NewWorkflowClient(tinkClient) - if factory == nil { - mdClient, err = metadataclient.NewMetadataClient(mdConfig) - if err != nil { - return nil, fmt.Errorf("failed to create metadata client: %w", err) - } + tmplClient := client.NewTemplateClient(tinkClient) - if err := tinkclient.Setup(); err != nil { - return nil, fmt.Errorf("failed to setup tink-server client: %w", err) - } + d := driver{ + ClusterName: tinkSpec.ClusterName.Value, + TinkClient: tinkClient, + HardwareRef: tinkSpec.HardwareRef, + HardwareClient: *hwClient, + WorkflowClient: *wkClient, + TemplateClient: *tmplClient, + OSImageURL: tinkSpec.OSImageURL.Value, + } - hwClient = tinkerbellclient.NewHardwareClient(tinkclient.HardwareClient) - tmplClient = tinkerbellclient.NewTemplateClient(tinkclient.TemplateClient) - wflClient = tinkerbellclient.NewWorkflowClient(tinkclient.WorkflowClient, tinkerbellclient.NewHardwareClient(tinkclient.HardwareClient)) - } else { - mdClient, hwClient, tmplClient, wflClient = factory() + return &d, nil +} + +func (d *driver) GetServer(ctx context.Context) (plugins.Server, error) { + targetHardware, err := d.HardwareClient.GetHardware(ctx, d.HardwareRef) + if err != nil { + return nil, err } - d := &driver{ - TinkServerAddress: tinkServerAddress, - ImageRepoAddress: imageRepoAddress, - metadataClient: mdClient, - hardwareClient: hwClient, - templateClient: tmplClient, - workflowClient: wflClient, + if targetHardware.Spec.Metadata != nil && targetHardware.Spec.Metadata.State != tinktypes.Staged { + return nil, cloudprovidererrors.ErrInstanceNotFound } - return d, nil + server := tinktypes.Hardware{Hardware: targetHardware} + return &server, nil } -func (d *driver) GetServer(ctx context.Context, uid types.UID, hwSpec runtime.RawExtension) (plugins.Server, error) { - hw := HardwareSpec{} - if err := json.Unmarshal(hwSpec.Raw, &hw); err != nil { - return nil, fmt.Errorf("failed to unmarshal tinkerbell hardware spec: %w", err) +func (d *driver) ProvisionServer(ctx context.Context, _ *zap.SugaredLogger, meta metav1.ObjectMeta, _ runtime.RawExtension, userdata string) (plugins.Server, error) { + // Get the hardware object from tinkerbell + hardware, err := d.HardwareClient.GetHardware(ctx, d.HardwareRef) + if err != nil { + return nil, err } - fetchedHW, err := d.hardwareClient.Get(ctx, string(uid), hw.GetIPAddress(), - hw.GetMACAddress()) - if err != nil { - if resourceNotFoundErr(err) { - return nil, cloudprovidererrors.ErrInstanceNotFound + var allowProvision bool + for _, iface := range hardware.Spec.Interfaces { + if iface.Netboot != nil && iface.Netboot.AllowPXE != nil && iface.Netboot.AllowPXE == ptr.Bool(false) { + continue } - return nil, fmt.Errorf("failed to get hardware: %w", err) - } + if iface.Netboot != nil && iface.Netboot.AllowWorkflow != nil && iface.Netboot.AllowWorkflow == ptr.Bool(false) { + continue + } - return &HardwareSpec{ - Hardware: tinkpkg.HardwareWrapper{ - Hardware: fetchedHW, - }, - }, nil -} + allowProvision = true + } -func (d *driver) ProvisionServer(ctx context.Context, uid types.UID, cfg *plugins.CloudConfigSettings, hwSpec runtime.RawExtension) (plugins.Server, error) { - hw := HardwareSpec{} - if err := json.Unmarshal(hwSpec.Raw, &hw); err != nil { - return nil, fmt.Errorf("failed to unmarshal tinkerbell hardware spec: %w", err) + if !allowProvision { + return nil, fmt.Errorf("server %s is not allowed to be provisioned; either hardware allowPXE or allowWorkflow is set to false", hardware.Name) } - hw.Hardware.Id = string(uid) - _, err := d.hardwareClient.Get(ctx, hw.Hardware.Id, "", "") + + // Create template if it doesn't exist + err = d.TemplateClient.CreateTemplate(ctx, hardware, d.HardwareRef.Namespace, d.OSImageURL) if err != nil { - if resourceNotFoundErr(err) { - cfg, err := d.metadataClient.GetMachineMetadata() - if err != nil { - return nil, fmt.Errorf("failed to get metadata configs: %w", err) - } - - hw.Hardware.Network.Interfaces[0].Dhcp.Mac = cfg.MACAddress - - ip, netmask, _, err := util.CIDRToIPAndNetMask(cfg.CIDR) - if err != nil { - return nil, fmt.Errorf("failed to parse CIDR: %w", err) - } - dhcpIP := &hardware.Hardware_DHCP_IP{ - Address: ip, - Netmask: netmask, - Gateway: cfg.Gateway, - } - hw.Hardware.Network.Interfaces[0].Dhcp.Ip = dhcpIP - - if err := d.hardwareClient.Create(ctx, hw.Hardware.Hardware); err != nil { - return nil, fmt.Errorf("failed to register hardware to tink-server: %w", err) - } - } + return nil, err } - // cfg.SecretName has the same name as the machine name - workflowTemplate, err := d.templateClient.Get(ctx, "", cfg.SecretName) - if err != nil { - if resourceNotFoundErr(err) { - tmpl := createTemplate(d.TinkServerAddress, d.ImageRepoAddress, cfg) - payload, err := yaml.Marshal(tmpl) - if err != nil { - return nil, fmt.Errorf("failed marshalling workflow template: %w", err) - } - - workflowTemplate = &tinktmpl.WorkflowTemplate{ - Name: tmpl.Name, - Id: tmpl.ID, - Data: string(payload), - } - - if err := d.templateClient.Create(ctx, workflowTemplate); err != nil { - return nil, fmt.Errorf("failed to create workflow template: %w", err) - } - } + // Create Workflow to match the template and server + server := tinktypes.Hardware{Hardware: hardware} + if err = d.WorkflowClient.CreateWorkflow(ctx, userdata, server.Name, client.ProvisionWorkerNodeTemplate, server); err != nil { + return nil, err } - if _, err := d.workflowClient.Create(ctx, workflowTemplate.Id, hw.GetID()); err != nil { - return nil, fmt.Errorf("failed to provision server id %s running template id %s: %w", workflowTemplate.Id, hw.GetID(), err) + // Set the HardwareID with machine UID. The hardware object is claimed by the machine. + if err = d.HardwareClient.SetHardwareID(ctx, hardware, string(meta.UID)); err != nil { + return nil, err } - return &hw, nil + return &server, nil } -func (d *driver) Validate(hwSpec runtime.RawExtension) error { - hw := HardwareSpec{} - if err := json.Unmarshal(hwSpec.Raw, &hw); err != nil { - return fmt.Errorf("failed to unmarshal tinkerbell hardware spec: %w", err) - } +func (d *driver) Validate(_ runtime.RawExtension) error { + return nil +} - if hw.Hardware.Hardware == nil { - return fmt.Errorf("tinkerbell hardware data can not be empty") +func (d *driver) DeprovisionServer(ctx context.Context) error { + // Get the hardware object from tinkerbell cluster + targetHardware, err := d.HardwareClient.GetHardware(ctx, d.HardwareRef) + if err != nil { + return err } - if hw.Hardware.Network == nil { - return fmt.Errorf("tinkerbell hardware network configs can not be empty") + // Delete the associated Workflow. + workflowName := targetHardware.Name + "-workflow" // Assuming workflow names are derived from hardware names + if err := d.WorkflowClient.DeleteWorkflow(ctx, workflowName, targetHardware.Namespace); err != nil { + return fmt.Errorf("failed to delete workflow %s: %w", workflowName, err) } - if hw.Hardware.Metadata == "" { - return fmt.Errorf("tinkerbell hardware metadata can not be empty") + // Reset the hardware ID and state in the tinkerbell cluster. + if err := d.HardwareClient.SetHardwareID(ctx, targetHardware, ""); err != nil { + return fmt.Errorf("failed to reset hardware ID for %s: %w", targetHardware.Name, err) } return nil } -func (d *driver) DeprovisionServer(ctx context.Context, uid types.UID) error { - if err := d.hardwareClient.Delete(ctx, string(uid)); err != nil { - if resourceNotFoundErr(err) { - return nil +func GetConfig(driverConfig tinktypes.TinkerbellPluginSpec, valueFromStringOrEnvVar func(configVar providerconfigtypes.ConfigVarString, envVarName string) (string, error)) (*tinktypes.Config, error) { + config := tinktypes.Config{} + var err error + // Kubeconfig was specified directly in the Machine/MachineDeployment CR. In this case we need to ensure that the value is base64 encoded. + if driverConfig.Auth.Kubeconfig.Value != "" { + val, err := base64.StdEncoding.DecodeString(driverConfig.Auth.Kubeconfig.Value) + if err != nil { + // An error here means that this is not a valid base64 string + // We can be more explicit here with the error for visibility. Webhook will return this error if we hit this scenario. + return nil, fmt.Errorf("failed to decode base64 encoded kubeconfig. Expected value is a base64 encoded Kubeconfig in JSON or YAML format: %w", err) + } + config.Kubeconfig = string(val) + } else { + // Environment variable or secret reference was used for providing the value of kubeconfig + // We have to be lenient in this case and allow unencoded values as well. + // TODO(mq): Replace this field with a reference to a secret instead of having it inlined. + config.Kubeconfig, err = valueFromStringOrEnvVar(driverConfig.Auth.Kubeconfig, "TINK_KUBECONFIG") + if err != nil { + return nil, fmt.Errorf(`failed to get value of "kubeconfig" field: %w`, err) } - return fmt.Errorf("failed to delete tinkerbell hardware data: %w", err) + } + config.ClusterName, err = valueFromStringOrEnvVar(driverConfig.ClusterName, "CLUSTER_NAME") + if err != nil { + return nil, fmt.Errorf(`failed to get value of "clusterName" field: %w`, err) } - return nil -} - -func resourceNotFoundErr(err error) bool { - switch err.Error() { - case fmt.Sprintf("hardware %s", tinkerbellclient.ErrNotFound.Error()): - return true - case fmt.Sprintf("template %s", tinkerbellclient.ErrNotFound.Error()): - return true + config.OSImageURL, err = valueFromStringOrEnvVar(driverConfig.OSImageURL, "OS_IMAGE_URL") + if err != nil { + return nil, fmt.Errorf(`failed to get value of "OSImageURL" field: %w`, err) } - return false + config.RestConfig, err = clientcmd.RESTConfigFromKubeConfig([]byte(config.Kubeconfig)) + if err != nil { + return nil, fmt.Errorf("failed to decode kubeconfig: %w", err) + } + return &config, nil } diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver_test.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver_test.go deleted file mode 100644 index 229dd6817..000000000 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver_test.go +++ /dev/null @@ -1,346 +0,0 @@ -/* -Copyright 2021 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tinkerbell - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "reflect" - "testing" - - "github.com/tinkerbell/tink/protos/hardware" - "github.com/tinkerbell/tink/protos/template" - "github.com/tinkerbell/tink/workflow" - - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins" - tinkerbellclient "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata" - - "k8s.io/apimachinery/pkg/runtime" -) - -func TestNewTinkerbellDriver(t *testing.T) { - var testCases = []struct { - name string - tinkServer string - imageRepoServer string - clientFactor ClientFactory - errorIsExpected bool - }{ - { - name: "create new tinkerbell driver failure, missing image repo server", - tinkServer: "10.129.8.102", - imageRepoServer: "", - errorIsExpected: true, - }, - { - name: "create new tinkerbell driver failure, missing tink server", - tinkServer: "", - imageRepoServer: "10.129.8.102:8080", - errorIsExpected: true, - }, - { - name: "create new tinkerbell driver success", - tinkServer: "10.129.8.102", - imageRepoServer: "10.129.8.102:8080", - clientFactor: func() (metadata.Client, tinkerbellclient.HardwareClient, tinkerbellclient.TemplateClient, tinkerbellclient.WorkflowClient) { - return &fakeMetadataClient{}, &fakeHardwareClient{}, &fakeTemplateClient{}, &fakeWorkflowClient{} - }, - errorIsExpected: false, - }, - } - - for _, test := range testCases { - t.Run(test.name, func(t *testing.T) { - _, err := NewTinkerbellDriver(nil, test.clientFactor, test.tinkServer, test.imageRepoServer) - if err != nil { - if test.errorIsExpected { - return - } - - t.Fatalf("failed to create tinkerbell client: %v", err) - } - }) - } -} - -func TestDriver_GetServer(t *testing.T) { - var testCases = []struct { - name string - tinkServer string - imageRepoServer string - hardwareSpec runtime.RawExtension - clientFactor ClientFactory - expectedHardwareSpec string - errorIsExpected bool - expectedError error - }{ - { - name: "failed to get server", - tinkServer: "10.129.8.102", - imageRepoServer: "10.129.8.102:8080", - hardwareSpec: runtime.RawExtension{Raw: []byte("{\n \"hardware\": {\n \"network\": {\n \"interfaces\": [\n {\n \"dhcp\": {\n \"ip\": {\n \"address\": \"10.129.8.90\"\n },\n \"mac\": \"18:C0:4D:B1:18:E3\"\n }\n }\n ]\n }\n }\n}")}, - clientFactor: func() (metadata.Client, tinkerbellclient.HardwareClient, tinkerbellclient.TemplateClient, tinkerbellclient.WorkflowClient) { - return &fakeMetadataClient{}, &fakeHardwareClient{ - err: &resourceError{ - resource: "hardware", - }, - }, &fakeTemplateClient{}, &fakeWorkflowClient{} - }, - errorIsExpected: true, - expectedError: cloudprovidererrors.ErrInstanceNotFound, - }, - { - name: "get server success", - tinkServer: "10.129.8.102", - imageRepoServer: "10.129.8.102:8080", - hardwareSpec: runtime.RawExtension{Raw: []byte("{\n \"hardware\": {\n \"network\": {\n \"interfaces\": [\n {\n \"dhcp\": {\n \"ip\": {\n \"address\": \"10.129.8.90\"\n },\n \"mac\": \"18:C0:4D:B1:18:E3\"\n }\n }\n ]\n }\n }\n}")}, - clientFactor: func() (metadata.Client, tinkerbellclient.HardwareClient, tinkerbellclient.TemplateClient, tinkerbellclient.WorkflowClient) { - return &fakeMetadataClient{}, &fakeHardwareClient{}, &fakeTemplateClient{}, &fakeWorkflowClient{} - }, - errorIsExpected: false, - expectedHardwareSpec: "{\n \"hardware\": {\n \"metadata\": {\n \"facility\": {\n \"facility_code\": \"ewr1\",\n \"plan_slug\": \"c2.medium.x86\",\n \"plan_version_slug\": \"\"\n },\n \"instance\": {\n \"operating_system_version\": {\n \"distro\": \"ubuntu\",\n \"os_slug\": \"ubuntu_18_04\",\n \"version\": \"18.04\"\n }\n },\n \"state\": \"\"\n },\n \"network\": {\n \"interfaces\": [\n {\n \"dhcp\": {\n \"arch\": \"x86_64\",\n \"ip\": {\n \"address\": \"10.129.8.90\",\n \"gateway\": \"10.129.8.89\",\n \"netmask\": \"255.255.255.252\"\n },\n \"mac\": \"18:C0:4D:B1:18:E3\",\n \"uefi\": false\n },\n \"netboot\": {\n \"allow_pxe\": true,\n \"allow_workflow\": true\n }\n }\n ]\n }\n }\n}", - }, - } - - for _, test := range testCases { - t.Run(test.name, func(t *testing.T) { - d, err := NewTinkerbellDriver(nil, test.clientFactor, test.tinkServer, test.imageRepoServer) - if err != nil { - t.Fatalf("failed to create tinkerbell driver: %v", err) - } - - ctx := context.Background() - s, err := d.GetServer(ctx, "0eba0bf8-3772-4b4a-ab9f-6ebe93b90a94", test.hardwareSpec) - if err != nil { - if test.errorIsExpected && errors.Is(err, test.expectedError) { - return - } - - t.Fatalf("failed to execute get server: %v", err) - } - - hw := &HardwareSpec{} - if err := json.Unmarshal([]byte(test.expectedHardwareSpec), hw); err != nil { - t.Fatal(err) - } - - if !reflect.DeepEqual(hw, s) { - t.Fatal("server spec and hardware spec mismatched") - } - }) - } -} - -func TestDriver_ProvisionServer(t *testing.T) { - var testCases = []struct { - name string - tinkServer string - imageRepoServer string - hardwareSpec runtime.RawExtension - clientFactory ClientFactory - cloudConfig *plugins.CloudConfigSettings - expectedHardwareSpec string - errorIsExpected bool - expectedError error - }{ - { - name: "provision server success", - tinkServer: "10.129.8.102", - imageRepoServer: "10.129.8.102:8080", - hardwareSpec: runtime.RawExtension{Raw: []byte("{\n \"hardware\": {\n \"metadata\": {\n \"facility\": {\n \"facility_code\": \"ewr1\",\n \"plan_slug\": \"c2.medium.x86\",\n \"plan_version_slug\": \"\"\n },\n \"instance\": {\n \"operating_system_version\": {\n \"distro\": \"ubuntu\",\n \"os_slug\": \"ubuntu_18_04\",\n \"version\": \"18.04\"\n }\n },\n \"state\": \"\"\n },\n \"network\": {\n \"interfaces\": [\n {\n \"dhcp\": {\n \"arch\": \"x86_64\",\n \"ip\": {\n \"address\": \"10.129.8.90\",\n \"gateway\": \"10.129.8.89\",\n \"netmask\": \"255.255.255.252\"\n },\n \"mac\": \"18:C0:4D:B1:18:E3\",\n \"uefi\": false\n },\n \"netboot\": {\n \"allow_pxe\": true,\n \"allow_workflow\": true\n }\n }\n ]\n }\n }\n}")}, - clientFactory: func() (metadata.Client, tinkerbellclient.HardwareClient, tinkerbellclient.TemplateClient, tinkerbellclient.WorkflowClient) { - return &fakeMetadataClient{}, &fakeHardwareClient{ - err: &resourceError{ - resource: "hardware", - }, - }, &fakeTemplateClient{}, &fakeWorkflowClient{} - }, - cloudConfig: &plugins.CloudConfigSettings{ - Token: "test-token", - Namespace: "kube-system", - SecretName: "test-secret", - ClusterHost: "10.10.10.10", - }, - expectedHardwareSpec: "{\n \"hardware\": {\n \"id\": \"0eba0bf8-3772-4b4a-ab9f-6ebe93b90a94\",\n \"metadata\": {\n \"facility\": {\n \"facility_code\": \"ewr1\",\n \"plan_slug\": \"c2.medium.x86\",\n \"plan_version_slug\": \"\"\n },\n \"instance\": {\n \"operating_system_version\": {\n \"distro\": \"ubuntu\",\n \"os_slug\": \"ubuntu_18_04\",\n \"version\": \"18.04\"\n }\n },\n \"state\": \"\"\n },\n \"network\": {\n \"interfaces\": [\n {\n \"dhcp\": {\n \"arch\": \"x86_64\",\n \"ip\": {\n \"address\": \"10.129.8.90\",\n \"gateway\": \"10.129.8.89\",\n \"netmask\": \"255.255.255.252\"\n },\n \"mac\": \"18:C0:4D:B1:18:E3\",\n \"uefi\": false\n },\n \"netboot\": {\n \"allow_pxe\": true,\n \"allow_workflow\": true\n }\n }\n ]\n }\n }\n}", - errorIsExpected: false, - }, - } - - for _, test := range testCases { - t.Run(test.name, func(t *testing.T) { - d, err := NewTinkerbellDriver(nil, test.clientFactory, test.tinkServer, test.imageRepoServer) - if err != nil { - t.Fatalf("failed to create tinkerbell driver: %v", err) - } - - ctx := context.Background() - s, err := d.ProvisionServer(ctx, "0eba0bf8-3772-4b4a-ab9f-6ebe93b90a94", test.cloudConfig, test.hardwareSpec) - if err != nil { - t.Fatalf("failed to execute provision server: %v", err) - } - - hw := &HardwareSpec{} - if err := json.Unmarshal([]byte(test.expectedHardwareSpec), hw); err != nil { - t.Fatal(err) - } - - if !reflect.DeepEqual(hw, s) { - t.Fatal("server spec and hardware spec mismatched") - } - }) - } -} - -type fakeMetadataClient struct{} - -func (f *fakeMetadataClient) GetMachineMetadata() (*metadata.MachineMetadata, error) { - return &metadata.MachineMetadata{ - CIDR: "10.129.8.90/30", - MACAddress: "18:C0:4D:B1:18:E3", - Gateway: "10.129.8.89", - }, nil -} - -type fakeHardwareClient struct { - err *resourceError -} - -func (f *fakeHardwareClient) Get(_ context.Context, _ string, _ string, _ string) (*hardware.Hardware, error) { - if f.err != nil { - return nil, f.err - } - - return &hardware.Hardware{ - Metadata: "{\"facility\":{\"facility_code\":\"ewr1\",\"plan_slug\":\"c2.medium.x86\",\"plan_version_slug\":\"\"},\"instance\":{\"operating_system_version\":{\"distro\":\"ubuntu\",\"os_slug\":\"ubuntu_18_04\",\"version\":\"18.04\"}},\"state\":\"\"}", - Network: &hardware.Hardware_Network{ - Interfaces: []*hardware.Hardware_Network_Interface{ - { - Dhcp: &hardware.Hardware_DHCP{ - Arch: "x86_64", - Uefi: false, - Mac: "18:C0:4D:B1:18:E3", - Ip: &hardware.Hardware_DHCP_IP{ - Address: "10.129.8.90", - Netmask: "255.255.255.252", - Gateway: "10.129.8.89", - }, - }, - Netboot: &hardware.Hardware_Netboot{ - AllowPxe: true, - AllowWorkflow: true, - }, - }, - }, - }, - }, nil -} - -func (f *fakeHardwareClient) Delete(_ context.Context, _ string) error { - return nil -} - -func (f *fakeHardwareClient) Create(_ context.Context, hw *hardware.Hardware) error { - expectedHW := &hardware.Hardware{ - Id: "0eba0bf8-3772-4b4a-ab9f-6ebe93b90a94", - Metadata: "{\"facility\":{\"facility_code\":\"ewr1\",\"plan_slug\":\"c2.medium.x86\",\"plan_version_slug\":\"\"},\"instance\":{\"operating_system_version\":{\"distro\":\"ubuntu\",\"os_slug\":\"ubuntu_18_04\",\"version\":\"18.04\"}},\"state\":\"\"}", - Network: &hardware.Hardware_Network{ - Interfaces: []*hardware.Hardware_Network_Interface{ - { - Dhcp: &hardware.Hardware_DHCP{ - Arch: "x86_64", - Uefi: false, - Mac: "18:C0:4D:B1:18:E3", - Ip: &hardware.Hardware_DHCP_IP{ - Address: "10.129.8.90", - Netmask: "255.255.255.252", - Gateway: "10.129.8.89", - }, - }, - Netboot: &hardware.Hardware_Netboot{ - AllowPxe: true, - AllowWorkflow: true, - }, - }, - }, - }, - } - - if !reflect.DeepEqual(hw, expectedHW) { - return errors.New("unexpected hardware data") - } - - return nil -} - -type fakeTemplateClient struct{} - -func (f *fakeTemplateClient) Get(_ context.Context, _ string, _ string) (*template.WorkflowTemplate, error) { - wfl := &workflow.Workflow{ - Version: "0.1", - Name: "fake_template", - GlobalTimeout: 6000, - Tasks: []workflow.Task{ - { - Name: "disk-wipe", - WorkerAddr: "{{.device_1}}", - Volumes: []string{ - "/dev:/dev", - "/dev/console:/dev/console", - "/lib/firmware:/lib/firmware:ro", - }, - Actions: []workflow.Action{ - { - Name: "disk-wipe", - Image: "disk-wipe:v1", - Timeout: 90, - }, - }, - }, - }, - } - - payload, err := json.Marshal(wfl) - if err != nil { - return nil, err - } - - return &template.WorkflowTemplate{ - Data: string(payload), - }, nil -} - -func (f *fakeTemplateClient) Create(_ context.Context, _ *template.WorkflowTemplate) error { - return nil -} - -type fakeWorkflowClient struct{} - -func (f *fakeWorkflowClient) Create(_ context.Context, _ string, _ string) (string, error) { - return "", nil -} - -type resourceError struct { - resource string -} - -func (re *resourceError) Error() string { - return fmt.Sprintf("%s %s", re.resource, tinkerbellclient.ErrNotFound.Error()) -} diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/hardware.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/hardware.go deleted file mode 100644 index 22ecd02a7..000000000 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/hardware.go +++ /dev/null @@ -1,64 +0,0 @@ -/* -Copyright 2021 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tinkerbell - -import ( - "encoding/json" - - "github.com/tinkerbell/tink/pkg" -) - -type HardwareSpec struct { - Hardware pkg.HardwareWrapper `json:"hardware"` -} - -func (h *HardwareSpec) GetName() string { - return "" -} - -func (h *HardwareSpec) GetID() string { - return h.Hardware.Id -} - -func (h *HardwareSpec) GetIPAddress() string { - interfaces := h.Hardware.Network.Interfaces - if len(interfaces) > 0 && interfaces[0].Dhcp.Ip != nil { - return h.Hardware.Network.Interfaces[0].Dhcp.Ip.Address - } - - return "" -} - -func (h *HardwareSpec) GetMACAddress() string { - if len(h.Hardware.Network.Interfaces) > 0 { - return h.Hardware.Network.Interfaces[0].Dhcp.Mac - } - - return "" -} - -func (h *HardwareSpec) GetStatus() string { - metadata := struct { - State string `json:"state"` - }{} - - if err := json.Unmarshal([]byte(h.Hardware.Metadata), &metadata); err != nil { - return "" - } - - return metadata.State -} diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata/client.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata/client.go deleted file mode 100644 index 29ce3d9c8..000000000 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata/client.go +++ /dev/null @@ -1,123 +0,0 @@ -/* -Copyright 2021 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package metadata - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "time" -) - -type MachineMetadata struct { - CIDR string `json:"cidr,omitempty"` - MACAddress string `json:"mac_address,omitempty"` - Gateway string `json:"gateway,omitempty"` - Status string `json:"status,omitempty"` -} - -type Config struct { - Endpoint string `json:"endpoint,omitempty"` - AuthConfig *AuthConfig `json:"authConfig,omitempty"` -} - -type AuthMethod string - -const ( - BasicAuth AuthMethod = "BasicAuth" - BearerToken AuthMethod = "BearerToken" - - defaultTimeout = 30 * time.Second -) - -type AuthConfig struct { - AuthMethod AuthMethod `json:"authMethod"` - Username string `json:"username"` - Password string `json:"password"` - Token string `json:"token"` -} - -type Client interface { - GetMachineMetadata() (*MachineMetadata, error) -} - -type defaultClient struct { - metadataEndpoint string - authConfig *AuthConfig - client *http.Client -} - -func NewMetadataClient(cfg *Config) (Client, error) { - if cfg.Endpoint == "" { - return nil, errors.New("machine metadata endpoint cannot be empty") - } - - client := http.DefaultClient - client.Timeout = defaultTimeout - - return &defaultClient{ - metadataEndpoint: cfg.Endpoint, - authConfig: cfg.AuthConfig, - client: client, - }, nil -} - -func (d *defaultClient) GetMachineMetadata() (*MachineMetadata, error) { - req, err := http.NewRequest(http.MethodGet, d.metadataEndpoint, nil) - // TODO: Fix this - req = req.WithContext(context.TODO()) - if err != nil { - return nil, fmt.Errorf("failed to create a get metadata request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - d.getAuthMethod(req) - - res, err := d.client.Do(req) - if err != nil { - return nil, fmt.Errorf("failed to execute get metadata request: %w", err) - } - - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to execute get metadata request with status code: %v", res.StatusCode) - } - data, err := io.ReadAll(res.Body) - if err != nil { - return nil, fmt.Errorf("failed to read response body: %w", err) - } - - mdConfig := &MachineMetadata{} - if err := json.Unmarshal(data, mdConfig); err != nil { - return nil, fmt.Errorf("failed to unmarshal metadata config: %w", err) - } - - return mdConfig, nil -} - -func (d *defaultClient) getAuthMethod(req *http.Request) { - switch d.authConfig.AuthMethod { - case BasicAuth: - req.SetBasicAuth(d.authConfig.Username, d.authConfig.Password) - case BearerToken: - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", d.authConfig.Token)) - } -} diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/template.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/template.go deleted file mode 100644 index 5d0902d26..000000000 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/template.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2021 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package tinkerbell - -import ( - "github.com/tinkerbell/tink/workflow" - - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins" -) - -func createTemplate(tinkServerAddress, imageRepoAddress string, cfg *plugins.CloudConfigSettings) *workflow.Workflow { - return &workflow.Workflow{ - Version: "0.1", - Name: cfg.SecretName, - ID: "", - GlobalTimeout: 6000, - Tasks: []workflow.Task{ - { - Name: "os-installation", - WorkerAddr: "{{.device_1}}", - Volumes: []string{ - "/dev:/dev", - "/dev/console:/dev/console", - "/lib/firmware:/lib/firmware:ro", - }, - Actions: []workflow.Action{ - { - Name: "disk-wipe", - Image: "disk-wipe:v1", - Timeout: 90, - }, - { - Name: "disk-partition", - Image: "disk-partition:v1", - Timeout: 180, - Environment: map[string]string{ - "MIRROR_HOST": tinkServerAddress, - }, - Volumes: []string{ - "/statedir:/statedir", - }, - }, - { - Name: "install-root-fs", - Image: "install-root-fs:v1", - Timeout: 600, - Environment: map[string]string{ - "MIRROR_HOST": imageRepoAddress, - }, - Volumes: nil, - }, - { - Name: "install-grub", - Image: "install-grub:v1", - Timeout: 600, - Environment: map[string]string{ - "MIRROR_HOST": imageRepoAddress, - }, - Volumes: []string{ - "/statedir:/statedir", - }, - }, - { - Name: "cloud-init", - Image: "cloud-init:v1", - Timeout: 600, - Environment: map[string]string{ - "MIRROR_HOST": imageRepoAddress, - "CLOUD_INIT_TOKEN": cfg.Token, - "CLOUD_INIT_SETTINGS_NAMESPACE": cfg.Namespace, - "SECRET_NAME": cfg.SecretName, - "CLUSTER_HOST": cfg.ClusterHost, - }, - }, - }, - }, - }, - } -} diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/hardware.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/hardware.go new file mode 100644 index 000000000..4e124b279 --- /dev/null +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/hardware.go @@ -0,0 +1,68 @@ +/* +Copyright 2024 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins" + tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" +) + +type Hardware struct { + *tinkv1alpha1.Hardware `json:"hardware"` +} + +var _ plugins.Server = &Hardware{} + +func (h *Hardware) GetName() string { + return h.Name +} + +func (h *Hardware) GetID() string { + if h.Spec.Metadata != nil && + h.Spec.Metadata.Instance != nil { + return h.Spec.Metadata.Instance.ID + } + + return "" +} + +func (h *Hardware) GetIPAddress() string { + if h.Spec.Metadata != nil && h.Spec.Metadata.State == Staged { + interfaces := h.Spec.Interfaces + if len(interfaces) > 0 && interfaces[0].DHCP.IP != nil { + return interfaces[0].DHCP.IP.Address + } + } + + return "" +} + +func (h *Hardware) GetMACAddress() string { + if len(h.Spec.Interfaces) > 0 { + return h.Spec.Interfaces[0].DHCP.MAC + } + + return "" +} + +func (h *Hardware) GetStatus() string { + if h.Status.State != "" { + return string(h.Status.State) + } + + return Unknown +} diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/types.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/types.go new file mode 100644 index 000000000..ba0ecce58 --- /dev/null +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/types.go @@ -0,0 +1,63 @@ +/* +Copyright 2024 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" +) + +const ( + Unknown string = "Unknown" + Staged string = "Staged" + Provisioned string = "Provisioned" + Decommissioned string = "Decommissioned" +) + +// TinkerbellPluginSpec defines the required information for the Tinkerbell plugin. +type TinkerbellPluginSpec struct { + + // ClusterName specifies the name of the Tinkerbell cluster. This is used to identify + // the cluster within a larger infrastructure or across multiple clusters. + ClusterName providerconfigtypes.ConfigVarString `json:"clusterName"` + + // Auth contains the kubeconfig credentials needed to authenticate against the + // Tinkerbell cluster API. This field is optional and should be provided if authentication is required. + Auth Auth `json:"auth,omitempty"` + + // OSImageURL is the URL where the OS image for the Tinkerbell template is located. + // This URL is used to download and stream the OS image during the provisioning process. + OSImageURL providerconfigtypes.ConfigVarString `json:"osImageUrl"` + + // HardwareRef specifies the unique identifier of a single hardware object in the user-cluster + // that corresponds to the machine deployment. This ensures a one-to-one mapping between a deployment + // and a hardware object in the Tinkerbell cluster. + HardwareRef types.NamespacedName `json:"hardwareRef"` +} + +// Auth. +type Auth struct { + Kubeconfig providerconfigtypes.ConfigVarString `json:"kubeconfig,omitempty"` +} + +type Config struct { + Kubeconfig string + ClusterName string + RestConfig *rest.Config + OSImageURL string +} diff --git a/pkg/cloudprovider/provider/baremetal/provider.go b/pkg/cloudprovider/provider/baremetal/provider.go index 55afc2c15..0e7cba678 100644 --- a/pkg/cloudprovider/provider/baremetal/provider.go +++ b/pkg/cloudprovider/provider/baremetal/provider.go @@ -29,8 +29,8 @@ import ( cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata" + tink "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell" + tinktypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" baremetaltypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/types" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" @@ -88,6 +88,10 @@ type Config struct { } func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { + if provSpec.Value == nil { + return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") + } + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { return nil, nil, err @@ -103,36 +107,6 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } c := Config{} - endpoint, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.MetadataClient.Endpoint, "METADATA_SERVER_ENDPOINT") - if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of \"endpoint\" field: %w`, err) - } - authMethod, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.MetadataClient.AuthMethod, "METADATA_SERVER_AUTH_METHOD") - if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of \"authMethod\" field: %w`, err) - } - username, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.MetadataClient.Username, "METADATA_SERVER_USERNAME") - if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of \"username\" field: %w`, err) - } - password, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.MetadataClient.Password, "METADATA_SERVER_PASSWORD") - if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of \"password\" field: %w`, err) - } - token, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.MetadataClient.Token, "METADATA_SERVER_TOKEN") - if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of \"token\" field: %w`, err) - } - - mdCfg := &metadata.Config{ - Endpoint: endpoint, - AuthConfig: &metadata.AuthConfig{ - AuthMethod: metadata.AuthMethod(authMethod), - Username: username, - Password: password, - Token: token, - }, - } driverName, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.Driver) if err != nil { @@ -144,16 +118,19 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p switch c.driverName { case plugins.Tinkerbell: - driverConfig := struct { - ProvisionerIPAddress string `json:"provisionerIPAddress"` - MirrorHost string `json:"mirrorHost"` - }{} + driverConfig := &tinktypes.TinkerbellPluginSpec{} if err := json.Unmarshal(c.driverSpec.Raw, &driverConfig); err != nil { return nil, nil, fmt.Errorf("failed to unmarshal tinkerbell driver spec: %w", err) } - c.driver, err = tinkerbell.NewTinkerbellDriver(mdCfg, nil, driverConfig.ProvisionerIPAddress, driverConfig.MirrorHost) + tinkConfig, err := tink.GetConfig(*driverConfig, p.configVarResolver.GetConfigVarStringValueOrEnv) + + if err != nil { + return nil, nil, err + } + + c.driver, err = tink.NewTinkerbellDriver(*tinkConfig, driverConfig) if err != nil { return nil, nil, fmt.Errorf("failed to create a tinkerbell driver: %w", err) } @@ -195,7 +172,7 @@ func (p provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *cluste } } - server, err := c.driver.GetServer(ctx, machine.UID, c.driverSpec) + server, err := c.driver.GetServer(ctx) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return nil, cloudprovidererrors.ErrInstanceNotFound @@ -209,7 +186,11 @@ func (p provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *cluste }, nil } -func (p provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (config string, name string, err error) { + return "", "", nil +} + +func (p provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -218,23 +199,7 @@ func (p provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *clu } } - if err := util.CreateMachineCloudInitSecret(ctx, userdata, machine.Name, data.Client); err != nil { - return nil, fmt.Errorf("failed to create cloud-init secret for machine %s: %w", machine.Name, err) - } - - token, apiServer, err := util.ExtractTokenAndAPIServer(ctx, userdata, data.Client) - if err != nil { - return nil, fmt.Errorf("failed to extarct token and api server address: %w", err) - } - - cfg := &plugins.CloudConfigSettings{ - Token: token, - Namespace: util.CloudInitNamespace, - SecretName: machine.Name, - ClusterHost: apiServer, - } - - server, err := c.driver.ProvisionServer(ctx, machine.UID, cfg, c.driverSpec) + server, err := c.driver.ProvisionServer(ctx, log, machine.ObjectMeta, c.driverSpec, userdata) if err != nil { return nil, fmt.Errorf("failed to provision server: %w", err) } @@ -253,7 +218,7 @@ func (p provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *cl } } - if err := c.driver.DeprovisionServer(ctx, machine.UID); err != nil { + if err := c.driver.DeprovisionServer(ctx); err != nil { return false, fmt.Errorf("failed to de-provision server: %w", err) } diff --git a/pkg/cloudprovider/provider/baremetal/types/types.go b/pkg/cloudprovider/provider/baremetal/types/types.go index 374cc2fb2..028477859 100644 --- a/pkg/cloudprovider/provider/baremetal/types/types.go +++ b/pkg/cloudprovider/provider/baremetal/types/types.go @@ -24,17 +24,8 @@ import ( ) type RawConfig struct { - MetadataClient *MetadataClientConfig `json:"metadataClientConfig"` - Driver providerconfigtypes.ConfigVarString `json:"driver"` - DriverSpec runtime.RawExtension `json:"driverSpec"` -} - -type MetadataClientConfig struct { - Endpoint providerconfigtypes.ConfigVarString `json:"endpoint,omitempty"` - AuthMethod providerconfigtypes.ConfigVarString `json:"authMethod,omitempty"` - Username providerconfigtypes.ConfigVarString `json:"username,omitempty"` - Password providerconfigtypes.ConfigVarString `json:"password,omitempty"` - Token providerconfigtypes.ConfigVarString `json:"token,omitempty"` + Driver providerconfigtypes.ConfigVarString `json:"driver"` + DriverSpec runtime.RawExtension `json:"driverSpec"` } func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { From 23dc258ca1204a873b7a0bb80c6749180c544340 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 30 Jul 2024 19:15:13 +0200 Subject: [PATCH 407/489] fix hardware metadata validation (#1832) Signed-off-by: moadqassem --- .../provider/baremetal/plugins/tinkerbell/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go index 787cad8a0..a7bb65d39 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go @@ -89,7 +89,7 @@ func (d *driver) GetServer(ctx context.Context) (plugins.Server, error) { return nil, err } - if targetHardware.Spec.Metadata != nil && targetHardware.Spec.Metadata.State != tinktypes.Staged { + if targetHardware.Spec.Metadata == nil || targetHardware.Spec.Metadata.State == "" { return nil, cloudprovidererrors.ErrInstanceNotFound } From fda0fa87dda07ae2e1e2ce88d776622e517d4fa6 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 30 Jul 2024 20:46:14 +0200 Subject: [PATCH 408/489] refactor tb machine cleanup (#1831) Signed-off-by: moadqassem --- .../plugins/tinkerbell/client/hardware.go | 1 - .../plugins/tinkerbell/client/workflow.go | 61 ++++++++++++------- .../baremetal/plugins/tinkerbell/driver.go | 8 +-- .../plugins/tinkerbell/types/types.go | 2 + 4 files changed, 45 insertions(+), 27 deletions(-) diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/hardware.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/hardware.go index 2d3b1178c..f8a39362d 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/hardware.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/hardware.go @@ -62,7 +62,6 @@ func (h *HardwareClient) SetHardwareID(ctx context.Context, hardware *tinkv1alph } hardware.Spec.Metadata.Instance.ID = newID - // Set the new ID hardware.Spec.Metadata.State = tbtypes.Staged if newID == "" { // Machine has been deprovisioned diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go index 313609c9f..cee68a942 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go @@ -20,12 +20,14 @@ import ( "context" "encoding/base64" "fmt" - kerrors "k8s.io/apimachinery/pkg/api/errors" + "time" tink "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" + kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -42,7 +44,7 @@ func NewWorkflowClient(k8sClient client.Client) *WorkflowClient { } // CreateWorkflow creates a new Tinkerbell Workflow resource in the cluster. -func (w *WorkflowClient) CreateWorkflow(ctx context.Context, userData, workflowName, templateRef string, hardware tink.Hardware) error { +func (w *WorkflowClient) CreateWorkflow(ctx context.Context, userData, templateRef string, hardware tink.Hardware) error { // Construct the Workflow object ifaceConfig := hardware.Spec.Interfaces[0].DHCP dnsNameservers := "1.1.1.1" @@ -51,11 +53,14 @@ func (w *WorkflowClient) CreateWorkflow(ctx context.Context, userData, workflowN dnsNameservers = ns } + workflowName := fmt.Sprintf("%s-%s-%s", hardware.Name, templateRef, time.Now().Format("20060102150405")) workflow := &tinkv1alpha1.Workflow{ ObjectMeta: metav1.ObjectMeta{ - // TODO(MQ): generalize the naming of the workflow and implement a function that can be used across the provider. - Name: workflowName + "-workflow", + Name: workflowName, Namespace: hardware.Namespace, + Labels: map[string]string{ + tink.HardwareRefLabel: hardware.Name, + }, }, Spec: tinkv1alpha1.WorkflowSpec{ TemplateRef: templateRef, @@ -80,23 +85,6 @@ func (w *WorkflowClient) CreateWorkflow(ctx context.Context, userData, workflowN return nil } -// DeleteWorkflow deletes an existing Tinkerbell Workflow resource from the cluster. -func (w *WorkflowClient) DeleteWorkflow(ctx context.Context, name string, namespace string) error { - workflow := &tinkv1alpha1.Workflow{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - } - if err := w.tinkclient.Delete(ctx, workflow); err != nil { - if !kerrors.IsNotFound(err) { - return fmt.Errorf("failed to delete workflow: %w", err) - } - } - - return nil -} - // GetWorkflow retrieves a Tinkerbell Workflow resource from the cluster. func (w *WorkflowClient) GetWorkflow(ctx context.Context, name string, namespace string) (*tinkv1alpha1.Workflow, error) { workflow := &tinkv1alpha1.Workflow{} @@ -105,3 +93,34 @@ func (w *WorkflowClient) GetWorkflow(ctx context.Context, name string, namespace } return workflow, nil } + +// CleanupWorkflows would delete all workflows that are assigned to a de-provisioned hardware, and they are in a pending +// state, to avoid the situation of re-running a workflow for a de-provisioned machine. +func (w *WorkflowClient) CleanupWorkflows(ctx context.Context, hardwareName, namespace string) error { + workflows := &tinkv1alpha1.WorkflowList{} + if err := w.tinkclient.List(ctx, workflows, &client.ListOptions{ + Namespace: namespace, + LabelSelector: labels.SelectorFromSet(map[string]string{ + tink.HardwareRefLabel: hardwareName, + }), + }); err != nil { + if kerrors.IsNotFound(err) { + return nil + } + + return fmt.Errorf("failed to fetch workflows: %w", err) + } + + for _, workflow := range workflows.Items { + if workflow.Status.State == tinkv1alpha1.WorkflowStatePending || + workflow.Status.State == tinkv1alpha1.WorkflowStateTimeout { + if err := w.tinkclient.Delete(ctx, &workflow); err != nil { + if !kerrors.IsNotFound(err) { + return fmt.Errorf("failed to delete workflow: %w", err) + } + } + } + } + + return nil +} diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go index a7bb65d39..08f44f12e 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go @@ -129,7 +129,7 @@ func (d *driver) ProvisionServer(ctx context.Context, _ *zap.SugaredLogger, meta // Create Workflow to match the template and server server := tinktypes.Hardware{Hardware: hardware} - if err = d.WorkflowClient.CreateWorkflow(ctx, userdata, server.Name, client.ProvisionWorkerNodeTemplate, server); err != nil { + if err = d.WorkflowClient.CreateWorkflow(ctx, userdata, client.ProvisionWorkerNodeTemplate, server); err != nil { return nil, err } @@ -152,10 +152,8 @@ func (d *driver) DeprovisionServer(ctx context.Context) error { return err } - // Delete the associated Workflow. - workflowName := targetHardware.Name + "-workflow" // Assuming workflow names are derived from hardware names - if err := d.WorkflowClient.DeleteWorkflow(ctx, workflowName, targetHardware.Namespace); err != nil { - return fmt.Errorf("failed to delete workflow %s: %w", workflowName, err) + if err := d.WorkflowClient.CleanupWorkflows(ctx, targetHardware.Name, targetHardware.Namespace); err != nil { + return fmt.Errorf("failed to cleanup workflows for hardware %s: %w", targetHardware.Name, err) } // Reset the hardware ID and state in the tinkerbell cluster. diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/types.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/types.go index ba0ecce58..fe5d64bed 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/types.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/types.go @@ -27,6 +27,8 @@ const ( Staged string = "Staged" Provisioned string = "Provisioned" Decommissioned string = "Decommissioned" + + HardwareRefLabel = "app.kubernetes.io/hardware-reference" ) // TinkerbellPluginSpec defines the required information for the Tinkerbell plugin. From 5349a3cb7f1da6efcd295d0a342c634314732ef0 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 31 Jul 2024 21:22:14 +0500 Subject: [PATCH 409/489] AWS: Increase IMDS hop limit for EC2 instances (#1833) Signed-off-by: Waleed Malik --- pkg/cloudprovider/provider/aws/provider.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 6f9ad9a4b..26acb7c42 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -61,6 +61,11 @@ const ( // Interval and timeout for polling. pollInterval = 2 * time.Second pollTimeout = 5 * time.Minute + // The maximum number of hops that the metadata service can be forwarded to, defaults to 2. + // We need to set this to a higher value i.e. 3 to ensure that it is not blocked by extra hops that are introduced either by CNI or other networking components. With lower + // limits AWS metadata service is not reachable from the container network in such a scenario. + // For example: https://github.com/cilium/cilium/issues/25232 + awsMetadataHTTPPutResponseHopLimit = 3 ) var ( @@ -760,6 +765,9 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * assignPublicIP := config.AssignPublicIP == nil || *config.AssignPublicIP instanceRequest := &ec2.RunInstancesInput{ + MetadataOptions: &ec2types.InstanceMetadataOptionsRequest{ + HttpPutResponseHopLimit: aws.Int32(awsMetadataHTTPPutResponseHopLimit), + }, ImageId: aws.String(amiID), InstanceMarketOptions: instanceMarketOptions, BlockDeviceMappings: []ec2types.BlockDeviceMapping{ From aad4dd9b15b9075f5a8473f5b2e367d870329a4c Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Wed, 14 Aug 2024 23:16:20 +0200 Subject: [PATCH 410/489] bump to Go 1.23.0 (#1838) * bump to Go 1.23.0 * fix lint issues --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 14 +++++++------- .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 10 +++++----- .prow/verify.yaml | 16 ++++++++-------- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- pkg/cloudprovider/provider/alibaba/provider.go | 4 +--- pkg/controller/util/machine_deployment.go | 8 ++++---- 22 files changed, 49 insertions(+), 51 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 335a7e498..91e6465ed 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -119,7 +119,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 3e0d190bd..9fe30cd6a 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - /bin/bash - -c @@ -54,7 +54,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index 8a19efd95..918e28af9 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 60ab55363..0ccb25085 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 56933396c..607b60611 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -61,7 +61,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -126,7 +126,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -157,7 +157,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -188,7 +188,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -219,7 +219,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 5775b0724..6413dc422 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -93,7 +93,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 41bc48629..31b383d96 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 8bcb58d1c..3214bec42 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 43b17c324..db96bab7e 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 27601cde5..68e93701e 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -26,7 +26,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index da5f331ac..ba1110a2e 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 752e61e71..47b4e01fd 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 0669747b2..748655019 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 2d10d1236..00fa7ad29 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 8a1cc2a4c..d31118f03 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 27494b3fa..2b58d1000 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 21c27cbad..622721b44 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -60,7 +60,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -92,7 +92,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -124,7 +124,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -156,7 +156,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-kind-0.23-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 003609082..915ab182e 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-0 command: - make args: @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-0 command: - make args: @@ -63,7 +63,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-0 command: - make args: @@ -83,7 +83,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-0 command: - make args: @@ -102,7 +102,7 @@ presubmits: clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-0 command: - "/usr/local/bin/shfmt" args: @@ -130,7 +130,7 @@ presubmits: clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-0 command: - "./hack/verify-boilerplate.sh" resources: @@ -149,7 +149,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-0 command: - ./hack/verify-licenses.sh resources: @@ -165,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.22-node-20-11 + - image: quay.io/kubermatic/build:go-1.23-node-20-0 command: - make args: diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 3bb93987d..b14cd3911 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-20-11 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.23-node-20-0 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index cef2e0291..a511a46cc 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.22-node-20-11 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.23-node-20-0 containerize ./hack/verify-licenses.sh go mod vendor diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index 472eac148..0a01c6695 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -423,9 +423,7 @@ func getInstance(client *ecs.Client, instanceName string, uid string) (*ecs.Inst return nil, fmt.Errorf("failed to describe instance with instanceName: %s: %w", instanceName, err) } - if response.Instances.Instance == nil || - len(response.Instances.Instance) == 0 || - response.GetHttpStatus() == http.StatusNotFound { + if len(response.Instances.Instance) == 0 || response.GetHttpStatus() == http.StatusNotFound { return nil, cloudprovidererrors.ErrInstanceNotFound } diff --git a/pkg/controller/util/machine_deployment.go b/pkg/controller/util/machine_deployment.go index 5b7a764ad..081862b2e 100644 --- a/pkg/controller/util/machine_deployment.go +++ b/pkg/controller/util/machine_deployment.go @@ -132,7 +132,7 @@ func SetDeploymentRevision(deployment *v1alpha1.MachineDeployment, revision stri // MaxRevision finds the highest revision in the machine sets. func MaxRevision(log *zap.SugaredLogger, allMSs []*v1alpha1.MachineSet) int64 { - max := int64(0) + maxRev := int64(0) for _, ms := range allMSs { if v, err := Revision(ms); err != nil { log.Debugw( @@ -140,11 +140,11 @@ func MaxRevision(log *zap.SugaredLogger, allMSs []*v1alpha1.MachineSet) int64 { "machinset", client.ObjectKeyFromObject(ms), zap.Error(err), ) - } else if v > max { - max = v + } else if v > maxRev { + maxRev = v } } - return max + return maxRev } // Revision returns the revision number of the input object. From 67426e6f31d9e6b860b6eef5e44dbe7cddeaa297 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Thu, 15 Aug 2024 12:30:21 +0200 Subject: [PATCH 411/489] bump Dockerfile to golang 1.23.0 (#1839) --- Dockerfile | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3428f4fc6..b8c22b311 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.22.5 +ARG GO_VERSION=1.23.0 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index ef58a6c26..9ed60e822 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.22.5 +GO_VERSION ?= 1.23.0 GOOS ?= $(shell go env GOOS) From 646e5aa5f8e6c64520a037cda5e84b64d452a4b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Thu, 15 Aug 2024 13:55:21 +0200 Subject: [PATCH 412/489] Add support for Kubernetes v1.31 (#1840) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- README.md | 1 + examples/alibaba-machinedeployment.yaml | 2 +- examples/anexia-machinedeployment.yaml | 2 +- examples/aws-machinedeployment.yaml | 2 +- examples/azure-machinedeployment.yaml | 2 +- examples/digitalocean-machinedeployment.yaml | 2 +- examples/equinixmetal-machinedeployment.yaml | 2 +- examples/gce-machinedeployment.yaml | 2 +- examples/hetzner-machinedeployment.yaml | 2 +- examples/kubevirt-machinedeployment.yaml | 2 +- examples/linode-machinedeployment.yaml | 2 +- examples/nutanix-machinedeployment.yaml | 2 +- examples/opennebula-machinedeployment.yaml | 2 +- examples/openstack-machinedeployment.yaml | 2 +- examples/scaleway-machinedeployment.yaml | 2 +- ...ware-cloud-director-machinedeployment.yaml | 2 +- ...e-datastore-cluster-machinedeployment.yaml | 2 +- examples/vsphere-machinedeployment.yaml | 2 +- examples/vultr-machinedeployment.yaml | 2 +- go.mod | 21 +++++---- go.sum | 46 +++++++++++-------- pkg/controller/machine/controller.go | 2 +- test/e2e/provisioning/all_e2e_test.go | 20 ++++---- test/e2e/provisioning/helper.go | 7 +-- ...hinedeployment-azure-redhat-satellite.yaml | 2 +- 25 files changed, 73 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index 201b49511..b835c768d 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ machine-controller tries to follow the Kubernetes version Currently supported K8S versions are: +- 1.31 - 1.30 - 1.29 - 1.28 diff --git a/examples/alibaba-machinedeployment.yaml b/examples/alibaba-machinedeployment.yaml index 123a2279c..0b00f70d6 100644 --- a/examples/alibaba-machinedeployment.yaml +++ b/examples/alibaba-machinedeployment.yaml @@ -62,4 +62,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index 85b5f986f..9049c60bd 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -59,4 +59,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/aws-machinedeployment.yaml b/examples/aws-machinedeployment.yaml index 5fc670744..4881ad415 100644 --- a/examples/aws-machinedeployment.yaml +++ b/examples/aws-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/azure-machinedeployment.yaml b/examples/azure-machinedeployment.yaml index 81372e487..b2d22d9e2 100644 --- a/examples/azure-machinedeployment.yaml +++ b/examples/azure-machinedeployment.yaml @@ -91,4 +91,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/digitalocean-machinedeployment.yaml b/examples/digitalocean-machinedeployment.yaml index 30855854e..a0e5646a3 100644 --- a/examples/digitalocean-machinedeployment.yaml +++ b/examples/digitalocean-machinedeployment.yaml @@ -57,4 +57,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/equinixmetal-machinedeployment.yaml b/examples/equinixmetal-machinedeployment.yaml index c553209c6..9246f24ec 100644 --- a/examples/equinixmetal-machinedeployment.yaml +++ b/examples/equinixmetal-machinedeployment.yaml @@ -51,4 +51,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index 277e8a2d1..eaffe350b 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -90,4 +90,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/hetzner-machinedeployment.yaml b/examples/hetzner-machinedeployment.yaml index 80b9184d7..7ef07fe16 100644 --- a/examples/hetzner-machinedeployment.yaml +++ b/examples/hetzner-machinedeployment.yaml @@ -67,4 +67,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index 377654ee5..4436edf8a 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -69,4 +69,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/linode-machinedeployment.yaml b/examples/linode-machinedeployment.yaml index e13b2f91d..52b5ab7ef 100644 --- a/examples/linode-machinedeployment.yaml +++ b/examples/linode-machinedeployment.yaml @@ -53,4 +53,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/nutanix-machinedeployment.yaml b/examples/nutanix-machinedeployment.yaml index 21e0b9958..41482e82d 100644 --- a/examples/nutanix-machinedeployment.yaml +++ b/examples/nutanix-machinedeployment.yaml @@ -82,4 +82,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/opennebula-machinedeployment.yaml b/examples/opennebula-machinedeployment.yaml index 68a32cab0..1b20c86c6 100644 --- a/examples/opennebula-machinedeployment.yaml +++ b/examples/opennebula-machinedeployment.yaml @@ -67,4 +67,4 @@ spec: # use cloud-init for flatcar as ignition doesn't know anything about OpenNebula yet provisioningUtility: "cloud-init" versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index 2c9255f9f..2f2becfd4 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -166,4 +166,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/scaleway-machinedeployment.yaml b/examples/scaleway-machinedeployment.yaml index db09377d5..f21bc387a 100644 --- a/examples/scaleway-machinedeployment.yaml +++ b/examples/scaleway-machinedeployment.yaml @@ -61,4 +61,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/vmware-cloud-director-machinedeployment.yaml b/examples/vmware-cloud-director-machinedeployment.yaml index ccf89ae24..b326cb494 100644 --- a/examples/vmware-cloud-director-machinedeployment.yaml +++ b/examples/vmware-cloud-director-machinedeployment.yaml @@ -89,4 +89,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml index a914c9808..d2590254f 100644 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ b/examples/vsphere-datastore-cluster-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index 47376de65..8b74af9a8 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/examples/vultr-machinedeployment.yaml b/examples/vultr-machinedeployment.yaml index 2a3a22a84..47d9dc51a 100644 --- a/examples/vultr-machinedeployment.yaml +++ b/examples/vultr-machinedeployment.yaml @@ -70,4 +70,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.29.6 + kubelet: 1.30.4 diff --git a/go.mod b/go.mod index 64672b4ba..5fd921b96 100644 --- a/go.mod +++ b/go.mod @@ -49,17 +49,17 @@ require ( gomodules.xyz/jsonpatch/v2 v2.4.0 google.golang.org/api v0.187.0 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.30.3 - k8s.io/apiextensions-apiserver v0.30.2 - k8s.io/apimachinery v0.30.3 - k8s.io/client-go v0.30.3 - k8s.io/cloud-provider v0.30.2 + k8s.io/api v0.31.0 + k8s.io/apiextensions-apiserver v0.31.0 + k8s.io/apimachinery v0.31.0 + k8s.io/client-go v0.31.0 + k8s.io/cloud-provider v0.31.0 k8s.io/klog v1.0.0 - k8s.io/kubectl v0.30.3 - k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 + k8s.io/kubectl v0.31.0 + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 kubevirt.io/api v1.2.2 kubevirt.io/containerized-data-importer-api v1.59.0 - sigs.k8s.io/controller-runtime v0.18.4 + sigs.k8s.io/controller-runtime v0.19.0 sigs.k8s.io/yaml v1.4.0 ) @@ -96,6 +96,7 @@ require ( github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect @@ -110,7 +111,7 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect + github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.5 // indirect @@ -142,6 +143,7 @@ require ( github.com/prometheus/procfs v0.15.1 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect + github.com/x448/float16 v0.8.4 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect @@ -164,6 +166,7 @@ require ( google.golang.org/grpc v1.65.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect + gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index e512e355a..318c25782 100644 --- a/go.sum +++ b/go.sum @@ -148,6 +148,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs= @@ -245,8 +247,8 @@ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 h1:k7nVchz72niMH6YLQNvHSdIE7iqsQxK1P41mySCvssg= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= +github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -456,6 +458,8 @@ github.com/vultr/govultr/v3 v3.9.0 h1:63V/22mpfquRA5DenJ9EF0VozHg0k+X4dhUWcDXHPy github.com/vultr/govultr/v3 v3.9.0/go.mod h1:Rd8ebpXm7jxH3MDmhnEs+zrlYW212ouhx+HeUMfHm2o= github.com/wI2L/jsondiff v0.2.0 h1:dE00WemBa1uCjrzQUUTE/17I6m5qAaN0EMFOg2Ynr/k= github.com/wI2L/jsondiff v0.2.0/go.mod h1:axTcwtBkY4TsKuV+RgoMhHyHKKFRI6nnjRLi8LLYQnA= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.0 h1:d70R37I0HrDLsafRrMBXyrD4lmQbCHE873t00Vr0gm0= @@ -486,8 +490,8 @@ go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= -go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= -go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -708,6 +712,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= @@ -730,17 +736,17 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= -k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ= -k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04= -k8s.io/apiextensions-apiserver v0.30.2 h1:l7Eue2t6QiLHErfn2vwK4KgF4NeDgjQkCXtEbOocKIE= -k8s.io/apiextensions-apiserver v0.30.2/go.mod h1:lsJFLYyK40iguuinsb3nt+Sj6CmodSI4ACDLep1rgjw= +k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= +k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= +k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= +k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= -k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= -k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= -k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= -k8s.io/cloud-provider v0.30.2 h1:yov6r02v7sMUNNvzEz51LtL2krn2c1wsC+dy/8BxKQI= -k8s.io/cloud-provider v0.30.2/go.mod h1:w69t2dSjDtI9BYK6SEqj6HmMKIojEk08fXRoUzjFN2I= +k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= +k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= +k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= +k8s.io/cloud-provider v0.31.0 h1:qNOs78I2/7zQmyStfDtY2M7EdilUl9fCSYMcqBju/tA= +k8s.io/cloud-provider v0.31.0/go.mod h1:QgUPqLoL6aXhLlrNg1U4IrJk/PvvxgeOnT2ixkgnqT0= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= @@ -756,12 +762,12 @@ k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lV k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20240703190633-0aa61b46e8c2 h1:T5TEV4a+pEjc+j9Xui3MGGeoDLIN6uzZrx8NYotFMgQ= k8s.io/kube-openapi v0.0.0-20240703190633-0aa61b46e8c2/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= -k8s.io/kubectl v0.30.3 h1:YIBBvMdTW0xcDpmrOBzcpUVsn+zOgjMYIu7kAq+yqiI= -k8s.io/kubectl v0.30.3/go.mod h1:IcR0I9RN2+zzTRUa1BzZCm4oM0NLOawE6RzlDvd1Fpo= +k8s.io/kubectl v0.31.0 h1:kANwAAPVY02r4U4jARP/C+Q1sssCcN/1p9Nk+7BQKVg= +k8s.io/kubectl v0.31.0/go.mod h1:pB47hhFypGsaHAPjlwrNbvhXgmuAr01ZBvAIIUaI8d4= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 h1:jgGTlFYnhF1PM1Ax/lAlxUPE+KfCIXHaathvJg1C3ak= -k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v1.2.2 h1:PeA937vsZawmKAsiiDQZJ/BbGH4OhEWsIzWrCNfmYXk= kubevirt.io/api v1.2.2/go.mod h1:SbeR9ma4EwnaOZEUkh/lNz0kzYm5LPpEDE30vKXC5Zg= kubevirt.io/containerized-data-importer-api v1.59.0 h1:GdDt9BlR0qHejpMaPfASbsG8JWDmBf1s7xZBj5W9qn0= @@ -769,8 +775,8 @@ kubevirt.io/containerized-data-importer-api v1.59.0/go.mod h1:4yOGtCE7HvgKp7wftZ kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 h1:fZYvD3/Vnitfkx6IJxjLAk8ugnZQ7CXVYcRfkSKmuZY= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= -sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= +sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= +sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= diff --git a/pkg/controller/machine/controller.go b/pkg/controller/machine/controller.go index 0a179c175..2a24dd29c 100644 --- a/pkg/controller/machine/controller.go +++ b/pkg/controller/machine/controller.go @@ -202,7 +202,7 @@ func Add( nodePortRange: nodePortRange, overrideBootstrapKubeletAPIServer: overrideBootstrapKubeletAPIServer, } - utilruntime.ErrorHandlers = append(utilruntime.ErrorHandlers, func(error) { + utilruntime.ErrorHandlers = append(utilruntime.ErrorHandlers, func(context.Context, error, string, ...interface{}) { reconciler.metrics.Errors.Add(1) }) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 61339d252..0f12248c6 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -85,7 +85,7 @@ const ( ) const ( - defaultKubernetesVersion = "1.29.6" + defaultKubernetesVersion = "1.29.8" awsDefaultKubernetesVersion = "1.26.12" defaultContainerRuntime = "containerd" ) @@ -348,7 +348,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.26. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.28.11", "1.29.6", "1.30.2"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.28.12", "1.29.8", "1.30.4", "1.31.0"))) runScenarios(context.Background(), t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -424,7 +424,7 @@ func TestAWSProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := Not(VersionSelector("1.28.11", "1.29.6", "1.30.2")) + selector := Not(VersionSelector("1.28.12", "1.29.8", "1.30.4", "1.31.0")) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -478,7 +478,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.28.11", "1.29.6", "1.30.2"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.28.12", "1.29.8", "1.30.4", "1.31.0"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -500,7 +500,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.28.11", "1.29.6", "1.30.2"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.28.12", "1.29.8", "1.30.4", "1.31.0"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -596,7 +596,7 @@ func TestAzureProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.30.2"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.30.4", "1.31.0"))) // act params := []string{ @@ -625,7 +625,7 @@ func TestAzureCustomImageReferenceProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.2"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.4", "1.31.0"))) // act params := []string{ fmt.Sprintf("<< AZURE_TENANT_ID >>=%s", azureTenantID), @@ -841,7 +841,7 @@ func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(Not(OsSelector("amzn2", "centos")), Not(VersionSelector("1.30.2"))) + selector := And(Not(OsSelector("amzn2", "centos")), Not(VersionSelector("1.30.4", "1.31.0"))) params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) @@ -853,7 +853,7 @@ func TestVsphereMultipleNICProvisioningE2E(t *testing.T) { t.Parallel() // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.2"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.4", "1.31.0"))) params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereMultipleNICManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) @@ -882,7 +882,7 @@ func TestVsphereDatastoreClusterProvisioningE2E(t *testing.T) { t.Parallel() // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(OsSelector("ubuntu", "centos", "rhel", "flatcar"), Not(VersionSelector("1.30.2"))) + selector := And(OsSelector("ubuntu", "centos", "rhel", "flatcar"), Not(VersionSelector("1.30.4", "1.31.0"))) params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereDSCManifest, fmt.Sprintf("vs-dsc-%s", *testRunIdentifier)) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index d5739465b..9390d7110 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -34,9 +34,10 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.28.11"), - semver.MustParse("v1.29.6"), - semver.MustParse("v1.30.2"), + semver.MustParse("v1.28.12"), + semver.MustParse("v1.29.8"), + semver.MustParse("v1.30.4"), + semver.MustParse("v1.31.0"), } operatingSystems = []providerconfigtypes.OperatingSystem{ diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml index 3cf509b0a..6f67ec0f7 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml @@ -55,4 +55,4 @@ spec: rhelOrganizationName: "" rhelActivationKey: "" versions: - kubelet: 1.29.6 + kubelet: 1.29.8 From 40b32abe43e43811413f21022fd079576d3f2a0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Fri, 16 Aug 2024 15:05:56 +0200 Subject: [PATCH 413/489] Remove CentOS tests (#1843) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- .prow/provider-aws.yaml | 31 --------------- test/e2e/provisioning/all_e2e_test.go | 39 ++++--------------- test/e2e/provisioning/helper.go | 7 ---- .../machinedeployment-digitalocean.yaml | 1 - 4 files changed, 7 insertions(+), 71 deletions(-) diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 607b60611..d478420a6 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -174,37 +174,6 @@ presubmits: limits: memory: 7Gi - - name: pull-machine-controller-e2e-aws-centos8 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAWSCentOS8ProvisioningE2E" - env: - - name: CLOUD_PROVIDER - value: aws - securityContext: - privileged: true - resources: - requests: - memory: 7Gi - cpu: 2 - limits: - memory: 7Gi - - name: pull-machine-controller-e2e-aws-assume-role always_run: false decorate: true diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 0f12248c6..9470065b6 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -302,7 +302,7 @@ func TestKubevirtProvisioningE2E(t *testing.T) { t.Fatal("Unable to run kubevirt tests, KUBEVIRT_E2E_TESTS_KUBECONFIG must be set") } - selector := OsSelector("ubuntu", "centos", "flatcar", "rockylinux") + selector := OsSelector("ubuntu", "flatcar", "rockylinux") params := []string{ fmt.Sprintf("<< KUBECONFIG_BASE64 >>=%s", safeBase64Encoding(kubevirtKubeconfig)), @@ -402,7 +402,7 @@ func TestDigitalOceanProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, DO_E2E_TESTS_TOKEN environment variable cannot be empty") } - selector := OsSelector("ubuntu", "centos", "rockylinux") + selector := OsSelector("ubuntu", "rockylinux") // act params := []string{fmt.Sprintf("<< DIGITALOCEAN_TOKEN >>=%s", doToken)} @@ -531,29 +531,6 @@ func TestAWSFlatcarCoreOSCloudInit8ProvisioningE2E(t *testing.T) { runScenarios(context.Background(), t, selector, params, AWSManifest, fmt.Sprintf("aws-%s", *testRunIdentifier)) } -func TestAWSCentOS8ProvisioningE2E(t *testing.T) { - t.Parallel() - - // test data - awsKeyID := os.Getenv("AWS_E2E_TESTS_KEY_ID") - awsSecret := os.Getenv("AWS_E2E_TESTS_SECRET") - if len(awsKeyID) == 0 || len(awsSecret) == 0 { - t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") - } - - amiID := "ami-032025b3afcbb6b34" // official "CentOS 8.2.2004 x86_64" - - params := []string{ - fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), - fmt.Sprintf("<< AWS_SECRET_ACCESS_KEY >>=%s", awsSecret), - fmt.Sprintf("<< AMI >>=%s", amiID), - } - - // We would like to test CentOS8 image only in this test as the other images are tested in TestAWSProvisioningE2E - selector := OsSelector("centos") - runScenarios(context.Background(), t, selector, params, AWSManifest, fmt.Sprintf("aws-%s", *testRunIdentifier)) -} - // TestAWSEbsEncryptionEnabledProvisioningE2E - a test suite that exercises AWS provider with ebs encryption enabled // by requesting nodes with different combination of container runtime type, container runtime version and the OS flavour. func TestAWSEbsEncryptionEnabledProvisioningE2E(t *testing.T) { @@ -686,7 +663,6 @@ func TestGCEProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, GOOGLE_SERVICE_ACCOUNT environment variable cannot be empty") } - // Act. GCE does not support CentOS. selector := OsSelector("ubuntu", "flatcar") params := []string{ fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>=%s", safeBase64Encoding(googleServiceAccount)), @@ -729,7 +705,7 @@ func TestEquinixMetalProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, METAL_PROJECT_ID environment variable cannot be empty") } - selector := And(OsSelector("ubuntu", "centos", "rockylinux", "flatcar"), Not(NameSelector("migrateUID"))) + selector := And(OsSelector("ubuntu", "rockylinux", "flatcar"), Not(NameSelector("migrateUID"))) // act params := []string{ @@ -776,7 +752,6 @@ func TestLinodeProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, LINODE_E2E_TESTS_TOKEN environment variable cannot be empty") } - // we're shimming userdata through Linode stackscripts and the stackscript hasn't been verified for use with centos selector := OsSelector("ubuntu") // act @@ -841,7 +816,7 @@ func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(Not(OsSelector("amzn2", "centos")), Not(VersionSelector("1.30.4", "1.31.0"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.30.4", "1.31.0"))) params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) @@ -882,7 +857,7 @@ func TestVsphereDatastoreClusterProvisioningE2E(t *testing.T) { t.Parallel() // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(OsSelector("ubuntu", "centos", "rhel", "flatcar"), Not(VersionSelector("1.30.4", "1.31.0"))) + selector := And(OsSelector("ubuntu", "rhel", "flatcar"), Not(VersionSelector("1.30.4", "1.31.0"))) params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereDSCManifest, fmt.Sprintf("vs-dsc-%s", *testRunIdentifier)) @@ -975,7 +950,7 @@ func TestNutanixProvisioningE2E(t *testing.T) { // exclude migrateUID test case because it's a no-op for Nutanix and runs from a different // location, thus possibly blocking access a HTTP proxy if it is configured. - selector := And(OsSelector("ubuntu", "centos"), Not(NameSelector("migrateUID"))) + selector := And(OsSelector("ubuntu"), Not(NameSelector("migrateUID"))) params := getNutanixTestParams(t) runScenarios(context.Background(), t, selector, params, nutanixManifest, fmt.Sprintf("nx-%s", *testRunIdentifier)) } @@ -1113,7 +1088,7 @@ func TestVultrProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, VULTR_API_KEY environment variable cannot be empty") } - selector := OsSelector("ubuntu", "centos", "rockylinux") + selector := OsSelector("ubuntu", "rockylinux") // act params := []string{fmt.Sprintf("<< VULTR_API_KEY >>=%s", apiKey)} diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 9390d7110..a947e17d9 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -42,7 +42,6 @@ var ( operatingSystems = []providerconfigtypes.OperatingSystem{ providerconfigtypes.OperatingSystemUbuntu, - providerconfigtypes.OperatingSystemCentOS, providerconfigtypes.OperatingSystemAmazonLinux2, providerconfigtypes.OperatingSystemRHEL, providerconfigtypes.OperatingSystemFlatcar, @@ -51,7 +50,6 @@ var ( openStackImages = map[string]string{ string(providerconfigtypes.OperatingSystemUbuntu): "kubermatic-ubuntu", - string(providerconfigtypes.OperatingSystemCentOS): "machine-controller-e2e-centos", string(providerconfigtypes.OperatingSystemRHEL): "machine-controller-e2e-rhel-8-5", string(providerconfigtypes.OperatingSystemFlatcar): "kubermatic-e2e-flatcar", string(providerconfigtypes.OperatingSystemRockyLinux): "machine-controller-e2e-rockylinux", @@ -63,7 +61,6 @@ var ( } vSphereOSImageTemplates = map[string]string{ - string(providerconfigtypes.OperatingSystemCentOS): "kkp-centos-7", string(providerconfigtypes.OperatingSystemFlatcar): "kkp-flatcar-3139.2.0", string(providerconfigtypes.OperatingSystemRHEL): "kkp-rhel-8.6", string(providerconfigtypes.OperatingSystemRockyLinux): "kkp-rockylinux-8", @@ -71,7 +68,6 @@ var ( } kubevirtImages = map[string]string{ - string(providerconfigtypes.OperatingSystemCentOS): "centos", string(providerconfigtypes.OperatingSystemFlatcar): "flatcar", string(providerconfigtypes.OperatingSystemRHEL): "rhel", string(providerconfigtypes.OperatingSystemRockyLinux): "rockylinux", @@ -245,9 +241,6 @@ func testScenario(ctx context.Context, t *testing.T, testCase scenario, cloudPro if strings.Contains(cloudProvider, string(providerconfigtypes.CloudProviderEquinixMetal)) { switch testCase.osName { - case string(providerconfigtypes.OperatingSystemCentOS): - scenarioParams = append(scenarioParams, fmt.Sprintf("<< INSTANCE_TYPE >>=%s", "m3.small.x86")) - scenarioParams = append(scenarioParams, fmt.Sprintf("<< METRO_CODE >>=%s", "AM")) case string(providerconfigtypes.OperatingSystemFlatcar): scenarioParams = append(scenarioParams, fmt.Sprintf("<< INSTANCE_TYPE >>=%s", "c3.small.x86")) scenarioParams = append(scenarioParams, fmt.Sprintf("<< METRO_CODE >>=%s", "NY")) diff --git a/test/e2e/provisioning/testdata/machinedeployment-digitalocean.yaml b/test/e2e/provisioning/testdata/machinedeployment-digitalocean.yaml index 19479c97d..6710de4c1 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-digitalocean.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-digitalocean.yaml @@ -35,7 +35,6 @@ spec: monitoring: false tags: - "machine-controller" - # Can be 'ubuntu' or 'centos' operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false From 107e3514d267ee1d626432e1aa91843f45b6e690 Mon Sep 17 00:00:00 2001 From: Mara Sophie Grosch Date: Wed, 21 Aug 2024 09:52:10 +0200 Subject: [PATCH 414/489] Anexia: extend network configuration (#1845) Can now create multiple network interfaces, each with multiple addresses (one from each configured prefix), easily allowing Machines to have e.g. an IPv4 and IPv6 address. Signed-off-by: Mara Sophie Grosch Co-authored-by: Mara Sophie Grosch --- examples/anexia-machinedeployment.yaml | 26 ++- .../provider/anexia/helper_test.go | 14 +- .../provider/anexia/network_provisioning.go | 158 +++++++++++++ pkg/cloudprovider/provider/anexia/provider.go | 184 ++------------- .../provider/anexia/provider_test.go | 31 ++- .../provider/anexia/resolve_config.go | 212 ++++++++++++++++++ .../provider/anexia/types/types.go | 44 +++- 7 files changed, 485 insertions(+), 184 deletions(-) create mode 100644 pkg/cloudprovider/provider/anexia/network_provisioning.go create mode 100644 pkg/cloudprovider/provider/anexia/resolve_config.go diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index 9049c60bd..0b875b4d8 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -30,7 +30,7 @@ spec: namespace: kube-system name: machine-controller-anexia key: token - vlanID: "<< ANEXIA_VLAN_ID >>" + # Currently only the "Flatcar Linux Stable" template is supported. # Use templateBuild to specify a build. If empty => latest # Alternatively use templateID for a specific template. @@ -48,6 +48,30 @@ spec: - size: 60 performanceType: ENT6 + # Each entry in this array will create a network interface in each + # Machine, connected to the given VLAN. + networks: + - vlan: "<< ANEXIA_VLAN_ID >>" + + # If prefixes are given, we reserve an IP address for each of + # them - if you give one IPv4 and one IPv6 prefix, your + # Machines will have dual-stack connectivity + # + # As an compatibility-aid for the old cloudProviderSpec.vlanID, + # which reserved an IP for the configured VLAN, you can also + # have an entry "" (empty string) to get the same behavior - + # but this is not recommended. + # + # Not configuring any prefix might be useful if you want to + # configure IP addresses on this interface via other means, + # e.g. a Layer2 load balancer. + # + # Each MachineDeployment needs at least one Network with at + # least one Prefix, because we have to know (and thus, reserve) + # at least one IP address for each Machine. + prefixes: + - "<< ANEXIA_PREFIX_ID >>" + # You may have this old disk config attribute in your config - please migrate to the disks attribute. # For now it is still recognized though. #diskSize: 60 diff --git a/pkg/cloudprovider/provider/anexia/helper_test.go b/pkg/cloudprovider/provider/anexia/helper_test.go index ff5a1f2be..eeb4000cf 100644 --- a/pkg/cloudprovider/provider/anexia/helper_test.go +++ b/pkg/cloudprovider/provider/anexia/helper_test.go @@ -90,8 +90,11 @@ func hookableConfig(hook func(*anxtypes.RawConfig)) anxtypes.RawConfig { {Size: 5, PerformanceType: newConfigVarString("ENT6")}, }, + Networks: []anxtypes.RawNetwork{ + {VlanID: newConfigVarString("test-vlan"), PrefixIDs: []types.ConfigVarString{newConfigVarString("test-prefix")}}, + }, + Token: newConfigVarString("test-token"), - VlanID: newConfigVarString("test-vlan"), LocationID: newConfigVarString("test-location"), TemplateID: newConfigVarString("test-template-id"), } @@ -112,7 +115,6 @@ func hookableReconcileContext(locationID string, templateID string, hook func(*r Status: &anxtypes.ProviderStatus{}, UserData: "", Config: resolvedConfig{ - VlanID: "VLAN-ID", LocationID: locationID, TemplateID: templateID, Disks: []resolvedDisk{ @@ -122,6 +124,14 @@ func hookableReconcileContext(locationID string, templateID string, hook func(*r }, }, }, + Networks: []resolvedNetwork{ + { + VlanID: "VLAN-ID", + Prefixes: []string{ + "Prefix-ID", + }, + }, + }, RawConfig: anxtypes.RawConfig{ CPUs: 5, Memory: 5, diff --git a/pkg/cloudprovider/provider/anexia/network_provisioning.go b/pkg/cloudprovider/provider/anexia/network_provisioning.go new file mode 100644 index 000000000..277685d30 --- /dev/null +++ b/pkg/cloudprovider/provider/anexia/network_provisioning.go @@ -0,0 +1,158 @@ +/* +Copyright 2024 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package anexia + +import ( + "context" + "sync" + "time" + + "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" + anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" + anxclient "go.anx.io/go-anxcloud/pkg/client" + anxaddr "go.anx.io/go-anxcloud/pkg/ipam/address" + anxvm "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" + "go.uber.org/zap" +) + +func networkInterfacesForProvisioning(ctx context.Context, log *zap.SugaredLogger, client anxclient.Client) ([]anxvm.Network, error) { + reconcileContext := getReconcileContext(ctx) + + config := reconcileContext.Config + status := reconcileContext.Status + + // make sure we have the status.Networks array allocated to fill it with + // data, warning if we already have something but not matching the + // configuration. + if len(status.Networks) != len(config.Networks) { + if len(status.Networks) != 0 { + log.Warn("size of status.Networks != config.Networks, this should not happen in normal operation - ignoring existing status") + } + + status.Networks = make([]anxtypes.NetworkStatus, len(config.Networks)) + } + + ret := make([]anxvm.Network, len(config.Networks)) + for netIndex, network := range config.Networks { + networkStatus := &status.Networks[netIndex] + addresses := make([]string, len(network.Prefixes)) + + for prefixIndex, prefix := range network.Prefixes { + // make sure we have the address status array allocated to fill it + // with our IP reserve status, warning if we already have something + // there but not matching the configuration. + if len(networkStatus.Addresses) != len(network.Prefixes) { + if len(networkStatus.Addresses) != 0 { + log.Warnf("size of status.Networks[%[1]v].Addresses != config.Networks[%[1]v].Prefixes, this should not happen in normal operation - ignoring existing status", netIndex) + } + + networkStatus.Addresses = make([]anxtypes.NetworkAddressStatus, len(network.Prefixes)) + } + + reservedIP, err := getIPAddress(ctx, log, &network, prefix, &networkStatus.Addresses[prefixIndex], client) + if err != nil { + return nil, newError(common.CreateMachineError, "failed to reserve IP: %v", err) + } + + addresses[prefixIndex] = reservedIP + } + + ret[netIndex] = anxvm.Network{ + VLAN: network.VlanID, + IPs: addresses, + + // the one NIC type supported by the ADC API + NICType: anxtypes.VmxNet3NIC, + } + } + + return ret, nil +} + +// ENGSUP-3404 is about a race condition when reserving IPs - two calls for one +// IP each, coming in at "nearly the same millisecond", can result in both +// reserving the same IP. +// +// The proposed fix was to reserve n IPs in one call, but that would require +// lots of architecture changes - we can't really do the "reserve IPs for all +// the Machines we want to create and then create the Machines" here. +// +// This mutex alleviates the issue enough, that we didn't see it in a long +// time. It's not impossible this race condition was fixed in some other change +// and we weren't told, but I'd rather not test this and risk having problems +// again.. it's not too expensive of a Mutex. +var _engsup3404mutex sync.Mutex + +func getIPAddress(ctx context.Context, log *zap.SugaredLogger, network *resolvedNetwork, prefix string, status *anxtypes.NetworkAddressStatus, client anxclient.Client) (string, error) { + reconcileContext := getReconcileContext(ctx) + + // only use IP if it is still unbound + if status.ReservedIP != "" && status.IPState == anxtypes.IPStateUnbound && (!status.IPProvisioningExpires.IsZero() && status.IPProvisioningExpires.After(time.Now())) { + log.Infow("Re-using already provisioned IP", "ip", status.ReservedIP) + return status.ReservedIP, nil + } + + _engsup3404mutex.Lock() + defer _engsup3404mutex.Unlock() + + log.Info("Creating a new IP for machine") + addrAPI := anxaddr.NewAPI(client) + config := reconcileContext.Config + + res, err := addrAPI.ReserveRandom(ctx, anxaddr.ReserveRandom{ + LocationID: config.LocationID, + VlanID: network.VlanID, + PrefixID: prefix, + ReservationPeriod: uint(anxtypes.IPProvisioningExpires / time.Second), + Count: 1, + }) + if err != nil { + return "", newError(common.InvalidConfigurationMachineError, "failed to reserve an ip address: %v", err) + } + + if len(res.Data) < 1 { + return "", newError(common.InsufficientResourcesMachineError, "no ip address is available for this machine") + } + + ip := res.Data[0].Address + status.ReservedIP = ip + status.IPState = anxtypes.IPStateUnbound + status.IPProvisioningExpires = time.Now().Add(anxtypes.IPProvisioningExpires) + + return ip, nil +} + +func networkReservedAddresses(status *anxtypes.ProviderStatus) []string { + ret := make([]string, 0) + for _, network := range status.Networks { + for _, address := range network.Addresses { + if address.ReservedIP != "" && address.IPState == anxtypes.IPStateBound { + ret = append(ret, address.ReservedIP) + } + } + } + + return ret +} + +func networkStatusMarkIPsBound(status *anxtypes.ProviderStatus) { + for network := range status.Networks { + for addr := range status.Networks[network].Addresses { + status.Networks[network].Addresses[addr].IPState = anxtypes.IPStateBound + } + } +} diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index a5c283603..4581fbc20 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -24,15 +24,11 @@ import ( "fmt" "net/http" "strings" - "sync" "time" "go.anx.io/go-anxcloud/pkg/api" - corev1 "go.anx.io/go-anxcloud/pkg/apis/core/v1" - vspherev1 "go.anx.io/go-anxcloud/pkg/apis/vsphere/v1" "go.anx.io/go-anxcloud/pkg/client" anxclient "go.anx.io/go-anxcloud/pkg/client" - anxaddr "go.anx.io/go-anxcloud/pkg/ipam/address" "go.anx.io/go-anxcloud/pkg/vsphere" "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/progress" anxvm "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" @@ -62,31 +58,15 @@ const ( var ( // ErrConfigDiskSizeAndDisks is returned when the config has both DiskSize and Disks set, which is unsupported. ErrConfigDiskSizeAndDisks = errors.New("both the deprecated DiskSize and new Disks attribute are set") + + // ErrConfigVlanIDAndNetworks is returned when the config has both VlanID and Networks set, which is unsupported. + ErrConfigVlanIDAndNetworks = errors.New("both the deprecated VlanID and new Networks attribute are set") ) type provider struct { configVarResolver *providerconfig.ConfigVarResolver } -// resolvedDisk contains the resolved values from types.RawDisk. -type resolvedDisk struct { - anxtypes.RawDisk - - PerformanceType string -} - -// resolvedConfig contains the resolved values from types.RawConfig. -type resolvedConfig struct { - anxtypes.RawConfig - - Token string - VlanID string - LocationID string - TemplateID string - - Disks []resolvedDisk -} - func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance instance.Instance, retErr error) { status := getProviderStatus(log, machine) log.Debugw("Machine status", "status", status) @@ -139,15 +119,10 @@ func provisionVM(ctx context.Context, log *zap.SugaredLogger, client anxclient.C log.Info("Machine does not contain a provisioningID yet. Starting to provision") config := reconcileContext.Config - reservedIP, err := getIPAddress(ctx, log, client) + networkInterfaces, err := networkInterfacesForProvisioning(ctx, log, client) if err != nil { - return newError(common.CreateMachineError, "failed to reserve IP: %v", err) + return fmt.Errorf("error generating network config for machine: %w", err) } - networkInterfaces := []anxvm.Network{{ - NICType: anxtypes.VmxNet3NIC, - IPs: []string{reservedIP}, - VLAN: config.VlanID, - }} vm := vmAPI.Provisioning().VM().NewDefinition( config.LocationID, @@ -210,7 +185,7 @@ func provisionVM(ctx context.Context, log *zap.SugaredLogger, client anxclient.C } // we successfully sent a VM provisioning request to the API, we consider the IP as 'Bound' now - status.IPState = anxtypes.IPStateBound + networkStatusMarkIPsBound(status) status.ProvisioningID = provisionResponse.Identifier err = updateMachineStatus(reconcileContext.Machine, *status, reconcileContext.ProviderData.Update) @@ -231,44 +206,6 @@ func provisionVM(ctx context.Context, log *zap.SugaredLogger, client anxclient.C return updateMachineStatus(reconcileContext.Machine, *status, reconcileContext.ProviderData.Update) } -var _engsup3404mutex sync.Mutex - -func getIPAddress(ctx context.Context, log *zap.SugaredLogger, client anxclient.Client) (string, error) { - reconcileContext := getReconcileContext(ctx) - status := reconcileContext.Status - - // only use IP if it is still unbound - if status.ReservedIP != "" && status.IPState == anxtypes.IPStateUnbound && (!status.IPProvisioningExpires.IsZero() && status.IPProvisioningExpires.After(time.Now())) { - log.Infow("Re-using already provisioned IP", "ip", status.ReservedIP) - return status.ReservedIP, nil - } - - _engsup3404mutex.Lock() - defer _engsup3404mutex.Unlock() - - log.Info("Creating a new IP for machine") - addrAPI := anxaddr.NewAPI(client) - config := reconcileContext.Config - res, err := addrAPI.ReserveRandom(ctx, anxaddr.ReserveRandom{ - LocationID: config.LocationID, - VlanID: config.VlanID, - Count: 1, - }) - if err != nil { - return "", newError(common.InvalidConfigurationMachineError, "failed to reserve an ip address: %v", err) - } - if len(res.Data) < 1 { - return "", newError(common.InsufficientResourcesMachineError, "no ip address is available for this machine") - } - - ip := res.Data[0].Address - status.ReservedIP = ip - status.IPState = anxtypes.IPStateUnbound - status.IPProvisioningExpires = time.Now().Add(anxtypes.IPProvisioningExpires) - - return ip, nil -} - func isAlreadyProvisioning(ctx context.Context) bool { status := getReconcileContext(ctx).Status condition := meta.FindStatusCondition(status.Conditions, ProvisionedType) @@ -297,95 +234,6 @@ func ensureConditions(status *anxtypes.ProviderStatus) { } } -func resolveTemplateID(ctx context.Context, a api.API, config anxtypes.RawConfig, configVarResolver *providerconfig.ConfigVarResolver, locationID string) (string, error) { - templateName, err := configVarResolver.GetConfigVarStringValue(config.Template) - if err != nil { - return "", fmt.Errorf("failed to get 'template': %w", err) - } - - templateBuild, err := configVarResolver.GetConfigVarStringValue(config.TemplateBuild) - if err != nil { - return "", fmt.Errorf("failed to get 'templateBuild': %w", err) - } - - template, err := vspherev1.FindNamedTemplate(ctx, a, templateName, templateBuild, corev1.Location{Identifier: locationID}) - if err != nil { - return "", fmt.Errorf("failed to retrieve named template: %w", err) - } - - return template.Identifier, nil -} - -func (p *provider) resolveConfig(ctx context.Context, log *zap.SugaredLogger, config anxtypes.RawConfig) (*resolvedConfig, error) { - var err error - ret := resolvedConfig{ - RawConfig: config, - } - - ret.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(config.Token, anxtypes.AnxTokenEnv) - if err != nil { - return nil, fmt.Errorf("failed to get 'token': %w", err) - } - - ret.LocationID, err = p.configVarResolver.GetConfigVarStringValue(config.LocationID) - if err != nil { - return nil, fmt.Errorf("failed to get 'locationID': %w", err) - } - - ret.TemplateID, err = p.configVarResolver.GetConfigVarStringValue(config.TemplateID) - if err != nil { - return nil, fmt.Errorf("failed to get 'templateID': %w", err) - } - - // when "templateID" is not set, we expect "template" to be - if ret.TemplateID == "" { - a, _, err := getClient(ret.Token, nil) - if err != nil { - return nil, fmt.Errorf("failed initializing API clients: %w", err) - } - - templateID, err := resolveTemplateID(ctx, a, config, p.configVarResolver, ret.LocationID) - if err != nil { - return nil, fmt.Errorf("failed retrieving template id from named template: %w", err) - } - - ret.TemplateID = templateID - } - - ret.VlanID, err = p.configVarResolver.GetConfigVarStringValue(config.VlanID) - if err != nil { - return nil, fmt.Errorf("failed to get 'vlanID': %w", err) - } - - if config.DiskSize != 0 { - if len(config.Disks) != 0 { - return nil, ErrConfigDiskSizeAndDisks - } - - log.Info("Configuration uses the deprecated DiskSize attribute, please migrate to the Disks array instead.") - - config.Disks = []anxtypes.RawDisk{ - { - Size: config.DiskSize, - }, - } - config.DiskSize = 0 - } - - ret.Disks = make([]resolvedDisk, len(config.Disks)) - - for idx, disk := range config.Disks { - ret.Disks[idx].RawDisk = disk - - ret.Disks[idx].PerformanceType, err = p.configVarResolver.GetConfigVarStringValue(disk.PerformanceType) - if err != nil { - return nil, fmt.Errorf("failed to get 'performanceType' of disk %v: %w", idx, err) - } - } - - return &ret, nil -} - func (p *provider) getConfig(ctx context.Context, log *zap.SugaredLogger, provSpec clusterv1alpha1.ProviderSpec) (*resolvedConfig, *providerconfigtypes.Config, error) { pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { @@ -456,8 +304,19 @@ func (p *provider) Validate(ctx context.Context, log *zap.SugaredLogger, machine return errors.New("no valid template configured") } - if config.VlanID == "" { - return errors.New("vlan id is missing") + if len(config.Networks) == 0 { + return errors.New("no networks configured") + } + + atLeastOneAddressSourceConfigured := false + for _, network := range config.Networks { + if len(network.Prefixes) > 0 { + atLeastOneAddressSourceConfigured = true + break + } + } + if !atLeastOneAddressSourceConfigured { + return errors.New("none of the configured networks define an address source, cannot create Machines without any IP") } return nil @@ -506,10 +365,7 @@ func (p *provider) Get(ctx context.Context, log *zap.SugaredLogger, machine *clu } instance := anexiaInstance{} - - if status.IPState == anxtypes.IPStateBound && status.ReservedIP != "" { - instance.reservedAddresses = []string{status.ReservedIP} - } + instance.reservedAddresses = networkReservedAddresses(&status) timeoutCtx, cancel := context.WithTimeout(ctx, anxtypes.GetRequestTimeout) defer cancel() diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index bae7a23c1..d26546fad 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -242,7 +242,7 @@ func TestAnexiaProvider(t *testing.T) { provider := New(nil).(*provider) for _, testCase := range testCases { - templateID, err := resolveTemplateID(context.Background(), a, testCase.config, provider.configVarResolver, "foo") + templateID, err := provider.resolveTemplateID(context.Background(), a, testCase.config, "foo") if testCase.expectedError != "" { if err != nil { testhelper.AssertErr(t, err) @@ -292,17 +292,25 @@ func TestAnexiaProvider(t *testing.T) { t.Run("Test getIPAddress", func(t *testing.T) { t.Parallel() providerStatus := &anxtypes.ProviderStatus{ - ReservedIP: "", - IPState: "", + Networks: []anxtypes.NetworkStatus{ + { + Addresses: []anxtypes.NetworkAddressStatus{ + { + ReservedIP: "", + IPState: "", + }, + }, + }, + }, } ctx := createReconcileContext(context.Background(), reconcileContext{Status: providerStatus}) t.Run("with unbound reserved IP", func(t *testing.T) { expectedIP := "8.8.8.8" - providerStatus.ReservedIP = expectedIP - providerStatus.IPState = anxtypes.IPStateUnbound - providerStatus.IPProvisioningExpires = time.Now().Add(anxtypes.IPProvisioningExpires) - reservedIP, err := getIPAddress(ctx, log, client) + providerStatus.Networks[0].Addresses[0].ReservedIP = expectedIP + providerStatus.Networks[0].Addresses[0].IPState = anxtypes.IPStateUnbound + providerStatus.Networks[0].Addresses[0].IPProvisioningExpires = time.Now().Add(anxtypes.IPProvisioningExpires) + reservedIP, err := getIPAddress(ctx, log, &resolvedNetwork{}, "Prefix-ID", &providerStatus.Networks[0].Addresses[0], client) testhelper.AssertNoErr(t, err) testhelper.AssertEquals(t, expectedIP, reservedIP) }) @@ -342,9 +350,14 @@ func TestValidate(t *testing.T) { Config: hookableConfig(func(c *anxtypes.RawConfig) { c.LocationID.Value = "" }), Error: errors.New("location id is missing"), }, + + ConfigTestCase{ + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Networks = []anxtypes.RawNetwork{} }), + Error: errors.New("no networks configured"), + }, ConfigTestCase{ - Config: hookableConfig(func(c *anxtypes.RawConfig) { c.VlanID.Value = "" }), - Error: errors.New("vlan id is missing"), + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.VlanID.Value = "legacy VLAN-ID" }), + Error: ErrConfigVlanIDAndNetworks, }, ConfigTestCase{ Config: hookableConfig(func(c *anxtypes.RawConfig) { c.DiskSize = 10; c.Disks = []anxtypes.RawDisk{} }), diff --git a/pkg/cloudprovider/provider/anexia/resolve_config.go b/pkg/cloudprovider/provider/anexia/resolve_config.go new file mode 100644 index 000000000..fbe146dcf --- /dev/null +++ b/pkg/cloudprovider/provider/anexia/resolve_config.go @@ -0,0 +1,212 @@ +/* +Copyright 2024 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package anexia + +import ( + "context" + "fmt" + + "go.uber.org/zap" + + "go.anx.io/go-anxcloud/pkg/api" + corev1 "go.anx.io/go-anxcloud/pkg/apis/core/v1" + vspherev1 "go.anx.io/go-anxcloud/pkg/apis/vsphere/v1" + + anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" +) + +// resolvedDisk contains the resolved values from types.RawDisk. +type resolvedDisk struct { + anxtypes.RawDisk + + PerformanceType string +} + +// resolvedNetwork contains the resolved values from types.RawNetwork. +type resolvedNetwork struct { + anxtypes.RawNetwork + + VlanID string + + // List of prefixes to each reserve an IP address from. + // + // Legacy compatibility: may contain an empty string as entry to reserve an IP address from the given VLAN instead of a specific prefix. + Prefixes []string +} + +// resolvedConfig contains the resolved values from types.RawConfig. +type resolvedConfig struct { + anxtypes.RawConfig + + Token string + LocationID string + TemplateID string + + Disks []resolvedDisk + Networks []resolvedNetwork +} + +func (p *provider) resolveTemplateID(ctx context.Context, a api.API, config anxtypes.RawConfig, locationID string) (string, error) { + templateName, err := p.configVarResolver.GetConfigVarStringValue(config.Template) + if err != nil { + return "", fmt.Errorf("failed to get 'template': %w", err) + } + + templateBuild, err := p.configVarResolver.GetConfigVarStringValue(config.TemplateBuild) + if err != nil { + return "", fmt.Errorf("failed to get 'templateBuild': %w", err) + } + + template, err := vspherev1.FindNamedTemplate(ctx, a, templateName, templateBuild, corev1.Location{Identifier: locationID}) + if err != nil { + return "", fmt.Errorf("failed to retrieve named template: %w", err) + } + + return template.Identifier, nil +} + +func (p *provider) resolveNetworkConfig(log *zap.SugaredLogger, config anxtypes.RawConfig) (*[]resolvedNetwork, error) { + legacyVlanIDConfig, _ := config.VlanID.MarshalJSON() + if string(legacyVlanIDConfig) != `""` { + if len(config.Networks) != 0 { + return nil, ErrConfigVlanIDAndNetworks + } + + log.Info("Configuration uses the deprecated VlanID attribute, please migrate to the Networks array instead.") + + vlanID, err := p.configVarResolver.GetConfigVarStringValue(config.VlanID) + if err != nil { + return nil, fmt.Errorf("failed to get 'vlanID': %w", err) + } + + return &[]resolvedNetwork{ + { + VlanID: vlanID, + Prefixes: []string{""}, + }, + }, nil + } + + ret := make([]resolvedNetwork, len(config.Networks)) + for netIndex, net := range config.Networks { + vlanID, err := p.configVarResolver.GetConfigVarStringValue(net.VlanID) + if err != nil { + return nil, fmt.Errorf("failed to get 'vlanID' for network %v: %w", netIndex, err) + } + + prefixes := make([]string, len(net.PrefixIDs)) + for prefixIndex, prefix := range net.PrefixIDs { + prefixID, err := p.configVarResolver.GetConfigVarStringValue(prefix) + if err != nil { + return nil, fmt.Errorf("failed to get 'prefixID' for network %v, prefix %v: %w", netIndex, prefixIndex, err) + } + + prefixes[prefixIndex] = prefixID + } + + ret[netIndex] = resolvedNetwork{ + VlanID: vlanID, + Prefixes: prefixes, + } + } + + return &ret, nil +} + +func (p *provider) resolveDiskConfig(log *zap.SugaredLogger, config anxtypes.RawConfig) (*[]resolvedDisk, error) { + if config.DiskSize != 0 { + if len(config.Disks) != 0 { + return nil, ErrConfigDiskSizeAndDisks + } + + log.Info("Configuration uses the deprecated DiskSize attribute, please migrate to the Disks array instead.") + + config.Disks = []anxtypes.RawDisk{ + { + Size: config.DiskSize, + }, + } + config.DiskSize = 0 + } + + ret := make([]resolvedDisk, len(config.Disks)) + + for idx, disk := range config.Disks { + performanceType, err := p.configVarResolver.GetConfigVarStringValue(disk.PerformanceType) + if err != nil { + return nil, fmt.Errorf("failed to get 'performanceType' of disk %v: %w", idx, err) + } + + ret[idx] = resolvedDisk{ + RawDisk: disk, + PerformanceType: performanceType, + } + } + + return &ret, nil +} + +func (p *provider) resolveConfig(ctx context.Context, log *zap.SugaredLogger, config anxtypes.RawConfig) (*resolvedConfig, error) { + var err error + ret := resolvedConfig{ + RawConfig: config, + } + + ret.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(config.Token, anxtypes.AnxTokenEnv) + if err != nil { + return nil, fmt.Errorf("failed to get 'token': %w", err) + } + + ret.LocationID, err = p.configVarResolver.GetConfigVarStringValue(config.LocationID) + if err != nil { + return nil, fmt.Errorf("failed to get 'locationID': %w", err) + } + + ret.TemplateID, err = p.configVarResolver.GetConfigVarStringValue(config.TemplateID) + if err != nil { + return nil, fmt.Errorf("failed to get 'templateID': %w", err) + } + + diskConfig, err := p.resolveDiskConfig(log, config) + if err != nil { + return nil, fmt.Errorf("failed to resolve disk config: %w", err) + } + ret.Disks = *diskConfig + + networkConfig, err := p.resolveNetworkConfig(log, config) + if err != nil { + return nil, fmt.Errorf("failed to resolve network config: %w", err) + } + ret.Networks = *networkConfig + + // when "templateID" is not set, we expect "template" to be + if ret.TemplateID == "" { + a, _, err := getClient(ret.Token, nil) + if err != nil { + return nil, fmt.Errorf("failed initializing API clients: %w", err) + } + + templateID, err := p.resolveTemplateID(ctx, a, config, ret.LocationID) + if err != nil { + return nil, fmt.Errorf("failed retrieving template id from named template: %w", err) + } + + ret.TemplateID = templateID + } + + return &ret, nil +} diff --git a/pkg/cloudprovider/provider/anexia/types/types.go b/pkg/cloudprovider/provider/anexia/types/types.go index 04cb2c992..8b7bfdff3 100644 --- a/pkg/cloudprovider/provider/anexia/types/types.go +++ b/pkg/cloudprovider/provider/anexia/types/types.go @@ -53,10 +53,20 @@ type RawDisk struct { PerformanceType providerconfigtypes.ConfigVarString `json:"performanceType"` } +// RawNetwork specifies a single network interface. +type RawNetwork struct { + // Identifier of the VLAN to attach this network interface to. + VlanID providerconfigtypes.ConfigVarString `json:"vlan"` + + // IDs of prefixes to reserve IP addresses from for each Machine on network interface. + // + // Empty list means that no IPs will be reserved, but the interface will still be added. + PrefixIDs []providerconfigtypes.ConfigVarString `json:"prefixes"` +} + // RawConfig contains all the configuration values for VMs to create, with some values maybe being fetched from secrets. type RawConfig struct { Token providerconfigtypes.ConfigVarString `json:"token,omitempty"` - VlanID providerconfigtypes.ConfigVarString `json:"vlanID"` LocationID providerconfigtypes.ConfigVarString `json:"locationID"` TemplateID providerconfigtypes.ConfigVarString `json:"templateID"` @@ -71,16 +81,34 @@ type RawConfig struct { DiskSize int `json:"diskSize"` Disks []RawDisk `json:"disks"` + + // Deprecated, use Networks instead. + VlanID providerconfigtypes.ConfigVarString `json:"vlanID"` + + // Configuration of the network interfaces. At least one entry with at + // least one Prefix is required. + Networks []RawNetwork `json:"networks"` +} + +type NetworkAddressStatus struct { + ReservedIP string `json:"reservedIP"` + IPState string `json:"ipState"` + IPProvisioningExpires time.Time `json:"ipProvisioningExpires"` +} + +type NetworkStatus struct { + // each entry belongs to a config.Networks.Prefix entry at the same index + Addresses []NetworkAddressStatus `json:"addresses"` } type ProviderStatus struct { - InstanceID string `json:"instanceID"` - ProvisioningID string `json:"provisioningID"` - DeprovisioningID string `json:"deprovisioningID"` - ReservedIP string `json:"reservedIP"` - IPState string `json:"ipState"` - IPProvisioningExpires time.Time `json:"ipProvisioningExpires"` - Conditions []v1.Condition `json:"conditions,omitempty"` + InstanceID string `json:"instanceID"` + ProvisioningID string `json:"provisioningID"` + DeprovisioningID string `json:"deprovisioningID"` + Conditions []v1.Condition `json:"conditions,omitempty"` + + // each entry belongs to the config.Networks entry at the same index + Networks []NetworkStatus `json:"networkStatus,omitempty"` } func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { From 3ead298cdda97fdb82de16df46adf463593d5d0f Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Mon, 26 Aug 2024 16:17:17 +0500 Subject: [PATCH 415/489] vSphere: support for VM Groups (#1847) * vSphere: support for VM Groups Signed-off-by: Waleed Malik * Use break instead of continue to exit loop Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- .gitignore | 2 +- .golangci.yml | 3 +- examples/vsphere-machinedeployment.yaml | 2 + go.mod | 6 +- go.sum | 12 +- .../provider/vsphere/provider.go | 23 +++- .../provider/vsphere/types/types.go | 15 ++- pkg/cloudprovider/provider/vsphere/vmgroup.go | 125 ++++++++++++++++++ 8 files changed, 167 insertions(+), 21 deletions(-) create mode 100644 pkg/cloudprovider/provider/vsphere/vmgroup.go diff --git a/.gitignore b/.gitignore index 7c379f554..784e3f33a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,5 +15,5 @@ examples/*.srl /vendor .vscode .gitpod.yml -cmd/machine-controller/__debug_bin +cmd/machine-controller/__debug_bin* !pkg diff --git a/.golangci.yml b/.golangci.yml index d8a5e1e44..458d9f387 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -55,7 +55,8 @@ issues: - func Convert_MachineDeployment_ProviderConfig_To_ProviderSpec should be ConvertMachineDeploymentProviderConfigToProviderSpec - func Convert_MachineSet_ProviderConfig_To_ProviderSpec should be ConvertMachineSetProviderConfigToProviderSpec - func Convert_Machine_ProviderConfig_To_ProviderSpec should be ConvertMachineProviderConfigToProviderSpec - - 'cyclomatic complexity 33 of func `\(\*provider\)\.Create` is high' + - 'cyclomatic complexity [0-9]+ of func `\(\*provider\)\.Create` is high' + - 'cyclomatic complexity [0-9]+ of func `\(\*provider\)\.Validate` is high' - "SA1019: s.server.IPv6 is deprecated" exclude-dirs: - pkg/machines diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index 8b74af9a8..2e3efd019 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -62,6 +62,8 @@ spec: cluster: cl-1 # Automatically create anti affinity rules for machines vmAntiAffinity: true + # Optional. Sets the VM group for the Machines in the MachineDeployment. + # vmGroup: "vmgroup-name" cpus: 2 memoryMB: 2048 # Optional: Resize the root disk to this size. Must be bigger than the existing size diff --git a/go.mod b/go.mod index 5fd921b96..edaed7260 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/tinkerbell/tink v0.10.0 github.com/vmware/go-vcloud-director/v2 v2.25.0 - github.com/vmware/govmomi v0.38.0 + github.com/vmware/govmomi v0.42.0 github.com/vultr/govultr/v3 v3.9.0 go.anx.io/go-anxcloud v0.7.2 go.uber.org/zap v1.27.0 @@ -154,10 +154,10 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect golang.org/x/net v0.27.0 // indirect - golang.org/x/sync v0.7.0 // indirect + golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/term v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.23.0 // indirect google.golang.org/genproto v0.0.0-20240701130421-f6361c86f094 // indirect diff --git a/go.sum b/go.sum index 318c25782..f79becfc4 100644 --- a/go.sum +++ b/go.sum @@ -452,8 +452,8 @@ github.com/vektah/gqlparser/v2 v2.2.0 h1:bAc3slekAAJW6sZTi07aGq0OrfaCjj4jxARAaC7 github.com/vektah/gqlparser/v2 v2.2.0/go.mod h1:i3mQIGIrbK2PD1RrCeMTlVbkF2FJ6WkU1KJlJlC+3F4= github.com/vmware/go-vcloud-director/v2 v2.25.0 h1:RcJ5FQRku3FvQktTi8YOZsRfvhfLm315Cme50M9x9MQ= github.com/vmware/go-vcloud-director/v2 v2.25.0/go.mod h1:7Of1qJja+LLNKVegjZG7uuhhy6xgGg3q7Fkw2CEP+Tw= -github.com/vmware/govmomi v0.38.0 h1:UvQpLAOjDpO0JUxoPCXnEzOlEa/9kejO6K58qOFr6cM= -github.com/vmware/govmomi v0.38.0/go.mod h1:mtGWtM+YhTADHlCgJBiskSRPOZRsN9MSjPzaZLte/oQ= +github.com/vmware/govmomi v0.42.0 h1:MbvAlVfjNBE1mHMaQ7yOSop1KLB0/93x6VAGuCtjqtI= +github.com/vmware/govmomi v0.42.0/go.mod h1:1H5LWwsBif8HKZqbFp0FdoKTHyJE4FzL6ACequMKYQg= github.com/vultr/govultr/v3 v3.9.0 h1:63V/22mpfquRA5DenJ9EF0VozHg0k+X4dhUWcDXHPyc= github.com/vultr/govultr/v3 v3.9.0/go.mod h1:Rd8ebpXm7jxH3MDmhnEs+zrlYW212ouhx+HeUMfHm2o= github.com/wI2L/jsondiff v0.2.0 h1:dE00WemBa1uCjrzQUUTE/17I6m5qAaN0EMFOg2Ynr/k= @@ -580,8 +580,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -634,8 +634,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 824952fbf..f5f9a1a0a 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -75,6 +75,7 @@ type Config struct { MemoryMB int64 DiskSizeGB *int64 Tags []tags.Tag + VMGroup string } // Ensures that Server implements Instance interface. @@ -210,6 +211,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, nil, err } + c.VMGroup, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VMGroup) + if err != nil { + return nil, nil, nil, err + } + c.CPUs = rawConfig.CPUs c.MemoryMB = rawConfig.MemoryMB c.DiskSizeGB = rawConfig.DiskSizeGB @@ -311,10 +317,13 @@ func (p *provider) Validate(ctx context.Context, log *zap.SugaredLogger, spec cl } } - if config.VMAntiAffinity { - if config.Cluster == "" { - return fmt.Errorf("cluster is required for vm anti affinity") - } + if config.VMAntiAffinity && config.Cluster == "" { + return fmt.Errorf("cluster is required for vm anti affinity") + } else if config.VMGroup != "" && config.Cluster == "" { + return fmt.Errorf("cluster is required for vm group") + } + + if config.Cluster != "" { _, err = session.Finder.ClusterComputeResource(ctx, config.Cluster) if err != nil { return fmt.Errorf("failed to get cluster %q, %w", config.Cluster, err) @@ -376,6 +385,12 @@ func (p *provider) create(ctx context.Context, log *zap.SugaredLogger, machine * return nil, fmt.Errorf("failed to attach tags: %w", err) } + if config.VMGroup != "" { + if err := p.addToVMGroup(ctx, log, session, machine, config); err != nil { + return nil, fmt.Errorf("failed to add VM to VM group: %w", err) + } + } + if config.VMAntiAffinity { if err := p.createOrUpdateVMAntiAffinityRule(ctx, log, session, machine, config); err != nil { return nil, fmt.Errorf("failed to add VM to anti affinity rule: %w", err) diff --git a/pkg/cloudprovider/provider/vsphere/types/types.go b/pkg/cloudprovider/provider/vsphere/types/types.go index b0112d03c..443cfa137 100644 --- a/pkg/cloudprovider/provider/vsphere/types/types.go +++ b/pkg/cloudprovider/provider/vsphere/types/types.go @@ -43,12 +43,15 @@ type RawConfig struct { DatastoreCluster providerconfigtypes.ConfigVarString `json:"datastoreCluster"` Datastore providerconfigtypes.ConfigVarString `json:"datastore"` - CPUs int32 `json:"cpus"` - MemoryMB int64 `json:"memoryMB"` - DiskSizeGB *int64 `json:"diskSizeGB,omitempty"` - Tags []Tag `json:"tags,omitempty"` - AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` - VMAntiAffinity providerconfigtypes.ConfigVarBool `json:"vmAntiAffinity"` + CPUs int32 `json:"cpus"` + MemoryMB int64 `json:"memoryMB"` + DiskSizeGB *int64 `json:"diskSizeGB,omitempty"` + Tags []Tag `json:"tags,omitempty"` + AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` + + // Placement rules + VMAntiAffinity providerconfigtypes.ConfigVarBool `json:"vmAntiAffinity"` + VMGroup providerconfigtypes.ConfigVarString `json:"vmGroup,omitempty"` } // Tag represents vsphere tag. diff --git a/pkg/cloudprovider/provider/vsphere/vmgroup.go b/pkg/cloudprovider/provider/vsphere/vmgroup.go new file mode 100644 index 000000000..79c87428b --- /dev/null +++ b/pkg/cloudprovider/provider/vsphere/vmgroup.go @@ -0,0 +1,125 @@ +/* +Copyright 2024 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vsphere + +import ( + "context" + "fmt" + "strings" + + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + + "github.com/vmware/govmomi/vim25/types" + "go.uber.org/zap" +) + +func (p *provider) addToVMGroup(ctx context.Context, log *zap.SugaredLogger, session *Session, machine *clusterv1alpha1.Machine, config *Config) error { + lock.Lock() + defer lock.Unlock() + + // Check if the VM group exists + vmGroup, err := findVMGroup(ctx, session, config.Cluster, config.VMGroup) + if err != nil { + return err + } + + // We have to find all VMs in the folder and add them to the VM group. VMGroup only contains VM reference ID which is not enough to + // identify the VM by name. + machineSetName := machine.Name[:strings.LastIndex(machine.Name, "-")] + vmsInFolder, err := session.Finder.VirtualMachineList(ctx, strings.Join([]string{config.Folder, "*"}, "/")) + if err != nil { + return fmt.Errorf("failed to find VMs in folder: %w", err) + } + + var vmRefs []types.ManagedObjectReference + for _, vm := range vmsInFolder { + // Only add VMs with the same machineSetName to the rule and exclude the machine itself if it is being deleted + if strings.HasPrefix(vm.Name(), machineSetName) && !(vm.Name() == machine.Name && machine.DeletionTimestamp != nil) { + vmRefs = append(vmRefs, vm.Reference()) + } + } + + var vmRefsToAdd []types.ManagedObjectReference + for _, vm := range vmRefs { + found := false + for _, existingVM := range vmGroup.Vm { + if existingVM.Value == vm.Value { + log.Debugf("VM %s already in VM group %s", machine.Name, config.VMGroup) + found = true + break + } + } + if !found { + vmRefsToAdd = append(vmRefsToAdd, vm) + } + } + + // Add the VM to the VM group + vmGroup.Vm = append(vmGroup.Vm, vmRefsToAdd...) + cluster, err := session.Finder.ClusterComputeResource(ctx, config.Cluster) + if err != nil { + return err + } + + spec := &types.ClusterConfigSpecEx{ + GroupSpec: []types.ClusterGroupSpec{ + { + ArrayUpdateSpec: types.ArrayUpdateSpec{ + Operation: types.ArrayUpdateOperationEdit, + }, + Info: vmGroup, + }, + }, + } + + log.Debugf("Adding VM %s in VM group %s", machine.Name, config.VMGroup) + task, err := cluster.Reconfigure(ctx, spec, true) + if err != nil { + return err + } + + taskResult, err := task.WaitForResultEx(ctx) + if err != nil { + return fmt.Errorf("error waiting for cluster %v reconfiguration to complete", cluster.Name()) + } + if taskResult.State != types.TaskInfoStateSuccess { + return fmt.Errorf("cluster %v reconfiguration task was not successful", cluster.Name()) + } + log.Debugf("Successfully added VM %s in VM group %s", machine.Name, config.VMGroup) + return nil +} + +func findVMGroup(ctx context.Context, session *Session, clusterName, vmGroup string) (*types.ClusterVmGroup, error) { + cluster, err := session.Finder.ClusterComputeResource(ctx, clusterName) + if err != nil { + return nil, err + } + + clusterConfigInfoEx, err := cluster.Configuration(ctx) + if err != nil { + return nil, err + } + + for _, group := range clusterConfigInfoEx.Group { + if clusterVMGroup, ok := group.(*types.ClusterVmGroup); ok { + if clusterVMGroup.Name == vmGroup { + return clusterVMGroup, nil + } + } + } + return nil, fmt.Errorf("cannot find VM group %s", vmGroup) +} From 540163394d85f261e3633550521e5959d4d061e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Mon, 26 Aug 2024 14:45:18 +0200 Subject: [PATCH 416/489] Fix vSphere E2E tests (#1844) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- test/e2e/provisioning/all_e2e_test.go | 2 +- .../provisioning/testdata/machinedeployment-vsphere.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 9470065b6..d940442b1 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -816,7 +816,7 @@ func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.30.4", "1.31.0"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.4", "1.31.0"))) params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml index c53ba3f43..377d9e8f5 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml @@ -30,12 +30,12 @@ spec: username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' datacenter: 'Hamburg' - folder: '/Hamburg/vm/Kubermatic-ci' + folder: '/Hamburg/vm/Kubermatic-dev' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443/'. '/sdk' gets appended automatically - cluster: Kubermatic + cluster: 'vSAN Cluster' vmAntiAffinity: true - datastore: vsan + datastore: Datastore0-truenas cpus: 2 MemoryMB: 4096 diskSizeGB: << DISK_SIZE >> From 7051af1be2cebfd694420f1fdac9ca442efe6a79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Mudrini=C4=87?= Date: Mon, 26 Aug 2024 15:49:18 +0200 Subject: [PATCH 417/489] Move module to k8c.io/machine-controller (#1842) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Marko Mudrinić --- .prow/e2e-features.yaml | 4 ++ .prow/postsubmits.yaml | 2 + .prow/provider-alibaba.yaml | 1 + .prow/provider-anexia.yaml | 1 + .prow/provider-aws.yaml | 6 +++ .prow/provider-azure.yaml | 3 ++ .prow/provider-digitalocean.yaml | 1 + .prow/provider-equinix-metal.yaml | 1 + .prow/provider-gcp.yaml | 1 + .prow/provider-hetzner.yaml | 1 + .prow/provider-kubevirt.yaml | 1 + .prow/provider-linode.yaml | 1 + .prow/provider-nutanix.yaml | 1 + .prow/provider-openstack.yaml | 2 + .prow/provider-scaleway.yaml | 1 + .prow/provider-vmware-cloud-director.yaml | 1 + .prow/provider-vsphere.yaml | 5 ++ .prow/verify.yaml | 8 ++++ Dockerfile | 6 +-- Makefile | 6 +-- cmd/machine-controller/main.go | 26 +++++----- cmd/webhook/main.go | 8 ++-- docs/howto-provider.md | 12 ++--- go.mod | 2 +- pkg/admission/admission.go | 4 +- pkg/admission/machinedeployments.go | 2 +- pkg/admission/machinedeployments_test.go | 2 +- .../machinedeployments_validation.go | 6 +-- pkg/admission/machines.go | 10 ++-- pkg/admission/util.go | 2 +- .../v1alpha1/conversions/conversions.go | 6 +-- .../v1alpha1/conversions/conversions_test.go | 4 +- .../providerconfig_to_providerspec.go | 2 +- .../providerconfig_to_providerspec_test.go | 2 +- pkg/apis/cluster/v1alpha1/defaults.go | 2 +- pkg/apis/cluster/v1alpha1/doc.go | 2 +- pkg/apis/cluster/v1alpha1/machine_types.go | 2 +- .../v1alpha1/machinedeployment_types.go | 2 +- pkg/apis/cluster/v1alpha1/machineset_types.go | 2 +- .../cluster/v1alpha1/migrations/migrations.go | 20 ++++---- pkg/apis/cluster/v1alpha1/register.go | 2 +- .../cluster/v1alpha1/zz_generated.deepcopy.go | 2 +- pkg/cloudprovider/cache/cloudprovidercache.go | 2 +- .../cache/cloudprovidercache_test.go | 2 +- pkg/cloudprovider/errors/errors.go | 2 +- pkg/cloudprovider/provider.go | 48 +++++++++---------- .../provider/alibaba/provider.go | 20 ++++---- .../provider/alibaba/types/types.go | 4 +- .../provider/anexia/helper_test.go | 12 ++--- pkg/cloudprovider/provider/anexia/instance.go | 4 +- .../provider/anexia/network_provisioning.go | 5 +- pkg/cloudprovider/provider/anexia/provider.go | 20 ++++---- .../provider/anexia/provider_test.go | 10 ++-- .../provider/anexia/reconcile_context.go | 8 ++-- .../provider/anexia/resolve_config.go | 2 +- .../provider/anexia/types/types.go | 8 ++-- pkg/cloudprovider/provider/aws/provider.go | 20 ++++---- pkg/cloudprovider/provider/aws/types/types.go | 4 +- .../provider/azure/create_delete_resources.go | 2 +- pkg/cloudprovider/provider/azure/provider.go | 22 ++++----- .../provider/azure/types/types.go | 4 +- .../plugins/tinkerbell/client/hardware.go | 4 +- .../plugins/tinkerbell/client/workflow.go | 2 +- .../baremetal/plugins/tinkerbell/driver.go | 11 +++-- .../plugins/tinkerbell/types/hardware.go | 2 +- .../plugins/tinkerbell/types/types.go | 2 +- .../provider/baremetal/provider.go | 24 +++++----- .../provider/baremetal/types/types.go | 4 +- .../provider/digitalocean/provider.go | 20 ++++---- .../provider/digitalocean/types/types.go | 4 +- pkg/cloudprovider/provider/edge/provider.go | 8 ++-- .../provider/equinixmetal/provider.go | 16 +++---- .../provider/equinixmetal/types/types.go | 4 +- pkg/cloudprovider/provider/fake/provider.go | 10 ++-- pkg/cloudprovider/provider/gce/config.go | 8 ++-- pkg/cloudprovider/provider/gce/instance.go | 2 +- pkg/cloudprovider/provider/gce/provider.go | 14 +++--- .../provider/gce/provider_test.go | 4 +- pkg/cloudprovider/provider/gce/types/types.go | 6 +-- .../provider/hetzner/provider.go | 18 +++---- .../provider/hetzner/types/types.go | 4 +- .../provider/kubevirt/provider.go | 20 ++++---- .../provider/kubevirt/provider_test.go | 4 +- .../provider/kubevirt/types/types.go | 4 +- pkg/cloudprovider/provider/linode/provider.go | 18 +++---- .../provider/linode/types/types.go | 4 +- pkg/cloudprovider/provider/nutanix/client.go | 8 ++-- .../provider/nutanix/provider.go | 16 +++---- .../provider/nutanix/types/types.go | 4 +- .../provider/opennebula/provider.go | 16 +++---- .../provider/opennebula/types/types.go | 4 +- .../provider/openstack/provider.go | 18 +++---- .../provider/openstack/provider_test.go | 8 ++-- .../provider/openstack/types/types.go | 4 +- .../provider/scaleway/provider.go | 16 +++---- .../provider/scaleway/types/types.go | 4 +- .../provider/vmwareclouddirector/client.go | 2 +- .../provider/vmwareclouddirector/helper.go | 2 +- .../provider/vmwareclouddirector/provider.go | 14 +++--- .../vmwareclouddirector/types/types.go | 4 +- pkg/cloudprovider/provider/vsphere/client.go | 2 +- .../provider/vsphere/provider.go | 16 +++---- .../provider/vsphere/provider_test.go | 4 +- pkg/cloudprovider/provider/vsphere/rule.go | 2 +- .../provider/vsphere/types/types.go | 4 +- pkg/cloudprovider/provider/vsphere/vmgroup.go | 2 +- pkg/cloudprovider/provider/vultr/provider.go | 16 +++---- .../provider/vultr/types/types.go | 4 +- pkg/cloudprovider/testing/testing.go | 2 +- pkg/cloudprovider/types/types.go | 6 +-- pkg/cloudprovider/util/util.go | 6 +-- pkg/cloudprovider/util/util_test.go | 6 +-- pkg/cloudprovider/validationwrapper.go | 6 +-- pkg/controller/machine/controller.go | 32 ++++++------- pkg/controller/machine/controller_test.go | 8 ++-- pkg/controller/machine/metrics.go | 8 ++-- .../machinedeployment/controller.go | 4 +- pkg/controller/machinedeployment/rolling.go | 4 +- pkg/controller/machinedeployment/sync.go | 4 +- pkg/controller/machineset/controller.go | 2 +- pkg/controller/machineset/delete_policy.go | 2 +- pkg/controller/machineset/machine.go | 2 +- pkg/controller/machineset/status.go | 2 +- pkg/controller/nodecsrapprover/controller.go | 2 +- .../nodecsrapprover/controller_test.go | 2 +- pkg/controller/util/machine.go | 2 +- pkg/controller/util/machine_deployment.go | 4 +- pkg/health/readiness.go | 2 +- pkg/machines/register.go | 2 +- pkg/node/eviction/eviction.go | 4 +- pkg/node/flags.go | 2 +- pkg/node/poddeletion/pod_deletion.go | 2 +- pkg/providerconfig/types.go | 14 +++--- pkg/providerconfig/types/types.go | 6 +-- pkg/providerconfig/types_test.go | 2 +- pkg/rhsm/util.go | 6 +-- test/e2e/provisioning/all_e2e_test.go | 6 +-- test/e2e/provisioning/deploymentscenario.go | 2 +- test/e2e/provisioning/helper.go | 2 +- test/e2e/provisioning/migrateuidscenario.go | 12 ++--- test/e2e/provisioning/verify.go | 8 ++-- 141 files changed, 489 insertions(+), 446 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 91e6465ed..70c772aff 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -17,6 +17,7 @@ presubmits: always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-aws: "true" preset-azure: "true" @@ -52,6 +53,7 @@ presubmits: always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-openstack: "true" preset-hetzner: "true" @@ -82,6 +84,7 @@ presubmits: # run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-openstack: "true" preset-hetzner: "true" @@ -110,6 +113,7 @@ presubmits: always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-hetzner: "true" preset-e2e-ssh: "true" diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 9fe30cd6a..76efda442 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -17,6 +17,7 @@ postsubmits: always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller branches: - ^main$ # Match on tags @@ -48,6 +49,7 @@ postsubmits: always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller branches: - ^main$ labels: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index 918e28af9..65dbbe73b 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -18,6 +18,7 @@ presubmits: always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller max_concurrency: 1 labels: preset-alibaba: "true" diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 0ccb25085..f38501d29 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -21,6 +21,7 @@ presubmits: always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-hetzner: "true" preset-e2e-ssh: "true" diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index d478420a6..d6346e774 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -18,6 +18,7 @@ presubmits: # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-aws: "true" preset-hetzner: "true" @@ -51,6 +52,7 @@ presubmits: # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-aws: "true" preset-hetzner: "true" @@ -83,6 +85,7 @@ presubmits: # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-aws: "true" preset-hetzner: "true" @@ -115,6 +118,7 @@ presubmits: # run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-aws: "true" preset-hetzner: "true" @@ -147,6 +151,7 @@ presubmits: always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-aws: "true" preset-hetzner: "true" @@ -178,6 +183,7 @@ presubmits: always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-aws-assume-role: "true" preset-hetzner: "true" diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 6413dc422..e2819e0ba 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -17,6 +17,7 @@ presubmits: run_if_changed: "(pkg/cloudprovider/provider/azure/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-azure: "true" preset-hetzner: "true" @@ -49,6 +50,7 @@ presubmits: run_if_changed: "(pkg/cloudprovider/provider/azure/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-azure: "true" preset-hetzner: "true" @@ -82,6 +84,7 @@ presubmits: always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-azure: "true" preset-hetzner: "true" diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 31b383d96..56471a0d7 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -17,6 +17,7 @@ presubmits: run_if_changed: "(pkg/cloudprovider/provider/digitalocean/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-digitalocean: "true" preset-hetzner: "true" diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 3214bec42..e56ddb1cd 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -18,6 +18,7 @@ presubmits: run_if_changed: "(pkg/cloudprovider/provider/equinixmetal/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-hetzner: "true" preset-e2e-ssh: "true" diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index db96bab7e..c17f705a4 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -17,6 +17,7 @@ presubmits: always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-gce: "true" preset-hetzner: "true" diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 68e93701e..efc74599a 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -17,6 +17,7 @@ presubmits: run_if_changed: "(pkg/cloudprovider/provider/hetzner/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-hetzner: "true" preset-e2e-ssh: "true" diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index ba1110a2e..f5463e98f 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -17,6 +17,7 @@ presubmits: run_if_changed: "(pkg/cloudprovider/provider/kubevirt/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller max_concurrency: 1 labels: preset-kubevirt: "true" diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 47b4e01fd..3ef294ee7 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -18,6 +18,7 @@ presubmits: optional: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-hetzner: "true" preset-e2e-ssh: "true" diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 748655019..6c1909d78 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -17,6 +17,7 @@ presubmits: run_if_changed: "(pkg/cloudprovider/provider/nutanix/)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-hetzner: "true" preset-e2e-ssh: "true" diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 00fa7ad29..aaa20bbd1 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -18,6 +18,7 @@ presubmits: # run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-openstack: "true" preset-hetzner: "true" @@ -51,6 +52,7 @@ presubmits: # run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-openstack: "true" preset-hetzner: "true" diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index d31118f03..10fea9e33 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -17,6 +17,7 @@ presubmits: always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-scaleway: "true" preset-hetzner: "true" diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 2b58d1000..e5259a8b0 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -19,6 +19,7 @@ presubmits: # Please check: https://github.com/kubermatic/machine-controller/issues/1619 optional: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller run_if_changed: "(pkg/cloudprovider/provider/vmwareclouddirector/)" labels: preset-vcloud-director: "true" diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 622721b44..8ba485055 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -17,6 +17,7 @@ presubmits: run_if_changed: "(pkg/cloudprovider/provider/vsphere/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-hetzner: "true" preset-e2e-ssh: "true" @@ -49,6 +50,7 @@ presubmits: always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-vsphere: "true" preset-rhel: "true" @@ -81,6 +83,7 @@ presubmits: always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-vsphere: "true" preset-rhel: "true" @@ -113,6 +116,7 @@ presubmits: always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-hetzner: "true" preset-e2e-ssh: "true" @@ -145,6 +149,7 @@ presubmits: always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-hetzner: "true" preset-e2e-ssh: "true" diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 915ab182e..5e9b21719 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -17,6 +17,7 @@ presubmits: always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-goproxy: "true" spec: @@ -38,6 +39,7 @@ presubmits: always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-goproxy: "true" spec: @@ -59,6 +61,7 @@ presubmits: always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-goproxy: "true" spec: @@ -79,6 +82,7 @@ presubmits: always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-goproxy: "true" spec: @@ -100,6 +104,7 @@ presubmits: run_if_changed: "^hack/" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller spec: containers: - image: quay.io/kubermatic/build:go-1.23-node-20-0 @@ -128,6 +133,7 @@ presubmits: always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller spec: containers: - image: quay.io/kubermatic/build:go-1.23-node-20-0 @@ -145,6 +151,7 @@ presubmits: run_if_changed: "^go.(mod|sum)$" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-goproxy: "true" spec: @@ -161,6 +168,7 @@ presubmits: always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" + path_alias: k8c.io/machine-controller labels: preset-goproxy: "true" spec: diff --git a/Dockerfile b/Dockerfile index b8c22b311..0cf006c9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ ARG GO_VERSION=1.23.0 FROM docker.io/golang:${GO_VERSION} AS builder -WORKDIR /go/src/github.com/kubermatic/machine-controller +WORKDIR /go/src/k8c.io/machine-controller COPY . . RUN make all @@ -23,7 +23,7 @@ FROM alpine:3.19 RUN apk add --no-cache ca-certificates cdrkit COPY --from=builder \ - /go/src/github.com/kubermatic/machine-controller/machine-controller \ - /go/src/github.com/kubermatic/machine-controller/webhook \ + /go/src/k8c.io/machine-controller/machine-controller \ + /go/src/k8c.io/machine-controller/webhook \ /usr/local/bin/ USER nobody diff --git a/Makefile b/Makefile index 9ed60e822..7bd67c081 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ build-machine-controller: machine-controller GOOS=$(GOOS) go build -v \ $(LDFLAGS) \ -o $@ \ - github.com/kubermatic/machine-controller/cmd/$* + k8c.io/machine-controller/cmd/$* .PHONY: clean clean: @@ -81,10 +81,10 @@ docker-image-publish: docker-image .PHONY: test-unit-docker test-unit-docker: @docker run --rm \ - -v $$PWD:/go/src/github.com/kubermatic/machine-controller \ + -v $$PWD:/go/src/k8c.io/machine-controller \ -v $$PWD/.buildcache:/cache \ -e GOCACHE=/cache \ - -w /go/src/github.com/kubermatic/machine-controller \ + -w /go/src/k8c.io/machine-controller \ golang:$(GO_VERSION) \ make test-unit "GOFLAGS=$(GOFLAGS)" diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 8926b61c8..6e1d25a90 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -30,19 +30,19 @@ import ( "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1/migrations" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - clusterinfo "github.com/kubermatic/machine-controller/pkg/clusterinfo" - machinecontroller "github.com/kubermatic/machine-controller/pkg/controller/machine" - machinedeploymentcontroller "github.com/kubermatic/machine-controller/pkg/controller/machinedeployment" - machinesetcontroller "github.com/kubermatic/machine-controller/pkg/controller/machineset" - "github.com/kubermatic/machine-controller/pkg/controller/nodecsrapprover" - "github.com/kubermatic/machine-controller/pkg/health" - machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" - machinesv1alpha1 "github.com/kubermatic/machine-controller/pkg/machines/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/node" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1/migrations" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/cloudprovider/util" + clusterinfo "k8c.io/machine-controller/pkg/clusterinfo" + machinecontroller "k8c.io/machine-controller/pkg/controller/machine" + machinedeploymentcontroller "k8c.io/machine-controller/pkg/controller/machinedeployment" + machinesetcontroller "k8c.io/machine-controller/pkg/controller/machineset" + "k8c.io/machine-controller/pkg/controller/nodecsrapprover" + "k8c.io/machine-controller/pkg/health" + machinecontrollerlog "k8c.io/machine-controller/pkg/log" + machinesv1alpha1 "k8c.io/machine-controller/pkg/machines/v1alpha1" + "k8c.io/machine-controller/pkg/node" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/types" diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index aecc45282..a041e93e2 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -24,10 +24,10 @@ import ( "github.com/go-logr/zapr" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/admission" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - machinecontrollerlog "github.com/kubermatic/machine-controller/pkg/log" - "github.com/kubermatic/machine-controller/pkg/node" + "k8c.io/machine-controller/pkg/admission" + "k8c.io/machine-controller/pkg/cloudprovider/util" + machinecontrollerlog "k8c.io/machine-controller/pkg/log" + "k8c.io/machine-controller/pkg/node" "k8s.io/client-go/tools/clientcmd" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/docs/howto-provider.md b/docs/howto-provider.md index 886620ae3..b6bc583d1 100644 --- a/docs/howto-provider.md +++ b/docs/howto-provider.md @@ -6,7 +6,7 @@ ### Interface description -The interface a cloud provider has to implement is located in the package `github.com/kubermatic/machine-controller/pkg/cloudprovider/cloud`. It is named `Provider` and defines a small set of functions: +The interface a cloud provider has to implement is located in the package `k8c.io/machine-controller/pkg/cloudprovider/cloud`. It is named `Provider` and defines a small set of functions: ```go AddDefaults(spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) @@ -26,7 +26,7 @@ Get(machine *v1alpha1.Machine) (instance.Instance, error) `Get` gets a node that is associated with the given machine. Note that this method can return a so called _terminal error_, which indicates that a manual interaction is required to recover from this state. See `v1alpha1.MachineStatus` for more info and `errors.TerminalError` type. -In case the instance cannot be found, the returned error has to be `github.com/kubermatic/machine-controller/pkg/cloudprovider/errors.ErrInstanceNotFound` for proper evaluation by the machine controller. +In case the instance cannot be found, the returned error has to be `k8c.io/machine-controller/pkg/cloudprovider/errors.ErrInstanceNotFound` for proper evaluation by the machine controller. ```go Create(machine *v1alpha1.Machine, data *cloud.MachineCreateDeleteData, userdata string) (instance.Instance, error) @@ -60,7 +60,7 @@ SetMetricsForMachines(machines v1alpha1.MachineList) error ### Implementation hints -Provider implementations are located in individual packages in `github.com/kubermatic/machine-controller/pkg/cloudprovider/provider`. Here see e.g. `hetzner` as a straight and good understandable implementation. Other implementations are there too, helping to understand the needed tasks inside and around the `Provider` interface implementation. +Provider implementations are located in individual packages in `k8c.io/machine-controller/pkg/cloudprovider/provider`. Here see e.g. `hetzner` as a straight and good understandable implementation. Other implementations are there too, helping to understand the needed tasks inside and around the `Provider` interface implementation. When retrieving the individual configuration from the provider specification a type for unmarshalling is needed. Here first the provider configuration is read and based on it the individual values of the configuration are retrieved. Typically the access data (token, ID/key combination, document with all information) alternatively can be passed via an environment variable. According methods of the used `providerconfig.ConfigVarResolver` do support this. @@ -69,15 +69,15 @@ For creation of new machines the support of the possible information has to be c ## Integrate provider into the Machine Controller -For each cloud provider a unique string constant has to be defined in file `types.go` in package `github.com/kubermatic/machine-controller/pkg/providerconfig`. Registration based on this constant is done in file `provider.go` in package `github.com/kubermatic/machine-controller/pkg/cloudprovider`. +For each cloud provider a unique string constant has to be defined in file `types.go` in package `k8c.io/machine-controller/pkg/providerconfig`. Registration based on this constant is done in file `provider.go` in package `k8c.io/machine-controller/pkg/cloudprovider`. ## Add example manifest -For documentation of the different configuration options an according example manifest with helpful comments has to be added to `github.com/kubermatic/machine-controller/examples`. Naming scheme is `-machinedeployment.yaml`. +For documentation of the different configuration options an according example manifest with helpful comments has to be added to `k8c.io/machine-controller/examples`. Naming scheme is `-machinedeployment.yaml`. ## Integrate provider into CI -Like the example manifest a more concrete one named `machinedeployment-.yaml` has to be added to `github.com/kubermatic/machine-controller/test/e2e/provisioning/testdata`. Additionally file `all_e2e_test.go` in package `github.com/kubermatic/machine-controller/test/e2e/provisioning` contains all provider tests. Like the existing ones the test for the new provider has to be placed here. Mainly it's the retrieval of test data, especially the access data, from the environment and the starting of the test scenarios. +Like the example manifest a more concrete one named `machinedeployment-.yaml` has to be added to `k8c.io/machine-controller/test/e2e/provisioning/testdata`. Additionally file `all_e2e_test.go` in package `k8c.io/machine-controller/test/e2e/provisioning` contains all provider tests. Like the existing ones the test for the new provider has to be placed here. Mainly it's the retrieval of test data, especially the access data, from the environment and the starting of the test scenarios. Now the provider is ready to be added into the project for CI tests. diff --git a/go.mod b/go.mod index edaed7260..5fd961c75 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/kubermatic/machine-controller +module k8c.io/machine-controller go 1.22.0 diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go index 0203c2a96..ddb8b486b 100644 --- a/pkg/admission/admission.go +++ b/pkg/admission/admission.go @@ -31,8 +31,8 @@ import ( "go.uber.org/zap" "gomodules.xyz/jsonpatch/v2" - machinecontroller "github.com/kubermatic/machine-controller/pkg/controller/machine" - "github.com/kubermatic/machine-controller/pkg/node" + machinecontroller "k8c.io/machine-controller/pkg/controller/machine" + "k8c.io/machine-controller/pkg/node" admissionv1 "k8s.io/api/admission/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" diff --git a/pkg/admission/machinedeployments.go b/pkg/admission/machinedeployments.go index 7b0b0585a..c380a2b6e 100644 --- a/pkg/admission/machinedeployments.go +++ b/pkg/admission/machinedeployments.go @@ -21,7 +21,7 @@ import ( "encoding/json" "fmt" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" admissionv1 "k8s.io/api/admission/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" diff --git a/pkg/admission/machinedeployments_test.go b/pkg/admission/machinedeployments_test.go index 865be9151..d33146359 100644 --- a/pkg/admission/machinedeployments_test.go +++ b/pkg/admission/machinedeployments_test.go @@ -19,7 +19,7 @@ package admission import ( "testing" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/pkg/admission/machinedeployments_validation.go b/pkg/admission/machinedeployments_validation.go index dd4a9c1d6..e2ae84463 100644 --- a/pkg/admission/machinedeployments_validation.go +++ b/pkg/admission/machinedeployments_validation.go @@ -20,9 +20,9 @@ import ( "encoding/json" "fmt" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index 379fe0a6e..b564176fc 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -24,11 +24,11 @@ import ( "github.com/Masterminds/semver/v3" "golang.org/x/crypto/ssh" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" admissionv1 "k8s.io/api/admission/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" diff --git a/pkg/admission/util.go b/pkg/admission/util.go index 8e95017a4..83124ed4b 100644 --- a/pkg/admission/util.go +++ b/pkg/admission/util.go @@ -20,7 +20,7 @@ import ( "encoding/json" "fmt" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) const cloudProviderPacket = "packet" diff --git a/pkg/apis/cluster/v1alpha1/conversions/conversions.go b/pkg/apis/cluster/v1alpha1/conversions/conversions.go index d33350b8e..d5a729f9b 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/conversions.go +++ b/pkg/apis/cluster/v1alpha1/conversions/conversions.go @@ -20,8 +20,8 @@ import ( "encoding/json" "fmt" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - machinesv1alpha1 "github.com/kubermatic/machine-controller/pkg/machines/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + machinesv1alpha1 "k8c.io/machine-controller/pkg/machines/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" @@ -36,7 +36,7 @@ func Convert_MachinesV1alpha1Machine_To_ClusterV1alpha1Machine(in *machinesv1alp out.CreationTimestamp = metav1.Time{} out.ObjectMeta.Namespace = metav1.NamespaceSystem - // github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1.MachineStatus and + // k8c.io/machine-controller/pkg/apis/cluster/v1alpha1.MachineStatus and // pkg/machines/v1alpha1.MachineStatus are semantically identical, the former // only has one additional field, so we cast by serializing and deserializing inStatusJSON, err := json.Marshal(in.Status) diff --git a/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go b/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go index 3ba9ce84e..563b2cdd8 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go +++ b/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go @@ -23,8 +23,8 @@ import ( "os" "testing" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - machinesv1alpha1 "github.com/kubermatic/machine-controller/pkg/machines/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + machinesv1alpha1 "k8c.io/machine-controller/pkg/machines/v1alpha1" kyaml "k8s.io/apimachinery/pkg/util/yaml" "sigs.k8s.io/yaml" diff --git a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec.go b/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec.go index d48b19462..eca0097e6 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec.go +++ b/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec.go @@ -20,7 +20,7 @@ import ( "encoding/json" "fmt" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go b/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go index 902a11cd2..56f906404 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go +++ b/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go @@ -22,7 +22,7 @@ import ( "os" "testing" - testhelper "github.com/kubermatic/machine-controller/pkg/test" + testhelper "k8c.io/machine-controller/pkg/test" "sigs.k8s.io/yaml" ) diff --git a/pkg/apis/cluster/v1alpha1/defaults.go b/pkg/apis/cluster/v1alpha1/defaults.go index 33be49315..ad61b3313 100644 --- a/pkg/apis/cluster/v1alpha1/defaults.go +++ b/pkg/apis/cluster/v1alpha1/defaults.go @@ -17,7 +17,7 @@ limitations under the License. package v1alpha1 import ( - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" + "k8c.io/machine-controller/pkg/apis/cluster/common" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" diff --git a/pkg/apis/cluster/v1alpha1/doc.go b/pkg/apis/cluster/v1alpha1/doc.go index 1d1025365..3a530de12 100644 --- a/pkg/apis/cluster/v1alpha1/doc.go +++ b/pkg/apis/cluster/v1alpha1/doc.go @@ -17,7 +17,7 @@ limitations under the License. // Package v1alpha1 contains API Schema definitions for the cluster v1alpha1 API group // +k8s:openapi-gen=true // +k8s:deepcopy-gen=package,register -// +k8s:conversion-gen=github.com/kubermatic/machine-controller/pkg/apis/cluster +// +k8s:conversion-gen=k8c.io/machine-controller/pkg/apis/cluster // +k8s:defaulter-gen=TypeMeta // +groupName=cluster.k8s.io package v1alpha1 diff --git a/pkg/apis/cluster/v1alpha1/machine_types.go b/pkg/apis/cluster/v1alpha1/machine_types.go index 4295c340e..01bdb8fb8 100644 --- a/pkg/apis/cluster/v1alpha1/machine_types.go +++ b/pkg/apis/cluster/v1alpha1/machine_types.go @@ -17,7 +17,7 @@ limitations under the License. package v1alpha1 import ( - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" + "k8c.io/machine-controller/pkg/apis/cluster/common" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/apis/cluster/v1alpha1/machinedeployment_types.go b/pkg/apis/cluster/v1alpha1/machinedeployment_types.go index 68aa5410d..110dc9175 100644 --- a/pkg/apis/cluster/v1alpha1/machinedeployment_types.go +++ b/pkg/apis/cluster/v1alpha1/machinedeployment_types.go @@ -17,7 +17,7 @@ limitations under the License. package v1alpha1 import ( - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" + "k8c.io/machine-controller/pkg/apis/cluster/common" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" diff --git a/pkg/apis/cluster/v1alpha1/machineset_types.go b/pkg/apis/cluster/v1alpha1/machineset_types.go index dadf49d07..2bce745e5 100644 --- a/pkg/apis/cluster/v1alpha1/machineset_types.go +++ b/pkg/apis/cluster/v1alpha1/machineset_types.go @@ -19,7 +19,7 @@ package v1alpha1 import ( "log" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" + "k8c.io/machine-controller/pkg/apis/cluster/common" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" diff --git a/pkg/apis/cluster/v1alpha1/migrations/migrations.go b/pkg/apis/cluster/v1alpha1/migrations/migrations.go index 08b70244a..f2e3e2f3a 100644 --- a/pkg/apis/cluster/v1alpha1/migrations/migrations.go +++ b/pkg/apis/cluster/v1alpha1/migrations/migrations.go @@ -24,16 +24,16 @@ import ( "go.uber.org/zap" - machinecontrolleradmission "github.com/kubermatic/machine-controller/pkg/admission" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1/conversions" - "github.com/kubermatic/machine-controller/pkg/cloudprovider" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - machinecontroller "github.com/kubermatic/machine-controller/pkg/controller/machine" - "github.com/kubermatic/machine-controller/pkg/machines" - machinesv1alpha1 "github.com/kubermatic/machine-controller/pkg/machines/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + machinecontrolleradmission "k8c.io/machine-controller/pkg/admission" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1/conversions" + "k8c.io/machine-controller/pkg/cloudprovider" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + machinecontroller "k8c.io/machine-controller/pkg/controller/machine" + "k8c.io/machine-controller/pkg/machines" + machinesv1alpha1 "k8c.io/machine-controller/pkg/machines/v1alpha1" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" diff --git a/pkg/apis/cluster/v1alpha1/register.go b/pkg/apis/cluster/v1alpha1/register.go index ce0eeac3e..f6fd7b6db 100644 --- a/pkg/apis/cluster/v1alpha1/register.go +++ b/pkg/apis/cluster/v1alpha1/register.go @@ -19,7 +19,7 @@ limitations under the License. // Package v1alpha1 contains API Schema definitions for the cluster v1alpha1 API group // +k8s:openapi-gen=true // +k8s:deepcopy-gen=package,register -// +k8s:conversion-gen=github.com/kubermatic/machine-controller/pkg/apis/cluster +// +k8s:conversion-gen=k8c.io/machine-controller/pkg/apis/cluster // +k8s:defaulter-gen=TypeMeta // +groupName=cluster.k8s.io package v1alpha1 diff --git a/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go index 9c64da837..e9a7bd9eb 100644 --- a/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ limitations under the License. package v1alpha1 import ( - common "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" + common "k8c.io/machine-controller/pkg/apis/cluster/common" v1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" intstr "k8s.io/apimachinery/pkg/util/intstr" diff --git a/pkg/cloudprovider/cache/cloudprovidercache.go b/pkg/cloudprovider/cache/cloudprovidercache.go index 7b2c576fe..8254ffd1a 100644 --- a/pkg/cloudprovider/cache/cloudprovidercache.go +++ b/pkg/cloudprovider/cache/cloudprovidercache.go @@ -24,7 +24,7 @@ import ( gocache "github.com/patrickmn/go-cache" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" ) type CloudproviderCache struct { diff --git a/pkg/cloudprovider/cache/cloudprovidercache_test.go b/pkg/cloudprovider/cache/cloudprovidercache_test.go index 1f948642c..9d8b0d7eb 100644 --- a/pkg/cloudprovider/cache/cloudprovidercache_test.go +++ b/pkg/cloudprovider/cache/cloudprovidercache_test.go @@ -20,7 +20,7 @@ import ( "errors" "testing" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/cloudprovider/errors/errors.go b/pkg/cloudprovider/errors/errors.go index d0df77409..81b9c55ea 100644 --- a/pkg/cloudprovider/errors/errors.go +++ b/pkg/cloudprovider/errors/errors.go @@ -20,7 +20,7 @@ import ( "errors" "fmt" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" + "k8c.io/machine-controller/pkg/apis/cluster/common" ) var ( diff --git a/pkg/cloudprovider/provider.go b/pkg/cloudprovider/provider.go index 4cd794157..23ea1671b 100644 --- a/pkg/cloudprovider/provider.go +++ b/pkg/cloudprovider/provider.go @@ -19,30 +19,30 @@ package cloudprovider import ( "errors" - cloudprovidercache "github.com/kubermatic/machine-controller/pkg/cloudprovider/cache" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/alibaba" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/aws" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/azure" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/digitalocean" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/edge" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/equinixmetal" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/fake" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/gce" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/hetzner" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/kubevirt" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/linode" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/nutanix" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/opennebula" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/openstack" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/scaleway" - vcd "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vmwareclouddirector" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vsphere" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vultr" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + cloudprovidercache "k8c.io/machine-controller/pkg/cloudprovider/cache" + "k8c.io/machine-controller/pkg/cloudprovider/provider/alibaba" + "k8c.io/machine-controller/pkg/cloudprovider/provider/anexia" + "k8c.io/machine-controller/pkg/cloudprovider/provider/aws" + "k8c.io/machine-controller/pkg/cloudprovider/provider/azure" + "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal" + "k8c.io/machine-controller/pkg/cloudprovider/provider/digitalocean" + "k8c.io/machine-controller/pkg/cloudprovider/provider/edge" + "k8c.io/machine-controller/pkg/cloudprovider/provider/equinixmetal" + "k8c.io/machine-controller/pkg/cloudprovider/provider/fake" + "k8c.io/machine-controller/pkg/cloudprovider/provider/gce" + "k8c.io/machine-controller/pkg/cloudprovider/provider/hetzner" + "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt" + "k8c.io/machine-controller/pkg/cloudprovider/provider/linode" + "k8c.io/machine-controller/pkg/cloudprovider/provider/nutanix" + "k8c.io/machine-controller/pkg/cloudprovider/provider/opennebula" + "k8c.io/machine-controller/pkg/cloudprovider/provider/openstack" + "k8c.io/machine-controller/pkg/cloudprovider/provider/scaleway" + vcd "k8c.io/machine-controller/pkg/cloudprovider/provider/vmwareclouddirector" + "k8c.io/machine-controller/pkg/cloudprovider/provider/vsphere" + "k8c.io/machine-controller/pkg/cloudprovider/provider/vultr" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) var ( diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index 0a01c6695..9f7f6a5bc 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -27,16 +27,16 @@ import ( "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - alibabatypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/alibaba/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - kuberneteshelper "github.com/kubermatic/machine-controller/pkg/kubernetes" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + alibabatypes "k8c.io/machine-controller/pkg/cloudprovider/provider/alibaba/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/cloudprovider/util" + kuberneteshelper "k8c.io/machine-controller/pkg/kubernetes" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cloudprovider/provider/alibaba/types/types.go b/pkg/cloudprovider/provider/alibaba/types/types.go index 9e58cd401..10d1022ef 100644 --- a/pkg/cloudprovider/provider/alibaba/types/types.go +++ b/pkg/cloudprovider/provider/alibaba/types/types.go @@ -17,8 +17,8 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) type RawConfig struct { diff --git a/pkg/cloudprovider/provider/anexia/helper_test.go b/pkg/cloudprovider/provider/anexia/helper_test.go index eeb4000cf..2936ccb53 100644 --- a/pkg/cloudprovider/provider/anexia/helper_test.go +++ b/pkg/cloudprovider/provider/anexia/helper_test.go @@ -22,12 +22,12 @@ import ( "github.com/gophercloud/gophercloud/testhelper" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + anxtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/anexia/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/providerconfig/types" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/cloudprovider/provider/anexia/instance.go b/pkg/cloudprovider/provider/anexia/instance.go index 0c8343b1f..8af72f93c 100644 --- a/pkg/cloudprovider/provider/anexia/instance.go +++ b/pkg/cloudprovider/provider/anexia/instance.go @@ -21,8 +21,8 @@ import ( "go.anx.io/go-anxcloud/pkg/vsphere/info" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + anxtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/anexia/types" v1 "k8s.io/api/core/v1" ) diff --git a/pkg/cloudprovider/provider/anexia/network_provisioning.go b/pkg/cloudprovider/provider/anexia/network_provisioning.go index 277685d30..963fb6f28 100644 --- a/pkg/cloudprovider/provider/anexia/network_provisioning.go +++ b/pkg/cloudprovider/provider/anexia/network_provisioning.go @@ -21,12 +21,13 @@ import ( "sync" "time" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" anxclient "go.anx.io/go-anxcloud/pkg/client" anxaddr "go.anx.io/go-anxcloud/pkg/ipam/address" anxvm "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" "go.uber.org/zap" + + "k8c.io/machine-controller/pkg/apis/cluster/common" + anxtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/anexia/types" ) func networkInterfacesForProvisioning(ctx context.Context, log *zap.SugaredLogger, client anxclient.Client) ([]anxvm.Network, error) { diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 4581fbc20..514c6d6a6 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -34,16 +34,16 @@ import ( anxvm "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - cloudproviderutil "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/common/ssh" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + anxtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/anexia/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + cloudproviderutil "k8c.io/machine-controller/pkg/cloudprovider/util" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" "k8s.io/apimachinery/pkg/api/meta" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index d26546fad..61b6e4ff6 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -41,11 +41,11 @@ import ( "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + anxtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/anexia/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/cloudprovider/provider/anexia/reconcile_context.go b/pkg/cloudprovider/provider/anexia/reconcile_context.go index 2a14d8724..276851a7f 100644 --- a/pkg/cloudprovider/provider/anexia/reconcile_context.go +++ b/pkg/cloudprovider/provider/anexia/reconcile_context.go @@ -19,10 +19,10 @@ package anexia import ( "context" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + anxtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/anexia/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) type contextKey byte diff --git a/pkg/cloudprovider/provider/anexia/resolve_config.go b/pkg/cloudprovider/provider/anexia/resolve_config.go index fbe146dcf..7dddd112a 100644 --- a/pkg/cloudprovider/provider/anexia/resolve_config.go +++ b/pkg/cloudprovider/provider/anexia/resolve_config.go @@ -26,7 +26,7 @@ import ( corev1 "go.anx.io/go-anxcloud/pkg/apis/core/v1" vspherev1 "go.anx.io/go-anxcloud/pkg/apis/vsphere/v1" - anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" + anxtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/anexia/types" ) // resolvedDisk contains the resolved values from types.RawDisk. diff --git a/pkg/cloudprovider/provider/anexia/types/types.go b/pkg/cloudprovider/provider/anexia/types/types.go index 8b7bfdff3..4a6788bc8 100644 --- a/pkg/cloudprovider/provider/anexia/types/types.go +++ b/pkg/cloudprovider/provider/anexia/types/types.go @@ -19,10 +19,10 @@ package types import ( "time" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 26acb7c42..0da17a53b 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -38,16 +38,16 @@ import ( "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - awstypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/aws/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - "github.com/kubermatic/machine-controller/pkg/userdata/convert" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + awstypes "k8c.io/machine-controller/pkg/cloudprovider/provider/aws/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/cloudprovider/util" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/userdata/convert" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/cloudprovider/provider/aws/types/types.go b/pkg/cloudprovider/provider/aws/types/types.go index 243eb209c..dae0a56fa 100644 --- a/pkg/cloudprovider/provider/aws/types/types.go +++ b/pkg/cloudprovider/provider/aws/types/types.go @@ -17,8 +17,8 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) type RawConfig struct { diff --git a/pkg/cloudprovider/provider/azure/create_delete_resources.go b/pkg/cloudprovider/provider/azure/create_delete_resources.go index f98c76304..9b57d9d50 100644 --- a/pkg/cloudprovider/provider/azure/create_delete_resources.go +++ b/pkg/cloudprovider/provider/azure/create_delete_resources.go @@ -26,7 +26,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" + "k8c.io/machine-controller/pkg/cloudprovider/util" "k8s.io/apimachinery/pkg/types" ) diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index f5ec24fa7..1289c3aa2 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -33,17 +33,17 @@ import ( gocache "github.com/patrickmn/go-cache" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - azuretypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/azure/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - kuberneteshelper "github.com/kubermatic/machine-controller/pkg/kubernetes" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/common/ssh" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + azuretypes "k8c.io/machine-controller/pkg/cloudprovider/provider/azure/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/cloudprovider/util" + kuberneteshelper "k8c.io/machine-controller/pkg/kubernetes" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cloudprovider/provider/azure/types/types.go b/pkg/cloudprovider/provider/azure/types/types.go index 7b472689e..12746fb32 100644 --- a/pkg/cloudprovider/provider/azure/types/types.go +++ b/pkg/cloudprovider/provider/azure/types/types.go @@ -17,8 +17,8 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) // RawConfig is a direct representation of an Azure machine object's configuration. diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/hardware.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/hardware.go index f8a39362d..e9ce4695f 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/hardware.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/hardware.go @@ -20,9 +20,9 @@ import ( "context" "fmt" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - tbtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/errors" + tbtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go index cee68a942..06c0e9a55 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go @@ -22,7 +22,7 @@ import ( "fmt" "time" - tink "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" + tink "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" kerrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go index 08f44f12e..ea95e78a6 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go @@ -20,16 +20,17 @@ import ( "context" "encoding/base64" "fmt" + "github.com/aws/smithy-go/ptr" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "go.uber.org/zap" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client" - tinktypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins" + "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client" + tinktypes "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/hardware.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/hardware.go index 4e124b279..b52434f35 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/hardware.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/hardware.go @@ -17,8 +17,8 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins" tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins" ) type Hardware struct { diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/types.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/types.go index fe5d64bed..012300cc5 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/types.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/types.go @@ -17,7 +17,7 @@ limitations under the License. package types import ( - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" ) diff --git a/pkg/cloudprovider/provider/baremetal/provider.go b/pkg/cloudprovider/provider/baremetal/provider.go index 0e7cba678..f4e18ed1f 100644 --- a/pkg/cloudprovider/provider/baremetal/provider.go +++ b/pkg/cloudprovider/provider/baremetal/provider.go @@ -24,18 +24,18 @@ import ( "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins" - tink "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell" - tinktypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" - baremetaltypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/baremetal/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins" + tink "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell" + tinktypes "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" + baremetaltypes "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/cloudprovider/util" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/cloudprovider/provider/baremetal/types/types.go b/pkg/cloudprovider/provider/baremetal/types/types.go index 028477859..80feebb40 100644 --- a/pkg/cloudprovider/provider/baremetal/types/types.go +++ b/pkg/cloudprovider/provider/baremetal/types/types.go @@ -17,8 +17,8 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" "k8s.io/apimachinery/pkg/runtime" ) diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index d7e9c9cfd..af82ff6e3 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -28,16 +28,16 @@ import ( "go.uber.org/zap" "golang.org/x/oauth2" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - digitaloceantypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/digitalocean/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/common/ssh" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + digitaloceantypes "k8c.io/machine-controller/pkg/cloudprovider/provider/digitalocean/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/cloudprovider/util" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cloudprovider/provider/digitalocean/types/types.go b/pkg/cloudprovider/provider/digitalocean/types/types.go index a0fdb6830..89175c734 100644 --- a/pkg/cloudprovider/provider/digitalocean/types/types.go +++ b/pkg/cloudprovider/provider/digitalocean/types/types.go @@ -17,8 +17,8 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) type RawConfig struct { diff --git a/pkg/cloudprovider/provider/edge/provider.go b/pkg/cloudprovider/provider/edge/provider.go index 9c4bd2ff9..ed3b2cd98 100644 --- a/pkg/cloudprovider/provider/edge/provider.go +++ b/pkg/cloudprovider/provider/edge/provider.go @@ -21,10 +21,10 @@ import ( "go.uber.org/zap" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/providerconfig" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index 1d197a76e..b4f38f5ae 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -27,14 +27,14 @@ import ( "github.com/packethost/packngo" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - equinixmetaltypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/equinixmetal/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + equinixmetaltypes "k8c.io/machine-controller/pkg/cloudprovider/provider/equinixmetal/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/cloudprovider/provider/equinixmetal/types/types.go b/pkg/cloudprovider/provider/equinixmetal/types/types.go index b34625af0..3cbcbb7dd 100644 --- a/pkg/cloudprovider/provider/equinixmetal/types/types.go +++ b/pkg/cloudprovider/provider/equinixmetal/types/types.go @@ -17,8 +17,8 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) type RawConfig struct { diff --git a/pkg/cloudprovider/provider/fake/provider.go b/pkg/cloudprovider/provider/fake/provider.go index 67afd2993..219050aff 100644 --- a/pkg/cloudprovider/provider/fake/provider.go +++ b/pkg/cloudprovider/provider/fake/provider.go @@ -23,11 +23,11 @@ import ( "go.uber.org/zap" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index e23c78b58..739587d44 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -32,10 +32,10 @@ import ( googleoauth "golang.org/x/oauth2/google" "google.golang.org/api/compute/v1" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - gcetypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/gce/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + gcetypes "k8c.io/machine-controller/pkg/cloudprovider/provider/gce/types" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) // Environment variables for the configuration of the Google Cloud project access. diff --git a/pkg/cloudprovider/provider/gce/instance.go b/pkg/cloudprovider/provider/gce/instance.go index 2b6476195..139556775 100644 --- a/pkg/cloudprovider/provider/gce/instance.go +++ b/pkg/cloudprovider/provider/gce/instance.go @@ -26,7 +26,7 @@ import ( "google.golang.org/api/compute/v1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" + "k8c.io/machine-controller/pkg/cloudprovider/instance" v1 "k8s.io/api/core/v1" ) diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 3e58ac482..1f18468ad 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -33,13 +33,13 @@ import ( compute "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - "github.com/kubermatic/machine-controller/pkg/providerconfig" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/cloudprovider/util" + "k8c.io/machine-controller/pkg/providerconfig" "k8s.io/apimachinery/pkg/types" ) diff --git a/pkg/cloudprovider/provider/gce/provider_test.go b/pkg/cloudprovider/provider/gce/provider_test.go index cf3de6de7..13ccf5368 100644 --- a/pkg/cloudprovider/provider/gce/provider_test.go +++ b/pkg/cloudprovider/provider/gce/provider_test.go @@ -26,8 +26,8 @@ import ( "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/providerconfig" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/providerconfig" "k8s.io/apimachinery/pkg/runtime" fake2 "sigs.k8s.io/controller-runtime/pkg/client/fake" diff --git a/pkg/cloudprovider/provider/gce/types/types.go b/pkg/cloudprovider/provider/gce/types/types.go index 096bc3f88..bb99621fa 100644 --- a/pkg/cloudprovider/provider/gce/types/types.go +++ b/pkg/cloudprovider/provider/gce/types/types.go @@ -20,9 +20,9 @@ import ( "encoding/json" "fmt" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" "k8s.io/apimachinery/pkg/runtime" ) diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 61fdb2284..8b613b007 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -27,15 +27,15 @@ import ( "github.com/hetznercloud/hcloud-go/v2/hcloud" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - hetznertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/hetzner/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/common/ssh" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + hetznertypes "k8c.io/machine-controller/pkg/cloudprovider/provider/hetzner/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cloudprovider/provider/hetzner/types/types.go b/pkg/cloudprovider/provider/hetzner/types/types.go index 1148497ae..0312d2a45 100644 --- a/pkg/cloudprovider/provider/hetzner/types/types.go +++ b/pkg/cloudprovider/provider/hetzner/types/types.go @@ -17,8 +17,8 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) type RawConfig struct { diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index b3cd9e109..3ae9f600a 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -31,16 +31,16 @@ import ( kubevirtv1 "kubevirt.io/api/core/v1" cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - kubevirttypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/kubevirt/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - netutil "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - controllerutil "github.com/kubermatic/machine-controller/pkg/controller/util" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + kubevirttypes "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + netutil "k8c.io/machine-controller/pkg/cloudprovider/util" + controllerutil "k8c.io/machine-controller/pkg/controller/util" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index 989296bab..0e69d6626 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -28,8 +28,8 @@ import ( kubevirtv1 "kubevirt.io/api/core/v1" cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" - cloudprovidertesting "github.com/kubermatic/machine-controller/pkg/cloudprovider/testing" - "github.com/kubermatic/machine-controller/pkg/providerconfig" + cloudprovidertesting "k8c.io/machine-controller/pkg/cloudprovider/testing" + "k8c.io/machine-controller/pkg/providerconfig" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index 0c854d4d9..a46fc88b3 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -19,8 +19,8 @@ package types import ( kubevirtv1 "kubevirt.io/api/core/v1" - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" ) diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index 628841414..f38e77948 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -32,15 +32,15 @@ import ( "go.uber.org/zap" "golang.org/x/oauth2" - common "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/common/ssh" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - linodetypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/linode/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + common "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/common/ssh" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + linodetypes "k8c.io/machine-controller/pkg/cloudprovider/provider/linode/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cloudprovider/provider/linode/types/types.go b/pkg/cloudprovider/provider/linode/types/types.go index f2f2b7ef3..955a8e3da 100644 --- a/pkg/cloudprovider/provider/linode/types/types.go +++ b/pkg/cloudprovider/provider/linode/types/types.go @@ -17,8 +17,8 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) type RawConfig struct { diff --git a/pkg/cloudprovider/provider/nutanix/client.go b/pkg/cloudprovider/provider/nutanix/client.go index 4394c3b48..205a525a1 100644 --- a/pkg/cloudprovider/provider/nutanix/client.go +++ b/pkg/cloudprovider/provider/nutanix/client.go @@ -28,10 +28,10 @@ import ( nutanixclient "github.com/nutanix-cloud-native/prism-go-client" nutanixv3 "github.com/nutanix-cloud-native/prism-go-client/v3" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - nutanixtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/nutanix/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + nutanixtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/nutanix/types" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 87f911220..e9a4d6f30 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -25,14 +25,14 @@ import ( "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - nutanixtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/nutanix/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + nutanixtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/nutanix/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" ktypes "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cloudprovider/provider/nutanix/types/types.go b/pkg/cloudprovider/provider/nutanix/types/types.go index a2283b721..e5fe32bde 100644 --- a/pkg/cloudprovider/provider/nutanix/types/types.go +++ b/pkg/cloudprovider/provider/nutanix/types/types.go @@ -17,8 +17,8 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) const ( diff --git a/pkg/cloudprovider/provider/opennebula/provider.go b/pkg/cloudprovider/provider/opennebula/provider.go index 3a2f65da3..da79637ff 100644 --- a/pkg/cloudprovider/provider/opennebula/provider.go +++ b/pkg/cloudprovider/provider/opennebula/provider.go @@ -31,14 +31,14 @@ import ( "github.com/OpenNebula/one/src/oca/go/src/goca/schemas/vm/keys" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - opennebulatypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/opennebula/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + opennebulatypes "k8c.io/machine-controller/pkg/cloudprovider/provider/opennebula/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cloudprovider/provider/opennebula/types/types.go b/pkg/cloudprovider/provider/opennebula/types/types.go index 6e69755fc..51b5f029f 100644 --- a/pkg/cloudprovider/provider/opennebula/types/types.go +++ b/pkg/cloudprovider/provider/opennebula/types/types.go @@ -17,8 +17,8 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) type RawConfig struct { diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 37d81d02c..861642155 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -36,15 +36,15 @@ import ( "github.com/gophercloud/gophercloud/pagination" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - openstacktypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/openstack/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - cloudproviderutil "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + openstacktypes "k8c.io/machine-controller/pkg/cloudprovider/provider/openstack/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + cloudproviderutil "k8c.io/machine-controller/pkg/cloudprovider/util" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index b7757b7e8..341791d9c 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -32,10 +32,10 @@ import ( "github.com/gophercloud/gophercloud/testhelper/client" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidertesting "github.com/kubermatic/machine-controller/pkg/cloudprovider/testing" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidertesting "k8c.io/machine-controller/pkg/cloudprovider/testing" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/providerconfig" "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/ptr" diff --git a/pkg/cloudprovider/provider/openstack/types/types.go b/pkg/cloudprovider/provider/openstack/types/types.go index 17aed61ee..8d5af36d4 100644 --- a/pkg/cloudprovider/provider/openstack/types/types.go +++ b/pkg/cloudprovider/provider/openstack/types/types.go @@ -17,8 +17,8 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) type RawConfig struct { diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go index ddd682704..632a6464f 100644 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ b/pkg/cloudprovider/provider/scaleway/provider.go @@ -27,14 +27,14 @@ import ( "github.com/scaleway/scaleway-sdk-go/validation" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - cloudInstance "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - scalewaytypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/scaleway/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + cloudInstance "k8c.io/machine-controller/pkg/cloudprovider/instance" + scalewaytypes "k8c.io/machine-controller/pkg/cloudprovider/provider/scaleway/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cloudprovider/provider/scaleway/types/types.go b/pkg/cloudprovider/provider/scaleway/types/types.go index 8ecbd5cb1..718f27f11 100644 --- a/pkg/cloudprovider/provider/scaleway/types/types.go +++ b/pkg/cloudprovider/provider/scaleway/types/types.go @@ -17,8 +17,8 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) type RawConfig struct { diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/client.go b/pkg/cloudprovider/provider/vmwareclouddirector/client.go index c7f6e7878..a628c660f 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/client.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/client.go @@ -26,7 +26,7 @@ import ( "github.com/vmware/go-vcloud-director/v2/govcd" "github.com/vmware/go-vcloud-director/v2/types/v56" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" ) type Client struct { diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go index bd11b5010..65338e64e 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go @@ -27,7 +27,7 @@ import ( "github.com/vmware/go-vcloud-director/v2/types/v56" vcdapitypes "github.com/vmware/go-vcloud-director/v2/types/v56" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8s.io/utils/ptr" ) diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go index 7d4888846..3479bfbe5 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -26,13 +26,13 @@ import ( "github.com/vmware/go-vcloud-director/v2/govcd" "go.uber.org/zap" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - vcdtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vmwareclouddirector/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + vcdtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/vmwareclouddirector/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go b/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go index 1b4cb7b6d..fe60277d3 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go @@ -17,8 +17,8 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) type IPAllocationMode string diff --git a/pkg/cloudprovider/provider/vsphere/client.go b/pkg/cloudprovider/provider/vsphere/client.go index e1e45b7e9..b706cef05 100644 --- a/pkg/cloudprovider/provider/vsphere/client.go +++ b/pkg/cloudprovider/provider/vsphere/client.go @@ -29,7 +29,7 @@ import ( "github.com/vmware/govmomi/vim25" "github.com/vmware/govmomi/vim25/soap" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" + "k8c.io/machine-controller/pkg/cloudprovider/util" utilruntime "k8s.io/apimachinery/pkg/util/runtime" ) diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index f5f9a1a0a..4bcd6686a 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -31,14 +31,14 @@ import ( "github.com/vmware/govmomi/vim25/types" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - vspheretypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vsphere/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + vspheretypes "k8c.io/machine-controller/pkg/cloudprovider/provider/vsphere/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" ktypes "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cloudprovider/provider/vsphere/provider_test.go b/pkg/cloudprovider/provider/vsphere/provider_test.go index 9a30be155..a3e15fef6 100644 --- a/pkg/cloudprovider/provider/vsphere/provider_test.go +++ b/pkg/cloudprovider/provider/vsphere/provider_test.go @@ -27,8 +27,8 @@ import ( "github.com/vmware/govmomi/simulator" "go.uber.org/zap" - cloudprovidertesting "github.com/kubermatic/machine-controller/pkg/cloudprovider/testing" - "github.com/kubermatic/machine-controller/pkg/providerconfig" + cloudprovidertesting "k8c.io/machine-controller/pkg/cloudprovider/testing" + "k8c.io/machine-controller/pkg/providerconfig" "k8s.io/utils/ptr" fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" diff --git a/pkg/cloudprovider/provider/vsphere/rule.go b/pkg/cloudprovider/provider/vsphere/rule.go index 98e713ba5..94408dc4b 100644 --- a/pkg/cloudprovider/provider/vsphere/rule.go +++ b/pkg/cloudprovider/provider/vsphere/rule.go @@ -29,7 +29,7 @@ import ( "github.com/vmware/govmomi/vim25/types" "go.uber.org/zap" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8s.io/utils/ptr" ) diff --git a/pkg/cloudprovider/provider/vsphere/types/types.go b/pkg/cloudprovider/provider/vsphere/types/types.go index 443cfa137..c8ab0c85a 100644 --- a/pkg/cloudprovider/provider/vsphere/types/types.go +++ b/pkg/cloudprovider/provider/vsphere/types/types.go @@ -17,8 +17,8 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) // RawConfig represents vsphere specific configuration. diff --git a/pkg/cloudprovider/provider/vsphere/vmgroup.go b/pkg/cloudprovider/provider/vsphere/vmgroup.go index 79c87428b..1b51a5468 100644 --- a/pkg/cloudprovider/provider/vsphere/vmgroup.go +++ b/pkg/cloudprovider/provider/vsphere/vmgroup.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/vmware/govmomi/vim25/types" "go.uber.org/zap" diff --git a/pkg/cloudprovider/provider/vultr/provider.go b/pkg/cloudprovider/provider/vultr/provider.go index 09e105c4f..f2b3c9698 100644 --- a/pkg/cloudprovider/provider/vultr/provider.go +++ b/pkg/cloudprovider/provider/vultr/provider.go @@ -29,14 +29,14 @@ import ( "go.uber.org/zap" "golang.org/x/oauth2" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - vultrtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vultr/types" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + vultrtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/vultr/types" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cloudprovider/provider/vultr/types/types.go b/pkg/cloudprovider/provider/vultr/types/types.go index 278ea6066..e016d6288 100644 --- a/pkg/cloudprovider/provider/vultr/types/types.go +++ b/pkg/cloudprovider/provider/vultr/types/types.go @@ -17,8 +17,8 @@ limitations under the License. package types import ( - "github.com/kubermatic/machine-controller/pkg/jsonutil" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/jsonutil" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) type RawConfig struct { diff --git a/pkg/cloudprovider/testing/testing.go b/pkg/cloudprovider/testing/testing.go index aae583cff..2069f4407 100644 --- a/pkg/cloudprovider/testing/testing.go +++ b/pkg/cloudprovider/testing/testing.go @@ -19,7 +19,7 @@ package testing import ( "testing" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/cloudprovider/types/types.go b/pkg/cloudprovider/types/types.go index 59e90d9d3..c8142d366 100644 --- a/pkg/cloudprovider/types/types.go +++ b/pkg/cloudprovider/types/types.go @@ -22,8 +22,8 @@ import ( "go.uber.org/zap" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/instance" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/types" @@ -48,7 +48,7 @@ type Provider interface { // which indicates that a manual interaction is required to recover from this state. // See v1alpha1.MachineStatus for more info and TerminalError type // - // In case the instance cannot be found, github.com/kubermatic/machine-controller/pkg/cloudprovider/errors/ErrInstanceNotFound will be returned + // In case the instance cannot be found, k8c.io/machine-controller/pkg/cloudprovider/errors/ErrInstanceNotFound will be returned Get(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *ProviderData) (instance.Instance, error) // Create creates a cloud instance according to the given machine diff --git a/pkg/cloudprovider/util/util.go b/pkg/cloudprovider/util/util.go index 63a15a83b..2bf06bdca 100644 --- a/pkg/cloudprovider/util/util.go +++ b/pkg/cloudprovider/util/util.go @@ -19,9 +19,9 @@ package util import ( "fmt" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - kuberneteshelper "github.com/kubermatic/machine-controller/pkg/kubernetes" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + kuberneteshelper "k8c.io/machine-controller/pkg/kubernetes" ) // RemoveFinalizerOnInstanceNotFound checks whether a finalizer exists and removes it on demand. diff --git a/pkg/cloudprovider/util/util_test.go b/pkg/cloudprovider/util/util_test.go index 1c15707bd..13734a75a 100644 --- a/pkg/cloudprovider/util/util_test.go +++ b/pkg/cloudprovider/util/util_test.go @@ -21,9 +21,9 @@ import ( "reflect" "testing" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cloudprovider/validationwrapper.go b/pkg/cloudprovider/validationwrapper.go index ba6390087..a3bb979a8 100644 --- a/pkg/cloudprovider/validationwrapper.go +++ b/pkg/cloudprovider/validationwrapper.go @@ -23,9 +23,9 @@ import ( "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" "k8s.io/apimachinery/pkg/types" ) diff --git a/pkg/controller/machine/controller.go b/pkg/controller/machine/controller.go index 2a24dd29c..87ed6849a 100644 --- a/pkg/controller/machine/controller.go +++ b/pkg/controller/machine/controller.go @@ -31,22 +31,22 @@ import ( "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/bootstrap" - "github.com/kubermatic/machine-controller/pkg/cloudprovider" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - controllerutil "github.com/kubermatic/machine-controller/pkg/controller/util" - kuberneteshelper "github.com/kubermatic/machine-controller/pkg/kubernetes" - "github.com/kubermatic/machine-controller/pkg/node/eviction" - "github.com/kubermatic/machine-controller/pkg/node/poddeletion" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - "github.com/kubermatic/machine-controller/pkg/rhsm" - "github.com/kubermatic/machine-controller/pkg/userdata/rhel" + "k8c.io/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/bootstrap" + "k8c.io/machine-controller/pkg/cloudprovider" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/cloudprovider/util" + controllerutil "k8c.io/machine-controller/pkg/controller/util" + kuberneteshelper "k8c.io/machine-controller/pkg/kubernetes" + "k8c.io/machine-controller/pkg/node/eviction" + "k8c.io/machine-controller/pkg/node/poddeletion" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/rhsm" + "k8c.io/machine-controller/pkg/userdata/rhel" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" diff --git a/pkg/controller/machine/controller_test.go b/pkg/controller/machine/controller_test.go index 0c98c04bc..f505c2434 100644 --- a/pkg/controller/machine/controller_test.go +++ b/pkg/controller/machine/controller_test.go @@ -25,10 +25,10 @@ import ( "github.com/go-test/deep" "go.uber.org/zap" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/controller/machine/metrics.go b/pkg/controller/machine/metrics.go index 8906279d6..b5f681bdb 100644 --- a/pkg/controller/machine/metrics.go +++ b/pkg/controller/machine/metrics.go @@ -23,10 +23,10 @@ import ( "github.com/prometheus/client_golang/prometheus" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" "k8s.io/apimachinery/pkg/api/equality" utilruntime "k8s.io/apimachinery/pkg/util/runtime" diff --git a/pkg/controller/machinedeployment/controller.go b/pkg/controller/machinedeployment/controller.go index 73530f52d..81753d7a4 100644 --- a/pkg/controller/machinedeployment/controller.go +++ b/pkg/controller/machinedeployment/controller.go @@ -25,8 +25,8 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/apis/cluster/common" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/controller/machinedeployment/rolling.go b/pkg/controller/machinedeployment/rolling.go index f4cb42676..000fb1ad6 100644 --- a/pkg/controller/machinedeployment/rolling.go +++ b/pkg/controller/machinedeployment/rolling.go @@ -23,8 +23,8 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - dutil "github.com/kubermatic/machine-controller/pkg/controller/util" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + dutil "k8c.io/machine-controller/pkg/controller/util" "k8s.io/utils/integer" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/controller/machinedeployment/sync.go b/pkg/controller/machinedeployment/sync.go index e9e9f071a..4c10bac48 100644 --- a/pkg/controller/machinedeployment/sync.go +++ b/pkg/controller/machinedeployment/sync.go @@ -26,8 +26,8 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - dutil "github.com/kubermatic/machine-controller/pkg/controller/util" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + dutil "k8c.io/machine-controller/pkg/controller/util" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/controller/machineset/controller.go b/pkg/controller/machineset/controller.go index 3290a56bd..000d37179 100644 --- a/pkg/controller/machineset/controller.go +++ b/pkg/controller/machineset/controller.go @@ -28,7 +28,7 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/controller/machineset/delete_policy.go b/pkg/controller/machineset/delete_policy.go index 44fee2cd3..c631a70e5 100644 --- a/pkg/controller/machineset/delete_policy.go +++ b/pkg/controller/machineset/delete_policy.go @@ -22,7 +22,7 @@ import ( "github.com/pkg/errors" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/pkg/controller/machineset/machine.go b/pkg/controller/machineset/machine.go index 8a3cb5d75..7c24891f0 100644 --- a/pkg/controller/machineset/machine.go +++ b/pkg/controller/machineset/machine.go @@ -21,7 +21,7 @@ import ( "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" diff --git a/pkg/controller/machineset/status.go b/pkg/controller/machineset/status.go index 0a207835c..0d724816b 100644 --- a/pkg/controller/machineset/status.go +++ b/pkg/controller/machineset/status.go @@ -23,7 +23,7 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/controller/nodecsrapprover/controller.go b/pkg/controller/nodecsrapprover/controller.go index 97ccb1357..b894ff46b 100644 --- a/pkg/controller/nodecsrapprover/controller.go +++ b/pkg/controller/nodecsrapprover/controller.go @@ -27,7 +27,7 @@ import ( "github.com/go-logr/zapr" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" certificatesv1 "k8s.io/api/certificates/v1" corev1 "k8s.io/api/core/v1" diff --git a/pkg/controller/nodecsrapprover/controller_test.go b/pkg/controller/nodecsrapprover/controller_test.go index cb71c523f..970045250 100644 --- a/pkg/controller/nodecsrapprover/controller_test.go +++ b/pkg/controller/nodecsrapprover/controller_test.go @@ -22,7 +22,7 @@ import ( "fmt" "testing" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" certificatesv1 "k8s.io/api/certificates/v1" corev1 "k8s.io/api/core/v1" diff --git a/pkg/controller/util/machine.go b/pkg/controller/util/machine.go index 69a2a833b..848308f9a 100644 --- a/pkg/controller/util/machine.go +++ b/pkg/controller/util/machine.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/controller/util/machine_deployment.go b/pkg/controller/util/machine_deployment.go index 081862b2e..a6b06365a 100644 --- a/pkg/controller/util/machine_deployment.go +++ b/pkg/controller/util/machine_deployment.go @@ -27,8 +27,8 @@ import ( "github.com/davecgh/go-spew/spew" "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/apis/cluster/common" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" v1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" diff --git a/pkg/health/readiness.go b/pkg/health/readiness.go index 8ec6176b6..85081fd42 100644 --- a/pkg/health/readiness.go +++ b/pkg/health/readiness.go @@ -23,7 +23,7 @@ import ( "go.uber.org/zap" - machinecontroller "github.com/kubermatic/machine-controller/pkg/controller/machine" + machinecontroller "k8c.io/machine-controller/pkg/controller/machine" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" diff --git a/pkg/machines/register.go b/pkg/machines/register.go index 64bca258e..76e3f862e 100644 --- a/pkg/machines/register.go +++ b/pkg/machines/register.go @@ -19,7 +19,7 @@ package machines import ( "reflect" - "github.com/kubermatic/machine-controller/pkg/machines/v1alpha1" + "k8c.io/machine-controller/pkg/machines/v1alpha1" ) type resource struct { diff --git a/pkg/node/eviction/eviction.go b/pkg/node/eviction/eviction.go index 9c2b6f1ec..740bafe87 100644 --- a/pkg/node/eviction/eviction.go +++ b/pkg/node/eviction/eviction.go @@ -23,8 +23,8 @@ import ( "go.uber.org/zap" - evictiontypes "github.com/kubermatic/machine-controller/pkg/node/eviction/types" - "github.com/kubermatic/machine-controller/pkg/node/nodemanager" + evictiontypes "k8c.io/machine-controller/pkg/node/eviction/types" + "k8c.io/machine-controller/pkg/node/nodemanager" corev1 "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1beta1" diff --git a/pkg/node/flags.go b/pkg/node/flags.go index ae8fe17f4..f101a3afb 100644 --- a/pkg/node/flags.go +++ b/pkg/node/flags.go @@ -22,7 +22,7 @@ import ( "strconv" "strings" - machinecontroller "github.com/kubermatic/machine-controller/pkg/controller/machine" + machinecontroller "k8c.io/machine-controller/pkg/controller/machine" ) func NewFlags(flagset *flag.FlagSet) *Flags { diff --git a/pkg/node/poddeletion/pod_deletion.go b/pkg/node/poddeletion/pod_deletion.go index af164fda5..bc3af41c4 100644 --- a/pkg/node/poddeletion/pod_deletion.go +++ b/pkg/node/poddeletion/pod_deletion.go @@ -23,7 +23,7 @@ import ( "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/node/nodemanager" + "k8c.io/machine-controller/pkg/node/nodemanager" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/pkg/providerconfig/types.go b/pkg/providerconfig/types.go index 511773c81..13d21b0a3 100644 --- a/pkg/providerconfig/types.go +++ b/pkg/providerconfig/types.go @@ -24,13 +24,13 @@ import ( "strconv" "time" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - "github.com/kubermatic/machine-controller/pkg/userdata/amzn2" - "github.com/kubermatic/machine-controller/pkg/userdata/centos" - "github.com/kubermatic/machine-controller/pkg/userdata/flatcar" - "github.com/kubermatic/machine-controller/pkg/userdata/rhel" - "github.com/kubermatic/machine-controller/pkg/userdata/rockylinux" - "github.com/kubermatic/machine-controller/pkg/userdata/ubuntu" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/userdata/amzn2" + "k8c.io/machine-controller/pkg/userdata/centos" + "k8c.io/machine-controller/pkg/userdata/flatcar" + "k8c.io/machine-controller/pkg/userdata/rhel" + "k8c.io/machine-controller/pkg/userdata/rockylinux" + "k8c.io/machine-controller/pkg/userdata/ubuntu" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 1a6a89640..5ca2a3169 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -23,9 +23,9 @@ import ( "fmt" "strconv" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - "github.com/kubermatic/machine-controller/pkg/jsonutil" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/util" + "k8c.io/machine-controller/pkg/jsonutil" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/providerconfig/types_test.go b/pkg/providerconfig/types_test.go index b9abc31ce..344e8ff26 100644 --- a/pkg/providerconfig/types_test.go +++ b/pkg/providerconfig/types_test.go @@ -19,7 +19,7 @@ package providerconfig import ( "testing" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" "k8s.io/apimachinery/pkg/runtime" ) diff --git a/pkg/rhsm/util.go b/pkg/rhsm/util.go index ed45078dc..6e8fcb7a9 100644 --- a/pkg/rhsm/util.go +++ b/pkg/rhsm/util.go @@ -17,9 +17,9 @@ limitations under the License. package rhsm import ( - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - kuberneteshelper "github.com/kubermatic/machine-controller/pkg/kubernetes" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/types" + kuberneteshelper "k8c.io/machine-controller/pkg/kubernetes" ) const ( diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index d940442b1..0aa013ed6 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -28,9 +28,9 @@ import ( "testing" "time" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - "github.com/kubermatic/machine-controller/pkg/userdata/flatcar" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/userdata/flatcar" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" diff --git a/test/e2e/provisioning/deploymentscenario.go b/test/e2e/provisioning/deploymentscenario.go index 12933296b..779e9bcad 100644 --- a/test/e2e/provisioning/deploymentscenario.go +++ b/test/e2e/provisioning/deploymentscenario.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index a947e17d9..ca42162e8 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -27,7 +27,7 @@ import ( "github.com/Masterminds/semver/v3" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" ) var ( diff --git a/test/e2e/provisioning/migrateuidscenario.go b/test/e2e/provisioning/migrateuidscenario.go index d846fa3e9..e443fc856 100644 --- a/test/e2e/provisioning/migrateuidscenario.go +++ b/test/e2e/provisioning/migrateuidscenario.go @@ -25,12 +25,12 @@ import ( "go.uber.org/zap" - "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/cloudprovider" - cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" - cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" - "github.com/kubermatic/machine-controller/pkg/providerconfig" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/providerconfig" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" diff --git a/test/e2e/provisioning/verify.go b/test/e2e/provisioning/verify.go index 1c74cae08..75e005db2 100644 --- a/test/e2e/provisioning/verify.go +++ b/test/e2e/provisioning/verify.go @@ -23,10 +23,10 @@ import ( "strings" "time" - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - machinecontroller "github.com/kubermatic/machine-controller/pkg/controller/machine" - evictiontypes "github.com/kubermatic/machine-controller/pkg/node/eviction/types" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + machinecontroller "k8c.io/machine-controller/pkg/controller/machine" + evictiontypes "k8c.io/machine-controller/pkg/node/eviction/types" + providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" From 93fb4f10595f3bca58afa0504e0a9988f9b07bfa Mon Sep 17 00:00:00 2001 From: Marques Johansson Date: Tue, 27 Aug 2024 04:01:19 -0400 Subject: [PATCH 418/489] add kubermatic to packngo User-Agent (#1848) Signed-off-by: Marques Johansson --- pkg/cloudprovider/provider/equinixmetal/provider.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index b4f38f5ae..445551d99 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -478,7 +478,9 @@ func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { } func getClient(apiKey string) *packngo.Client { - return packngo.NewClientWithAuth("kubermatic", apiKey, nil) + client := packngo.NewClientWithAuth("kubermatic", apiKey, nil) + client.UserAgent = fmt.Sprintf("kubermatic/machine-controller %s", client.UserAgent) + return client } func generateTag(ID string) string { From 64cc5c3b19c689e78a1d65201dba175f080afe89 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 27 Aug 2024 12:38:18 +0200 Subject: [PATCH 419/489] add navid as a maintainer (#1849) --- OWNERS_ALIASES | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index dc87eeb2a..2f38c4b67 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -10,3 +10,4 @@ aliases: - moelsayed - xmudrii - xrstf + - yaa110 From 62bda9721ad4c6e1d1666289a8b41d4fe2986e1d Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Wed, 28 Aug 2024 01:18:18 +0200 Subject: [PATCH 420/489] disable kubevirt tests and connectiton to dev is fixed (#1852) --- .prow/provider-kubevirt.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index f5463e98f..915fc2056 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -14,7 +14,8 @@ presubmits: - name: pull-machine-controller-e2e-kubevirt - run_if_changed: "(pkg/cloudprovider/provider/kubevirt/|pkg/userdata)" +# run_if_changed: "(pkg/cloudprovider/provider/kubevirt/|pkg/userdata)" + always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" path_alias: k8c.io/machine-controller From 24e281fdffffad7dca0fa75fccd4b42e31109ba3 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Wed, 28 Aug 2024 11:22:19 +0200 Subject: [PATCH 421/489] Add static up support for KubeVirt in KubeOVN (#1851) * add static up support for KubeVirt in KubeOVN * fix unit tests Signed-off-by: moadqassem --------- Signed-off-by: moadqassem --- pkg/cloudprovider/provider/kubevirt/provider.go | 8 ++++---- .../provider/kubevirt/testdata/affinity-no-values.yaml | 2 ++ .../provider/kubevirt/testdata/affinity.yaml | 2 ++ .../provider/kubevirt/testdata/custom-local-disk.yaml | 2 ++ .../provider/kubevirt/testdata/http-image-source.yaml | 2 ++ .../kubevirt/testdata/instancetype-preference-custom.yaml | 2 ++ .../testdata/instancetype-preference-standard.yaml | 2 ++ .../provider/kubevirt/testdata/nominal-case.yaml | 2 ++ .../provider/kubevirt/testdata/pvc-image-source.yaml | 2 ++ .../kubevirt/testdata/registry-image-source-pod.yaml | 2 ++ .../provider/kubevirt/testdata/registry-image-source.yaml | 2 ++ .../provider/kubevirt/testdata/secondary-disks.yaml | 2 ++ .../kubevirt/testdata/topologyspreadconstraints.yaml | 2 ++ 13 files changed, 28 insertions(+), 4 deletions(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 3ae9f600a..ef39118e1 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -637,17 +637,17 @@ func (p *provider) newVirtualMachine(_ context.Context, c *Config, pc *providerc var ( dataVolumeName = machine.Name - annotations map[string]string + annotations = map[string]string{} ) // Add machineName as prefix to secondaryDisks. addPrefixToSecondaryDisk(c.SecondaryDisks, dataVolumeName) if pc.OperatingSystem == providerconfigtypes.OperatingSystemFlatcar { - annotations = map[string]string{ - "kubevirt.io/ignitiondata": userdata, - } + annotations["kubevirt.io/ignitiondata"] = userdata } + annotations["ovn.kubernetes.io/allow_live_migration"] = "true" + defaultBridgeNetwork, err := defaultBridgeNetwork(macAddressGetter) if err != nil { return nil, fmt.Errorf("could not compute a random MAC address") diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml index 7d01a42c7..bac6e7a46 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml @@ -28,6 +28,8 @@ spec: runStrategy: Once template: metadata: + annotations: + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml index a28aded56..7f1889466 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml @@ -28,6 +28,8 @@ spec: runStrategy: Once template: metadata: + annotations: + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml index b77494b63..1d761053c 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml @@ -29,6 +29,8 @@ spec: template: metadata: creationTimestamp: null + annotations: + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml index caef1ed52..c5125c087 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml @@ -28,6 +28,8 @@ spec: template: metadata: creationTimestamp: null + annotations: + "ovn.kubernetes.io/allow_live_migration": "true" labels: kubevirt.io/vm: http-image-source cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml index ba699302e..a27fc8674 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml @@ -34,6 +34,8 @@ spec: name: custom-pref template: metadata: + annotations: + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml index 1f54c87a7..718e23d38 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml @@ -34,6 +34,8 @@ spec: template: metadata: creationTimestamp: null + annotations: + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml index 24afa309c..bb65126cf 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml @@ -28,6 +28,8 @@ spec: template: metadata: creationTimestamp: null + annotations: + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml index 978213ec3..20fd7a4ee 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml @@ -29,6 +29,8 @@ spec: template: metadata: creationTimestamp: null + annotations: + "ovn.kubernetes.io/allow_live_migration": "true" labels: kubevirt.io/vm: pvc-image-source cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml index 9a8115c1d..7c047af05 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml @@ -29,6 +29,8 @@ spec: template: metadata: creationTimestamp: null + annotations: + "ovn.kubernetes.io/allow_live_migration": "true" labels: kubevirt.io/vm: registry-image-source-pod cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml index ee0548b09..df5dedac2 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml @@ -29,6 +29,8 @@ spec: template: metadata: creationTimestamp: null + annotations: + "ovn.kubernetes.io/allow_live_migration": "true" labels: kubevirt.io/vm: registry-image-source cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml index 0ce57a4b7..bea429e6f 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml @@ -54,6 +54,8 @@ spec: template: metadata: creationTimestamp: null + annotations: + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml index 363460724..965429429 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml @@ -28,6 +28,8 @@ spec: template: metadata: creationTimestamp: null + annotations: + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker From c2e5055561d3a25ee45e4f9e485d184ab9c1aa5d Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Wed, 28 Aug 2024 14:29:20 +0200 Subject: [PATCH 422/489] copy annoations to VMI (#1853) --- pkg/cloudprovider/provider/kubevirt/provider.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index ef39118e1..388f5504d 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -648,6 +648,10 @@ func (p *provider) newVirtualMachine(_ context.Context, c *Config, pc *providerc annotations["ovn.kubernetes.io/allow_live_migration"] = "true" + for k, v := range machine.Annotations { + annotations[k] = v + } + defaultBridgeNetwork, err := defaultBridgeNetwork(macAddressGetter) if err != nil { return nil, fmt.Errorf("could not compute a random MAC address") From 4c08148545fa9d5acdae2718852a816319e40165 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Thu, 29 Aug 2024 15:56:20 +0500 Subject: [PATCH 423/489] Add sig-cluster-management as code owners (#1850) * Add sig-cluster-management as code owners Signed-off-by: Waleed Malik * Update OWNERS Signed-off-by: Waleed Malik --------- Signed-off-by: Waleed Malik --- OWNERS | 4 ++-- OWNERS_ALIASES | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/OWNERS b/OWNERS index 46fe4a023..74ccec94c 100644 --- a/OWNERS +++ b/OWNERS @@ -1,10 +1,10 @@ # See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md approvers: - - machine-controller-maintainers + - sig-cluster-management reviewers: - - machine-controller-maintainers + - sig-cluster-management labels: - sig/cluster-management diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 2f38c4b67..d604ea1c1 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -2,8 +2,9 @@ # To change team associations, update the GitHub teams via https://github.com/kubermatic/access. aliases: - machine-controller-maintainers: + sig-cluster-management: - ahmedwaleedmalik + - cnvergence - embik - kron4eg - moadqassem From 69383af8420c43e280a85fc8e35479a30a07971a Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Sun, 1 Sep 2024 22:36:23 +0200 Subject: [PATCH 424/489] Support Different Storage Targets for KubeVirt Provider (#1855) * use KV storage instead of PVC in datavolumes * support storage as a storage target for KubeVirt VMs * revert back the default network attachment * use ptr instead of pointer * add tests --- .../provider/kubevirt/provider.go | 63 ++++++++++++--- .../provider/kubevirt/provider_test.go | 7 ++ .../use-storage-as-storage-target.yaml | 80 +++++++++++++++++++ .../provider/kubevirt/types/types.go | 3 + 4 files changed, 140 insertions(+), 13 deletions(-) create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 388f5504d..021f200c9 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -98,6 +98,7 @@ type Config struct { Memory string Namespace string OSImageSource *cdiv1beta1.DataVolumeSource + StorageTarget StorageTarget StorageClassName string StorageAccessType corev1.PersistentVolumeAccessMode PVCSize resource.Quantity @@ -108,6 +109,14 @@ type Config struct { TopologySpreadConstraints []corev1.TopologySpreadConstraint } +// StorageTarget represents targeted storage definition that will be used to provision VirtualMachine volumes. Currently, +// there are two definitions, PVC and Storage. Default value is PVC. +type StorageTarget string + +const ( + Storage StorageTarget = "storage" +) + type AffinityType string const ( @@ -255,6 +264,12 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf(`failed to get value of "osImageSource" field: %w`, err) } + storageTarget, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.StorageTarget) + if err != nil { + return nil, nil, fmt.Errorf(`failed to get value of "storageTarget" field: %w`, err) + } + config.StorageTarget = StorageTarget(storageTarget) + pvcSize, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.Size) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "pvcSize" field: %w`, err) @@ -638,6 +653,7 @@ func (p *provider) newVirtualMachine(_ context.Context, c *Config, pc *providerc var ( dataVolumeName = machine.Name annotations = map[string]string{} + dvAnnotations = map[string]string{} ) // Add machineName as prefix to secondaryDisks. addPrefixToSecondaryDisk(c.SecondaryDisks, dataVolumeName) @@ -649,6 +665,11 @@ func (p *provider) newVirtualMachine(_ context.Context, c *Config, pc *providerc annotations["ovn.kubernetes.io/allow_live_migration"] = "true" for k, v := range machine.Annotations { + if strings.HasPrefix(k, "cdi.kubevirt.io") { + dvAnnotations[k] = v + continue + } + annotations[k] = v } @@ -681,8 +702,8 @@ func (p *provider) newVirtualMachine(_ context.Context, c *Config, pc *providerc }, Domain: kubevirtv1.DomainSpec{ Devices: kubevirtv1.Devices{ - Disks: getVMDisks(c), Interfaces: []kubevirtv1.Interface{*defaultBridgeNetwork}, + Disks: getVMDisks(c), }, Resources: resourceRequirements, }, @@ -694,7 +715,7 @@ func (p *provider) newVirtualMachine(_ context.Context, c *Config, pc *providerc TopologySpreadConstraints: getTopologySpreadConstraints(c, map[string]string{machineDeploymentLabelKey: labels[machineDeploymentLabelKey]}), }, }, - DataVolumeTemplates: getDataVolumeTemplates(c, dataVolumeName), + DataVolumeTemplates: getDataVolumeTemplates(c, dataVolumeName, dvAnnotations), }, } return virtualMachine, nil @@ -831,27 +852,43 @@ func getVMVolumes(config *Config, dataVolumeName string, userDataSecretName stri return volumes } -func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1.DataVolumeTemplateSpec { +func getDataVolumeTemplates(config *Config, dataVolumeName string, annotations map[string]string) []kubevirtv1.DataVolumeTemplateSpec { pvcRequest := corev1.ResourceList{corev1.ResourceStorage: config.PVCSize} dataVolumeTemplates := []kubevirtv1.DataVolumeTemplateSpec{ { ObjectMeta: metav1.ObjectMeta{ - Name: dataVolumeName, + Name: dataVolumeName, + Annotations: annotations, }, Spec: cdiv1beta1.DataVolumeSpec{ - PVC: &corev1.PersistentVolumeClaimSpec{ - StorageClassName: ptr.To(config.StorageClassName), - AccessModes: []corev1.PersistentVolumeAccessMode{ - config.StorageAccessType, - }, - Resources: corev1.VolumeResourceRequirements{ - Requests: pvcRequest, - }, - }, Source: config.OSImageSource, }, }, } + + switch config.StorageTarget { + case Storage: + dataVolumeTemplates[0].Spec.Storage = &cdiv1beta1.StorageSpec{ + StorageClassName: ptr.To(config.StorageClassName), + AccessModes: []corev1.PersistentVolumeAccessMode{ + config.StorageAccessType, + }, + Resources: corev1.ResourceRequirements{ + Requests: pvcRequest, + }, + } + default: + dataVolumeTemplates[0].Spec.PVC = &corev1.PersistentVolumeClaimSpec{ + StorageClassName: ptr.To(config.StorageClassName), + AccessModes: []corev1.PersistentVolumeAccessMode{ + config.StorageAccessType, + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: pvcRequest, + }, + } + } + for _, sd := range config.SecondaryDisks { dataVolumeTemplates = append(dataVolumeTemplates, kubevirtv1.DataVolumeTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index 0e69d6626..18bcea9da 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -59,6 +59,7 @@ type kubevirtProviderSpecConf struct { OsImageDV string // if OsImage from DV and not from http source Instancetype *kubevirtv1.InstancetypeMatcher Preference *kubevirtv1.PreferenceMatcher + StorageTarget StorageTarget OperatingSystem string TopologySpreadConstraint bool Affinity bool @@ -123,6 +124,9 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { "storageClassName": "longhorn3"}], {{- end }} "primaryDisk": { + {{- if .StorageTarget }} + "storageTarget": "{{ .StorageTarget }}", + {{- end }} {{- if .OsImageDV }} "osImage": "{{ .OsImageDV }}", {{- else }} @@ -217,6 +221,9 @@ func TestNewVirtualMachine(t *testing.T) { { name: "custom-local-disk", specConf: kubevirtProviderSpecConf{OsImageDV: "ns/dvname"}, + }, { + name: "use-storage-as-storage-target", + specConf: kubevirtProviderSpecConf{StorageTarget: Storage}, }, { name: "http-image-source", diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml new file mode 100644 index 000000000..c0246a801 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml @@ -0,0 +1,80 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: use-storage-as-storage-target + md: md-name + name: use-storage-as-storage-target + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: use-storage-as-storage-target + annotations: {} + spec: + source: + http: + url: "http://x.y.z.t/ubuntu.img" + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + runStrategy: Once + template: + metadata: + creationTimestamp: null + annotations: + "ovn.kubernetes.io/allow_live_migration": "true" + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: use-storage-as-storage-target + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: use-storage-as-storage-target + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index a46fc88b3..52a2c2e63 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -76,6 +76,9 @@ type Template struct { // PrimaryDisk. type PrimaryDisk struct { Disk + // StorageTarget describes which VirtualMachine storage target will be used in the DataVolumeTemplate. + StorageTarget providerconfigtypes.ConfigVarString `json:"storageTarget,omitempty"` + // OsImage describes the OS that will be installed on the VirtualMachine. OsImage providerconfigtypes.ConfigVarString `json:"osImage,omitempty"` // Source describes the VM Disk Image source. Source providerconfigtypes.ConfigVarString `json:"source,omitempty"` From 77e8b4e51fd4b6f006a76dd48ead19df3d175da6 Mon Sep 17 00:00:00 2001 From: Karol Szwaj Date: Sun, 8 Sep 2024 15:08:29 +0200 Subject: [PATCH 425/489] Remove random mac address as default option and add kubevirt bridge annotation (#1856) Signed-off-by: Karol Szwaj --- .../provider/kubevirt/provider.go | 32 ++++--------------- .../provider/kubevirt/provider_test.go | 11 +++---- .../kubevirt/testdata/affinity-no-values.yaml | 4 +-- .../provider/kubevirt/testdata/affinity.yaml | 4 +-- .../kubevirt/testdata/custom-local-disk.yaml | 4 +-- .../kubevirt/testdata/http-image-source.yaml | 4 +-- .../instancetype-preference-custom.yaml | 4 +-- .../instancetype-preference-standard.yaml | 4 +-- .../kubevirt/testdata/nominal-case.yaml | 4 +-- .../kubevirt/testdata/pvc-image-source.yaml | 4 +-- .../testdata/registry-image-source-pod.yaml | 4 +-- .../testdata/registry-image-source.yaml | 4 +-- .../kubevirt/testdata/secondary-disks.yaml | 4 +-- .../testdata/topologyspreadconstraints.yaml | 4 +-- .../use-storage-as-storage-target.yaml | 4 +-- 15 files changed, 36 insertions(+), 59 deletions(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 021f200c9..13bd1fca1 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -37,7 +37,6 @@ import ( "k8c.io/machine-controller/pkg/cloudprovider/instance" kubevirttypes "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - netutil "k8c.io/machine-controller/pkg/cloudprovider/util" controllerutil "k8c.io/machine-controller/pkg/controller/util" "k8c.io/machine-controller/pkg/providerconfig" providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" @@ -597,7 +596,7 @@ func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *cl userDataSecretName := fmt.Sprintf("userdata-%s-%s", machine.Name, strconv.Itoa(int(time.Now().Unix()))) virtualMachine, err := p.newVirtualMachine(ctx, c, pc, machine, userDataSecretName, userdata, - machineDeploymentNameAndRevisionForMachineGetter(ctx, machine, data.Client), randomMacAddressGetter) + machineDeploymentNameAndRevisionForMachineGetter(ctx, machine, data.Client)) if err != nil { return nil, fmt.Errorf("could not create a VirtualMachine manifest %w", err) } @@ -621,7 +620,7 @@ func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *cl } func (p *provider) newVirtualMachine(_ context.Context, c *Config, pc *providerconfigtypes.Config, machine *clusterv1alpha1.Machine, - userdataSecretName, userdata string, mdNameGetter machineDeploymentNameGetter, macAddressGetter macAddressGetter) (*kubevirtv1.VirtualMachine, error) { + userdataSecretName, userdata string, mdNameGetter machineDeploymentNameGetter) (*kubevirtv1.VirtualMachine, error) { // We add the timestamp because the secret name must be different when we recreate the VMI // because its pod got deleted // The secret has an ownerRef on the VMI so garbace collection will take care of cleaning up. @@ -663,6 +662,7 @@ func (p *provider) newVirtualMachine(_ context.Context, c *Config, pc *providerc } annotations["ovn.kubernetes.io/allow_live_migration"] = "true" + annotations["kubevirt.io/allow-pod-bridge-network-live-migration"] = "true" for k, v := range machine.Annotations { if strings.HasPrefix(k, "cdi.kubevirt.io") { @@ -673,11 +673,7 @@ func (p *provider) newVirtualMachine(_ context.Context, c *Config, pc *providerc annotations[k] = v } - defaultBridgeNetwork, err := defaultBridgeNetwork(macAddressGetter) - if err != nil { - return nil, fmt.Errorf("could not compute a random MAC address") - } - + defaultBridgeNetwork := defaultBridgeNetwork() runStrategyOnce := kubevirtv1.RunStrategyOnce virtualMachine := &kubevirtv1.VirtualMachine{ @@ -799,24 +795,8 @@ func getVMDisks(config *Config) []kubevirtv1.Disk { return disks } -type macAddressGetter func() (string, error) - -func randomMacAddressGetter() (string, error) { - mac, err := netutil.GenerateRandMAC() - if err != nil { - return "", err - } - return mac.String(), nil -} - -func defaultBridgeNetwork(macAddressGetter macAddressGetter) (*kubevirtv1.Interface, error) { - defaultBridgeNetwork := kubevirtv1.DefaultBridgeNetworkInterface() - mac, err := macAddressGetter() - if err != nil { - return nil, err - } - defaultBridgeNetwork.MacAddress = mac - return defaultBridgeNetwork, nil +func defaultBridgeNetwork() *kubevirtv1.Interface { + return kubevirtv1.DefaultBridgeNetworkInterface() } func getVMVolumes(config *Config, dataVolumeName string, userDataSecretName string) []kubevirtv1.Volume { diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index 18bcea9da..a1ef8545a 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -126,7 +126,7 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { "primaryDisk": { {{- if .StorageTarget }} "storageTarget": "{{ .StorageTarget }}", - {{- end }} + {{- end }} {{- if .OsImageDV }} "osImage": "{{ .OsImageDV }}", {{- else }} @@ -221,7 +221,8 @@ func TestNewVirtualMachine(t *testing.T) { { name: "custom-local-disk", specConf: kubevirtProviderSpecConf{OsImageDV: "ns/dvname"}, - }, { + }, + { name: "use-storage-as-storage-target", specConf: kubevirtProviderSpecConf{StorageTarget: Storage}, }, @@ -263,7 +264,7 @@ func TestNewVirtualMachine(t *testing.T) { c.Namespace = testNamespace // Check the created VirtualMachine - vm, _ := p.newVirtualMachine(context.TODO(), c, pc, machine, "udsn", userdata, fakeMachineDeploymentNameAndRevisionForMachineGetter(), fixedMacAddressGetter) + vm, _ := p.newVirtualMachine(context.TODO(), c, pc, machine, "udsn", userdata, fakeMachineDeploymentNameAndRevisionForMachineGetter()) vm.TypeMeta.APIVersion, vm.TypeMeta.Kind = kubevirtv1.VirtualMachineGroupVersionKind.ToAPIVersionAndKind() if !equality.Semantic.DeepEqual(vm, expectedVms[tt.name]) { @@ -289,10 +290,6 @@ func toVirtualMachines(objects []runtime.Object) map[string]*kubevirtv1.VirtualM return vms } -func fixedMacAddressGetter() (string, error) { - return "b6:f5:b4:fe:45:1d", nil -} - // runtimeFromYaml returns a list of Kubernetes runtime objects from their yaml templates. // It returns the objects for all files included in the ManifestFS folder, skipping (with error log) the yaml files // that would not contain correct yaml files. diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml index bac6e7a46..5d55f2071 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml @@ -30,6 +30,7 @@ spec: metadata: annotations: "ovn.kubernetes.io/allow_live_migration": "true" + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker @@ -53,8 +54,7 @@ spec: bus: virtio name: cloudinitdisk interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default + - name: default bridge: {} resources: limits: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml index 7f1889466..8d206aa0c 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml @@ -30,6 +30,7 @@ spec: metadata: annotations: "ovn.kubernetes.io/allow_live_migration": "true" + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker @@ -56,8 +57,7 @@ spec: bus: virtio name: cloudinitdisk interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default + - name: default bridge: {} resources: limits: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml index 1d761053c..767303e29 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml @@ -31,6 +31,7 @@ spec: creationTimestamp: null annotations: "ovn.kubernetes.io/allow_live_migration": "true" + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker @@ -48,8 +49,7 @@ spec: bus: virtio name: cloudinitdisk interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default + - name: default bridge: {} resources: limits: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml index c5125c087..93fe25186 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml @@ -30,6 +30,7 @@ spec: creationTimestamp: null annotations: "ovn.kubernetes.io/allow_live_migration": "true" + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: kubevirt.io/vm: http-image-source cluster.x-k8s.io/cluster-name: cluster-name @@ -47,8 +48,7 @@ spec: bus: virtio name: cloudinitdisk interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default + - name: default bridge: {} resources: limits: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml index a27fc8674..4830ac6c6 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml @@ -36,6 +36,7 @@ spec: metadata: annotations: "ovn.kubernetes.io/allow_live_migration": "true" + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker @@ -53,8 +54,7 @@ spec: bus: virtio name: cloudinitdisk interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default + - name: default bridge: {} networks: - name: default diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml index 718e23d38..20ddfe91e 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml @@ -36,6 +36,7 @@ spec: creationTimestamp: null annotations: "ovn.kubernetes.io/allow_live_migration": "true" + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker @@ -53,8 +54,7 @@ spec: bus: virtio name: cloudinitdisk interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default + - name: default bridge: {} networks: - name: default diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml index bb65126cf..c679f0da1 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml @@ -30,6 +30,7 @@ spec: creationTimestamp: null annotations: "ovn.kubernetes.io/allow_live_migration": "true" + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker @@ -47,8 +48,7 @@ spec: bus: virtio name: cloudinitdisk interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default + - name: default bridge: {} resources: limits: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml index 20fd7a4ee..7caf5b201 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml @@ -31,6 +31,7 @@ spec: creationTimestamp: null annotations: "ovn.kubernetes.io/allow_live_migration": "true" + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: kubevirt.io/vm: pvc-image-source cluster.x-k8s.io/cluster-name: cluster-name @@ -48,8 +49,7 @@ spec: bus: virtio name: cloudinitdisk interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default + - name: default bridge: {} resources: limits: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml index 7c047af05..f632c05f8 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml @@ -31,6 +31,7 @@ spec: creationTimestamp: null annotations: "ovn.kubernetes.io/allow_live_migration": "true" + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: kubevirt.io/vm: registry-image-source-pod cluster.x-k8s.io/cluster-name: cluster-name @@ -48,8 +49,7 @@ spec: bus: virtio name: cloudinitdisk interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default + - name: default bridge: {} resources: limits: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml index df5dedac2..3bfcd1a68 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml @@ -31,6 +31,7 @@ spec: creationTimestamp: null annotations: "ovn.kubernetes.io/allow_live_migration": "true" + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: kubevirt.io/vm: registry-image-source cluster.x-k8s.io/cluster-name: cluster-name @@ -48,8 +49,7 @@ spec: bus: virtio name: cloudinitdisk interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default + - name: default bridge: {} resources: limits: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml index bea429e6f..d563ea7eb 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml @@ -56,6 +56,7 @@ spec: creationTimestamp: null annotations: "ovn.kubernetes.io/allow_live_migration": "true" + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker @@ -79,8 +80,7 @@ spec: bus: virtio name: secondary-disks-secondarydisk1 interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default + - name: default bridge: {} resources: limits: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml index 965429429..ebd161b71 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml @@ -30,6 +30,7 @@ spec: creationTimestamp: null annotations: "ovn.kubernetes.io/allow_live_migration": "true" + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker @@ -47,8 +48,7 @@ spec: bus: virtio name: cloudinitdisk interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default + - name: default bridge: {} resources: limits: diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml index c0246a801..7ba8c9cef 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml @@ -31,6 +31,7 @@ spec: creationTimestamp: null annotations: "ovn.kubernetes.io/allow_live_migration": "true" + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker @@ -48,8 +49,7 @@ spec: bus: virtio name: cloudinitdisk interfaces: - - macAddress: b6:f5:b4:fe:45:1d - name: default + - name: default bridge: {} resources: limits: From 7f090ad5fc651ffb44d8f716735f32b692973506 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 13 Sep 2024 18:40:34 +0500 Subject: [PATCH 426/489] Upgrade to Go 1.23.1 (#1859) Signed-off-by: Waleed Malik --- .golangci.yml | 1 - .prow/e2e-features.yaml | 8 +- .prow/postsubmits.yaml | 4 +- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 12 +- .prow/provider-azure.yaml | 6 +- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 4 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 +- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 10 +- .prow/verify.yaml | 16 +- Dockerfile | 2 +- Makefile | 2 +- go.mod | 160 +++---- go.sum | 391 +++++++++--------- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- .../provider/kubevirt/provider.go | 2 +- 26 files changed, 325 insertions(+), 321 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 458d9f387..039495351 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -12,7 +12,6 @@ linters: - errcheck - errname - errorlint - - exportloopref - goconst - gocyclo - godot diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 70c772aff..d4cd4d85c 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -35,7 +35,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -65,7 +65,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -95,7 +95,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -123,7 +123,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 76efda442..744398c45 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -27,7 +27,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - /bin/bash - -c @@ -56,7 +56,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index 65dbbe73b..19956b730 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index f38501d29..7bedcd160 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -32,7 +32,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index d6346e774..b2c27d6bd 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -96,7 +96,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -130,7 +130,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -162,7 +162,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -194,7 +194,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index e2819e0ba..91d22f336 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -96,7 +96,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 56471a0d7..4f11b59f1 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index e56ddb1cd..a7f6b3657 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index c17f705a4..6921900b6 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index efc74599a..51db5d389 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 915fc2056..fa2dd7e6b 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-kubevirt -# run_if_changed: "(pkg/cloudprovider/provider/kubevirt/|pkg/userdata)" + # run_if_changed: "(pkg/cloudprovider/provider/kubevirt/|pkg/userdata)" always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 3ef294ee7..c2485844a 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 6c1909d78..cfc486b64 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index aaa20bbd1..47808b1f7 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -64,7 +64,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 10fea9e33..49b8bbd50 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index e5259a8b0..b43201c06 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -32,7 +32,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 8ba485055..f1217a51a 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -95,7 +95,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -128,7 +128,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -161,7 +161,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.23-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 5e9b21719..7afdc9774 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -22,7 +22,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-3 command: - make args: @@ -44,7 +44,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-3 command: - make args: @@ -66,7 +66,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-3 command: - make args: @@ -87,7 +87,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-3 command: - make args: @@ -107,7 +107,7 @@ presubmits: path_alias: k8c.io/machine-controller spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-3 command: - "/usr/local/bin/shfmt" args: @@ -136,7 +136,7 @@ presubmits: path_alias: k8c.io/machine-controller spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-3 command: - "./hack/verify-boilerplate.sh" resources: @@ -156,7 +156,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-3 command: - ./hack/verify-licenses.sh resources: @@ -173,7 +173,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-0 + - image: quay.io/kubermatic/build:go-1.23-node-20-3 command: - make args: diff --git a/Dockerfile b/Dockerfile index 0cf006c9b..04bae0024 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.23.0 +ARG GO_VERSION=1.23.1 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/k8c.io/machine-controller COPY . . diff --git a/Makefile b/Makefile index 7bd67c081..be85e3dff 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.23.0 +GO_VERSION ?= 1.23.1 GOOS ?= $(shell go env GOOS) diff --git a/go.mod b/go.mod index 5fd961c75..0385bc162 100644 --- a/go.mod +++ b/go.mod @@ -1,74 +1,74 @@ module k8c.io/machine-controller -go 1.22.0 +go 1.22.3 -toolchain go1.22.4 +toolchain go1.23.1 require ( - cloud.google.com/go/logging v1.10.0 - cloud.google.com/go/monitoring v1.20.1 + cloud.google.com/go/logging v1.11.0 + cloud.google.com/go/monitoring v1.21.1 github.com/Azure/azure-sdk-for-go v68.0.0+incompatible github.com/Azure/go-autorest/autorest/azure/auth v0.5.13 github.com/Azure/go-autorest/autorest/to v0.4.0 - github.com/Masterminds/semver/v3 v3.2.1 - github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240703112053-c8500c47bced - github.com/aliyun/alibaba-cloud-sdk-go v1.62.784 - github.com/aws/aws-sdk-go-v2 v1.30.1 - github.com/aws/aws-sdk-go-v2/config v1.27.24 - github.com/aws/aws-sdk-go-v2/credentials v1.17.24 - github.com/aws/aws-sdk-go-v2/service/ec2 v1.168.0 - github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 - github.com/aws/smithy-go v1.20.3 + github.com/Masterminds/semver/v3 v3.3.0 + github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240905143811-b2ab5b7c9c14 + github.com/aliyun/alibaba-cloud-sdk-go v1.63.15 + github.com/aws/aws-sdk-go-v2 v1.30.5 + github.com/aws/aws-sdk-go-v2/config v1.27.33 + github.com/aws/aws-sdk-go-v2/credentials v1.17.32 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.177.3 + github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 + github.com/aws/smithy-go v1.20.4 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc - github.com/digitalocean/godo v1.118.0 + github.com/digitalocean/godo v1.124.0 github.com/go-logr/logr v1.4.2 github.com/go-logr/zapr v1.3.0 github.com/go-test/deep v1.1.0 github.com/google/uuid v1.6.0 - github.com/gophercloud/gophercloud v1.12.0 + github.com/gophercloud/gophercloud v1.14.0 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb - github.com/hetznercloud/hcloud-go/v2 v2.10.2 - github.com/linode/linodego v1.36.1 - github.com/nutanix-cloud-native/prism-go-client v0.4.0 + github.com/hetznercloud/hcloud-go/v2 v2.13.1 + github.com/linode/linodego v1.40.0 + github.com/nutanix-cloud-native/prism-go-client v0.5.1 github.com/packethost/packngo v0.31.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 - github.com/prometheus/client_golang v1.19.1 - github.com/scaleway/scaleway-sdk-go v1.0.0-beta.28 + github.com/prometheus/client_golang v1.20.3 + github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 github.com/spf13/pflag v1.0.5 - github.com/tinkerbell/tink v0.10.0 + github.com/tinkerbell/tink v0.10.1 github.com/vmware/go-vcloud-director/v2 v2.25.0 - github.com/vmware/govmomi v0.42.0 - github.com/vultr/govultr/v3 v3.9.0 - go.anx.io/go-anxcloud v0.7.2 + github.com/vmware/govmomi v0.43.0 + github.com/vultr/govultr/v3 v3.9.1 + go.anx.io/go-anxcloud v0.7.3 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.25.0 - golang.org/x/oauth2 v0.21.0 + golang.org/x/crypto v0.27.0 + golang.org/x/oauth2 v0.23.0 gomodules.xyz/jsonpatch/v2 v2.4.0 - google.golang.org/api v0.187.0 + google.golang.org/api v0.197.0 gopkg.in/yaml.v3 v3.0.1 - k8s.io/api v0.31.0 - k8s.io/apiextensions-apiserver v0.31.0 - k8s.io/apimachinery v0.31.0 - k8s.io/client-go v0.31.0 - k8s.io/cloud-provider v0.31.0 + k8s.io/api v0.31.1 + k8s.io/apiextensions-apiserver v0.31.1 + k8s.io/apimachinery v0.31.1 + k8s.io/client-go v0.31.1 + k8s.io/cloud-provider v0.31.1 k8s.io/klog v1.0.0 - k8s.io/kubectl v0.31.0 - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 - kubevirt.io/api v1.2.2 - kubevirt.io/containerized-data-importer-api v1.59.0 + k8s.io/kubectl v0.31.1 + k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 + kubevirt.io/api v1.3.1 + kubevirt.io/containerized-data-importer-api v1.60.3 sigs.k8s.io/controller-runtime v0.19.0 sigs.k8s.io/yaml v1.4.0 ) require ( - cloud.google.com/go v0.115.0 // indirect - cloud.google.com/go/auth v0.6.1 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect - cloud.google.com/go/compute/metadata v0.4.0 // indirect - cloud.google.com/go/longrunning v0.5.9 // indirect + cloud.google.com/go v0.115.1 // indirect + cloud.google.com/go/auth v0.9.4 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect + cloud.google.com/go/compute/metadata v0.5.1 // indirect + cloud.google.com/go/longrunning v0.6.1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.29 // indirect github.com/Azure/go-autorest/autorest/adal v0.9.24 // indirect @@ -80,14 +80,15 @@ require ( github.com/PaesslerAG/gval v1.2.2 // indirect github.com/PaesslerAG/jsonpath v0.1.1 // indirect github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect @@ -98,10 +99,16 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/analysis v0.23.0 // indirect + github.com/go-openapi/errors v0.22.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/loads v0.22.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/strfmt v0.23.0 // indirect github.com/go-openapi/swag v0.23.0 // indirect - github.com/go-resty/resty/v2 v2.13.1 // indirect + github.com/go-openapi/validate v0.24.0 // indirect + github.com/go-resty/resty/v2 v2.14.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect @@ -111,10 +118,10 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect - github.com/google/s2a-go v0.1.7 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.5 // indirect + github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect + github.com/google/s2a-go v0.1.8 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect + github.com/googleapis/gax-go/v2 v2.13.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-version v1.7.0 // indirect @@ -122,48 +129,51 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/onsi/ginkgo/v2 v2.19.0 // indirect - github.com/onsi/gomega v1.33.1 // indirect - github.com/openshift/api v0.0.0-20240708071937-c9a91940bf0f // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/onsi/ginkgo/v2 v2.20.1 // indirect + github.com/onsi/gomega v1.34.1 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect github.com/peterhellberg/link v1.2.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/common v0.59.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/x448/float16 v0.8.4 // indirect + go.mongodb.org/mongo-driver v1.16.1 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect - go.opentelemetry.io/otel v1.28.0 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect - go.opentelemetry.io/otel/trace v1.28.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect + go.opentelemetry.io/otel v1.30.0 // indirect + go.opentelemetry.io/otel/metric v1.30.0 // indirect + go.opentelemetry.io/otel/trace v1.30.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect - golang.org/x/net v0.27.0 // indirect + golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect + golang.org/x/net v0.29.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/term v0.22.0 // indirect - golang.org/x/text v0.17.0 // indirect - golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.23.0 // indirect - google.golang.org/genproto v0.0.0-20240701130421-f6361c86f094 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect - google.golang.org/grpc v1.65.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/term v0.24.0 // indirect + golang.org/x/text v0.18.0 // indirect + golang.org/x/time v0.6.0 // indirect + golang.org/x/tools v0.25.0 // indirect + google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.2 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect @@ -171,7 +181,7 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20240703190633-0aa61b46e8c2 // indirect + k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect diff --git a/go.sum b/go.sum index f79becfc4..5f7cef617 100644 --- a/go.sum +++ b/go.sum @@ -1,23 +1,21 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.115.0 h1:CnFSK6Xo3lDYRoBKEcAtia6VSC837/ZkJuRduSFnr14= -cloud.google.com/go v0.115.0/go.mod h1:8jIM5vVgoAEoiVxQ/O4BFTfHqulPZgs/ufEzMcFMdWU= -cloud.google.com/go/auth v0.6.1 h1:T0Zw1XM5c1GlpN2HYr2s+m3vr1p2wy+8VN+Z1FKxW38= -cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4= -cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= -cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= -cloud.google.com/go/compute/metadata v0.4.0 h1:vHzJCWaM4g8XIcm8kopr3XmDA4Gy/lblD3EhhSux05c= -cloud.google.com/go/compute/metadata v0.4.0/go.mod h1:SIQh1Kkb4ZJ8zJ874fqVkslA29PRXuleyj6vOzlbK7M= -cloud.google.com/go/iam v1.1.9 h1:oSkYLVtVme29uGYrOcKcvJRht7cHJpYD09GM9JaR0TE= -cloud.google.com/go/iam v1.1.9/go.mod h1:Nt1eDWNYH9nGQg3d/mY7U1hvfGmsaG9o/kLGoLoLXjQ= -cloud.google.com/go/logging v1.10.0 h1:f+ZXMqyrSJ5vZ5pE/zr0xC8y/M9BLNzQeLBwfeZ+wY4= -cloud.google.com/go/logging v1.10.0/go.mod h1:EHOwcxlltJrYGqMGfghSet736KR3hX1MAj614mrMk9I= -cloud.google.com/go/longrunning v0.5.9 h1:haH9pAuXdPAMqHvzX0zlWQigXT7B0+CL4/2nXXdBo5k= -cloud.google.com/go/longrunning v0.5.9/go.mod h1:HD+0l9/OOW0za6UWdKJtXoFAX/BGg/3Wj8p10NeWF7c= -cloud.google.com/go/monitoring v1.20.1 h1:XmM6uk4+mI2ZhWdI2n/2GNhJdpeQN+1VdG2UWEDhX48= -cloud.google.com/go/monitoring v1.20.1/go.mod h1:FYSe/brgfuaXiEzOQFhTjsEsJv+WePyK71X7Y8qo6uQ= +cloud.google.com/go v0.115.1 h1:Jo0SM9cQnSkYfp44+v+NQXHpcHqlnRJk2qxh6yvxxxQ= +cloud.google.com/go v0.115.1/go.mod h1:DuujITeaufu3gL68/lOFIirVNJwQeyf5UXyi+Wbgknc= +cloud.google.com/go/auth v0.9.4 h1:DxF7imbEbiFu9+zdKC6cKBko1e8XeJnipNqIbWZ+kDI= +cloud.google.com/go/auth v0.9.4/go.mod h1:SHia8n6//Ya940F1rLimhJCjjx7KE17t0ctFEci3HkA= +cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= +cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= +cloud.google.com/go/compute/metadata v0.5.1 h1:NM6oZeZNlYjiwYje+sYFjEpP0Q0zCan1bmQW/KmIrGs= +cloud.google.com/go/compute/metadata v0.5.1/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= +cloud.google.com/go/iam v1.2.0 h1:kZKMKVNk/IsSSc/udOb83K0hL/Yh/Gcqpz+oAkoIFN8= +cloud.google.com/go/iam v1.2.0/go.mod h1:zITGuWgsLZxd8OwAlX+eMFgZDXzBm7icj1PVTYG766Q= +cloud.google.com/go/logging v1.11.0 h1:v3ktVzXMV7CwHq1MBF65wcqLMA7i+z3YxbUsoK7mOKs= +cloud.google.com/go/logging v1.11.0/go.mod h1:5LDiJC/RxTt+fHc1LAt20R9TKiUTReDg6RuuFOZ67+A= +cloud.google.com/go/longrunning v0.6.1 h1:lOLTFxYpr8hcRtcwWir5ITh1PAKUD/sG2lKrTSYjyMc= +cloud.google.com/go/longrunning v0.6.1/go.mod h1:nHISoOZpBcmlwbJmiVk5oDRz0qG/ZxPynEGs1iZ79s0= +cloud.google.com/go/monitoring v1.21.1 h1:zWtbIoBMnU5LP9A/fz8LmWMGHpk4skdfeiaa66QdFGc= +cloud.google.com/go/monitoring v1.21.1/go.mod h1:Rj++LKrlht9uBi8+Eb530dIrzG/cU/lB8mt+lbeFK1c= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/99designs/gqlgen v0.15.1 h1:48bRXecwlCNTa/n2bMSp2rQsXNxwZ54QHbiULNf78ec= -github.com/99designs/gqlgen v0.15.1/go.mod h1:nbeSjFkqphIqpZsYe1ULVz0yfH8hjpJdJIQoX/e0G2I= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -49,11 +47,11 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= -github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= -github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= +github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240703112053-c8500c47bced h1:yLjIFnPG7e6IVIDxFWp+dUkxlOgOkx8ttesy9dGAEms= -github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240703112053-c8500c47bced/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= +github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240905143811-b2ab5b7c9c14 h1:9uqKGeUuok/9Q5B5DzDM+bVgyEZVruzaflXw8WiaZ+Y= +github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240905143811-b2ab5b7c9c14/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= github.com/PaesslerAG/gval v1.2.2 h1:Y7iBzhgE09IGTt5QgGQ2IdaYYYOU134YGHBThD+wm9E= github.com/PaesslerAG/gval v1.2.2/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac= @@ -62,42 +60,42 @@ github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEs github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRBij0P8= -github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.784 h1:wtou656eZpKB1QFJ26ChcmpIZvAdtVZvxuhD7BJpZtA= -github.com/aliyun/alibaba-cloud-sdk-go v1.62.784/go.mod h1:SOSDHfe1kX91v3W5QiBsWSLqeLxImobbMX1mxrFHsVQ= +github.com/aliyun/alibaba-cloud-sdk-go v1.63.15 h1:r2uwBUQhLhcPzaWz9tRJqc8MjYwHb+oF2+Q6467BF14= +github.com/aliyun/alibaba-cloud-sdk-go v1.63.15/go.mod h1:SOSDHfe1kX91v3W5QiBsWSLqeLxImobbMX1mxrFHsVQ= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go-v2 v1.30.1 h1:4y/5Dvfrhd1MxRDD77SrfsDaj8kUkkljU7XE83NPV+o= -github.com/aws/aws-sdk-go-v2 v1.30.1/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= -github.com/aws/aws-sdk-go-v2/config v1.27.24 h1:NM9XicZ5o1CBU/MZaHwFtimRpWx9ohAUAqkG6AqSqPo= -github.com/aws/aws-sdk-go-v2/config v1.27.24/go.mod h1:aXzi6QJTuQRVVusAO8/NxpdTeTyr/wRcybdDtfUwJSs= -github.com/aws/aws-sdk-go-v2/credentials v1.17.24 h1:YclAsrnb1/GTQNt2nzv+756Iw4mF8AOzcDfweWwwm/M= -github.com/aws/aws-sdk-go-v2/credentials v1.17.24/go.mod h1:Hld7tmnAkoBQdTMNYZGzztzKRdA4fCdn9L83LOoigac= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 h1:Aznqksmd6Rfv2HQN9cpqIV/lQRMaIpJkLLaJ1ZI76no= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9/go.mod h1:WQr3MY7AxGNxaqAtsDWn+fBxmd4XvLkzeqQ8P1VM0/w= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 h1:5SAoZ4jYpGH4721ZNoS1znQrhOfZinOhc4XuTXx/nVc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13/go.mod h1:+rdA6ZLpaSeM7tSg/B0IEDinCIBJGmW8rKDFkYpP04g= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 h1:WIijqeaAO7TYFLbhsZmi2rgLEAtWOC1LhxCAVTJlSKw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13/go.mod h1:i+kbfa76PQbWw/ULoWnp51EYVWH4ENln76fLQE3lXT8= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.168.0 h1:xOPq0agGC1WMZvFpSZCKEjDVAQnLPZJZGvjuPVF2t9M= -github.com/aws/aws-sdk-go-v2/service/ec2 v1.168.0/go.mod h1:CtLD6CPq9z9dyMxV+H6/M5d9+/ea3dO80um029GXqV0= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 h1:I9zMeF107l0rJrpnHpjEiiTSCKYAIw8mALiXcPsGBiA= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15/go.mod h1:9xWJ3Q/S6Ojusz1UIkfycgD1mGirJfLLKqq3LPT7WN8= -github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 h1:p1GahKIjyMDZtiKoIn0/jAj/TkMzfzndDv5+zi2Mhgc= -github.com/aws/aws-sdk-go-v2/service/sso v1.22.1/go.mod h1:/vWdhoIoYA5hYoPZ6fm7Sv4d8701PiG5VKe8/pPJL60= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 h1:ORnrOK0C4WmYV/uYt3koHEWBLYsRDwk2Np+eEoyV4Z0= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2/go.mod h1:xyFHA4zGxgYkdD73VeezHt3vSKEG9EmFnGwoKlP00u4= -github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 h1:+woJ607dllHJQtsnJLi52ycuqHMwlW+Wqm2Ppsfp4nQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.30.1/go.mod h1:jiNR3JqT15Dm+QWq2SRgh0x0bCNSRP2L25+CqPNpJlQ= -github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= -github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g= +github.com/aws/aws-sdk-go-v2 v1.30.5/go.mod h1:CT+ZPWXbYrci8chcARI3OmI/qgd+f6WtuLOoaIA8PR0= +github.com/aws/aws-sdk-go-v2/config v1.27.33 h1:Nof9o/MsmH4oa0s2q9a0k7tMz5x/Yj5k06lDODWz3BU= +github.com/aws/aws-sdk-go-v2/config v1.27.33/go.mod h1:kEqdYzRb8dd8Sy2pOdEbExTTF5v7ozEXX0McgPE7xks= +github.com/aws/aws-sdk-go-v2/credentials v1.17.32 h1:7Cxhp/BnT2RcGy4VisJ9miUPecY+lyE9I8JvcZofn9I= +github.com/aws/aws-sdk-go-v2/credentials v1.17.32/go.mod h1:P5/QMF3/DCHbXGEGkdbilXHsyTBX5D3HSwcrSc9p20I= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 h1:pfQ2sqNpMVK6xz2RbqLEL0GH87JOwSxPV2rzm8Zsb74= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13/go.mod h1:NG7RXPUlqfsCLLFfi0+IpKN4sCB9D9fw/qTaSB+xRoU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17 h1:pI7Bzt0BJtYA0N/JEC6B8fJ4RBrEMi1LBrkMdFYNSnQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.17/go.mod h1:Dh5zzJYMtxfIjYW+/evjQ8uj2OyR/ve2KROHGHlSFqE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17 h1:Mqr/V5gvrhA2gvgnF42Zh5iMiQNcOYthFYwCyrnuWlc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.17/go.mod h1:aLJpZlCmjE+V+KtN1q1uyZkfnUWpQGpbsn89XPKyzfU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.177.3 h1:dqdCh1M8h+j8OGNUpxTs7eBPFr6lOdLpdlE6IPLLSq4= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.177.3/go.mod h1:TFSALWR7Xs7+KyMM87ZAYxncKFBvzEt2rpK/BJCH2ps= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4 h1:KypMCbLPPHEmf9DgMGw51jMj77VfGPAN2Kv4cfhlfgI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.4/go.mod h1:Vz1JQXliGcQktFTN/LN6uGppAIRoLBR2bMvIMP0gOjc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 h1:rfprUlsdzgl7ZL2KlXiUAoJnI/VxfHCvDFr2QDFj6u4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19/go.mod h1:SCWkEdRq8/7EK60NcvvQ6NXKuTcchAD4ROAsC37VEZE= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.7 h1:pIaGg+08llrP7Q5aiz9ICWbY8cqhTkyy+0SHvfzQpTc= +github.com/aws/aws-sdk-go-v2/service/sso v1.22.7/go.mod h1:eEygMHnTKH/3kNp9Jr1n3PdejuSNcgwLe1dWgQtO0VQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7 h1:/Cfdu0XV3mONYKaOt1Gr0k1KvQzkzPyiKUdlWJqy+J4= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.7/go.mod h1:bCbAxKDqNvkHxRaIMnyVPXPo+OaPRwvmgzMxbz1VKSA= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.7 h1:NKTa1eqZYw8tiHSRGpP0VtTdub/8KNk8sDkNPFaOKDE= +github.com/aws/aws-sdk-go-v2/service/sts v1.30.7/go.mod h1:NXi1dIAGteSaRLqYgarlhP/Ij0cFT+qmCwiJqWh/U5o= +github.com/aws/smithy-go v1.20.4 h1:2HK1zBdPgRbjFOHlfeQZfpC4r72MOb9bZkiFwggKO+4= +github.com/aws/smithy-go v1.20.4/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -108,17 +106,15 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creasty/defaults v1.5.2 h1:/VfB6uxpyp6h0fr7SPp7n8WJBoV8jfxQXPCnkVSjyls= -github.com/creasty/defaults v1.5.2/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY= +github.com/creasty/defaults v1.6.0 h1:ltuE9cfphUtlrBeomuu8PEyISTXnxqkBIoQfXgv7BSc= +github.com/creasty/defaults v1.6.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/digitalocean/godo v1.118.0 h1:lkzGFQmACrVCp7UqH1sAi4JK/PWwlc5aaxubgorKmC4= -github.com/digitalocean/godo v1.118.0/go.mod h1:Vk0vpCot2HOAJwc5WE8wljZGtJ3ZtWIc8MQ8rF38sdo= +github.com/digitalocean/godo v1.124.0 h1:qroI1QdtcgnXF/pefq9blZRbXqBw1Ry/aHh2pnu/328= +github.com/digitalocean/godo v1.124.0/go.mod h1:WQVH83OHUy6gC4gXpEVQKtxTd4L5oCp+5OialidkPLY= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= @@ -148,14 +144,12 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fullstorydev/grpcurl v1.8.7 h1:xJWosq3BQovQ4QrdPO72OrPiWuGgEsxY8ldYsJbPrqI= +github.com/fullstorydev/grpcurl v1.8.7/go.mod h1:pVtM4qe3CMoLaIzYS8uvTuDj2jVYmXqMUkZeijnXp/E= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs= -github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg= -github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8= -github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -167,6 +161,10 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= +github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= +github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= +github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= @@ -176,21 +174,27 @@ github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco= +github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= +github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= +github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= +github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/validator/v10 v10.10.1 h1:uA0+amWMiglNZKZ9FJRKUAe9U3RX91eVn1JYXMWt7ig= github.com/go-playground/validator/v10 v10.10.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= -github.com/go-resty/resty/v2 v2.13.1 h1:x+LHXBI2nMB1vqndymf26quycC4aggYJ7DECYbiz03g= -github.com/go-resty/resty/v2 v2.13.1/go.mod h1:GznXlLxkq6Nh4sU59rPmUw3VtgpO3aS96ORAI6Q7d+0= -github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= -github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/go-resty/resty/v2 v2.14.0 h1:/rhkzsAqGQkozwfKS5aFAbb6TyKd3zyFRWcdRXLPCAU= +github.com/go-resty/resty/v2 v2.14.0/go.mod h1:IW6mekUOsElt9C7oWr0XRt9BNSD6D5rr9mhk6NjmNHg= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= @@ -223,8 +227,6 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -247,26 +249,24 @@ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= -github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= -github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= -github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= +github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= -github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA= -github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E= +github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gTgghdIA6Stxb52D5RnLI1SLyw= +github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= +github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= +github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gophercloud/gophercloud v1.12.0 h1:Jrz16vPAL93l80q16fp8NplrTCp93y7rZh2P3Q4Yq7g= -github.com/gophercloud/gophercloud v1.12.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= +github.com/gophercloud/gophercloud v1.14.0 h1:Bt9zQDhPrbd4qX7EILGmy+i7GP35cc+AAL2+wIJpUE8= +github.com/gophercloud/gophercloud v1.14.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= @@ -276,18 +276,18 @@ github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISH github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb h1:tsEKRC3PU9rMw18w/uAptoijhgG4EvlA5kfJPtwrMDk= github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= -github.com/hetznercloud/hcloud-go/v2 v2.10.2 h1:9gyTUPhfNbfbS40Spgij5mV5k37bOZgt8iHKCbfGs5I= -github.com/hetznercloud/hcloud-go/v2 v2.10.2/go.mod h1:xQ+8KhIS62W0D78Dpi57jsufWh844gUw1az5OUvaeq8= +github.com/hetznercloud/hcloud-go/v2 v2.13.1 h1:jq0GP4QaYE5d8xR/Zw17s9qoaESRJMXfGmtD1a/qckQ= +github.com/hetznercloud/hcloud-go/v2 v2.13.1/go.mod h1:dhix40Br3fDiBhwaSG/zgaYOFFddpfBm/6R1Zz0IiF0= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= +github.com/jhump/protoreflect v1.14.0 h1:MBbQK392K3u8NTLbKOCIi3XdI+y+c6yt5oMq0X3xviw= +github.com/jhump/protoreflect v1.14.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI4tTg7puQFKI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -300,12 +300,12 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/k0kubun/pp/v3 v3.1.0 h1:ifxtqJkRZhw3h554/z/8zm6AAbyO4LLKDlA5eV+9O8Q= github.com/k0kubun/pp/v3 v3.1.0/go.mod h1:vIrP5CF0n78pKHm2Ku6GVerpZBJvscg48WepUYEk2gw= -github.com/keploy/go-sdk v0.7.2 h1:mvvjDRciMSFTgOF/KIGz38ElJZKkM1WlniaHseaPhpo= -github.com/keploy/go-sdk v0.7.2/go.mod h1:TtJIM+Gkq76FzfkD8W9u1F8NDkC9sVY8nYvmbRo1nhg= +github.com/keploy/go-sdk v0.9.0 h1:kpSNcCTDdELsa1gWyhoD9oV57SgSMbG/wq6Cjp4y7cY= +github.com/keploy/go-sdk v0.9.0/go.mod h1:vNKXoFd2MaK+Gly/K6XeP1Hs9dP834C74szH+vtBPwg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= -github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kolo/xmlrpc v0.0.0-20190717152603-07c4ee3fd181/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00= github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= @@ -317,17 +317,17 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/linode/linodego v1.36.1 h1:lxYBKWJCk6m9p/OdHQlgteyj4S0eglq3glmK16QxUHY= -github.com/linode/linodego v1.36.1/go.mod h1:KyV4OO/9/tAxaLSjyjFyOQBcS9bYUdei1hwk3nl0UjI= +github.com/linode/linodego v1.40.0 h1:7ESY0PwK94hoggoCtIroT1Xk6b1flrFBNZ6KwqbTqlI= +github.com/linode/linodego v1.40.0/go.mod h1:NsUw4l8QrLdIofRg1NYFBbW5ZERnmbZykVBszPZLORM= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matryer/moq v0.2.5 h1:BGQISyhl7Gc9W/gMYmAJONh9mT6AYeyeTjNupNPknMs= -github.com/matryer/moq v0.2.5/go.mod h1:9RtPYjTnH1bSBIkpvtHkFN7nbWAnO7oRpdJkEIn6UtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -355,27 +355,27 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nutanix-cloud-native/prism-go-client v0.4.0 h1:P9mLW6eyKMUXVQBzuVL5k7WjV1YwVu8XNpu2XAsRgGo= -github.com/nutanix-cloud-native/prism-go-client v0.4.0/go.mod h1:bHxgYigeclzjuaMEdjpsIEO4k7sjzP4Gr7ooF6nWXcI= +github.com/nutanix-cloud-native/prism-go-client v0.5.1 h1:ykiXPCILzEMORHz7XvI8KXNomChsdLIpOAlT/YqBCmo= +github.com/nutanix-cloud-native/prism-go-client v0.5.1/go.mod h1:QhLX+sEep0cStzHVYU6mPgIlnA8U3DySskagrbDprRk= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= -github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/ginkgo/v2 v2.20.1 h1:YlVIbqct+ZmnEph770q9Q7NVAz4wwIiVNahee6JyUzo= +github.com/onsi/ginkgo/v2 v2.20.1/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= -github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= -github.com/openshift/api v0.0.0-20240708071937-c9a91940bf0f h1:NmJAlN2fPnL86aq5BbEQJ62v/D16LzIaaQ0Qn72s87E= -github.com/openshift/api v0.0.0-20240708071937-c9a91940bf0f/go.mod h1:OOh6Qopf21pSzqNVCB5gomomBXb8o5sGKZxG2KNpaXM= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= @@ -394,25 +394,23 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= -github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4= +github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= -github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0= +github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.28 h1:2vT+ryIQGfF21HN/W5yn/CBPpsTJULuuepWfUq/geV4= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.28/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 h1:yoKAVkEVwAqbGbR8n87rHQ1dulL25rKloGadb3vm770= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30/go.mod h1:sH0u6fq6x4R5M7WxkoQFY/o7UaiItec0o1LinLCJNq8= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= @@ -434,66 +432,46 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= -github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= -github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= -github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tinkerbell/tink v0.10.0 h1:W34Psx6Yn5o0jILSgyjF8C7Cg6ivDcccUqJ6zG9zGzA= -github.com/tinkerbell/tink v0.10.0/go.mod h1:Dc8PpbvrC8Kfhr8yGhUsWBN6Dq6opRIX655KDTxLUJY= +github.com/tinkerbell/tink v0.10.1 h1:mxdPQf7n4nB/AVdjbqCm5c98vsITU35g7Yw5cdOWmCw= +github.com/tinkerbell/tink v0.10.1/go.mod h1:yULdVrzAfPnA8KdOkjvo8qDn6pw0JD6kBzF94gtXMjA= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M= -github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/vektah/gqlparser/v2 v2.2.0 h1:bAc3slekAAJW6sZTi07aGq0OrfaCjj4jxARAaC7g2EM= -github.com/vektah/gqlparser/v2 v2.2.0/go.mod h1:i3mQIGIrbK2PD1RrCeMTlVbkF2FJ6WkU1KJlJlC+3F4= github.com/vmware/go-vcloud-director/v2 v2.25.0 h1:RcJ5FQRku3FvQktTi8YOZsRfvhfLm315Cme50M9x9MQ= github.com/vmware/go-vcloud-director/v2 v2.25.0/go.mod h1:7Of1qJja+LLNKVegjZG7uuhhy6xgGg3q7Fkw2CEP+Tw= -github.com/vmware/govmomi v0.42.0 h1:MbvAlVfjNBE1mHMaQ7yOSop1KLB0/93x6VAGuCtjqtI= -github.com/vmware/govmomi v0.42.0/go.mod h1:1H5LWwsBif8HKZqbFp0FdoKTHyJE4FzL6ACequMKYQg= -github.com/vultr/govultr/v3 v3.9.0 h1:63V/22mpfquRA5DenJ9EF0VozHg0k+X4dhUWcDXHPyc= -github.com/vultr/govultr/v3 v3.9.0/go.mod h1:Rd8ebpXm7jxH3MDmhnEs+zrlYW212ouhx+HeUMfHm2o= -github.com/wI2L/jsondiff v0.2.0 h1:dE00WemBa1uCjrzQUUTE/17I6m5qAaN0EMFOg2Ynr/k= -github.com/wI2L/jsondiff v0.2.0/go.mod h1:axTcwtBkY4TsKuV+RgoMhHyHKKFRI6nnjRLi8LLYQnA= +github.com/vmware/govmomi v0.43.0 h1:7Kg3Bkdly+TrE67BYXzRq7ZrDnn7xqpKX95uEh2f9Go= +github.com/vmware/govmomi v0.43.0/go.mod h1:IOv5nTXCPqH9qVJAlRuAGffogaLsNs8aF+e7vLgsHJU= +github.com/vultr/govultr/v3 v3.9.1 h1:uxSIb8Miel7tqTs3ee+z3t+JelZikwqBBsZzCOPBy/8= +github.com/vultr/govultr/v3 v3.9.1/go.mod h1:Rd8ebpXm7jxH3MDmhnEs+zrlYW212ouhx+HeUMfHm2o= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.0 h1:d70R37I0HrDLsafRrMBXyrD4lmQbCHE873t00Vr0gm0= -github.com/xdg-go/scram v1.1.0/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/stringprep v1.0.2 h1:6iq84/ryjjeRmMJwxutI51F2GIPlP5BfTvXHeYjyhBc= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= -github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.anx.io/go-anxcloud v0.7.2 h1:Y7Rs9jI9G+QDWz7NHao+Prff/JNeG5ST4bNpO5d2qcM= -go.anx.io/go-anxcloud v0.7.2/go.mod h1:AfsjodrtGIEnzzVQ6cHTZxkoNoi3UmeNmHaF0oG9RjE= -go.keploy.io/server v0.7.12 h1:DKDSO6T9Q4d4A8MKL+sk7U26KRcvZ+ZG0mbFhYIJJyk= -go.keploy.io/server v0.7.12/go.mod h1:ch4rD1NCgtxozDHD9yVk+sLHWz5HgefOqrgEdEIgfBQ= -go.mongodb.org/mongo-driver v1.8.3 h1:TDKlTkGDKm9kkJVUOAXDK5/fkqKHJVwYQSpoRfB43R4= -go.mongodb.org/mongo-driver v1.8.3/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= +go.anx.io/go-anxcloud v0.7.3 h1:NWkm4KAg0GyJALBbSgp++J2K563lHQGDDVJAcM6CgUU= +go.anx.io/go-anxcloud v0.7.3/go.mod h1:RpJvC8ZmXNu9dSygIgZ0ossqPz0+6n9xDX9weeATmSo= +go.keploy.io/server v0.8.6 h1:czE9jaliyAkMMJcYnMPNuu6tun7UgwFbokxEG95vLN4= +go.keploy.io/server v0.8.6/go.mod h1:t7BPuZQSiC3PNHZ9dbn3e3VB61HNWwiqVmaRujfDFUg= +go.mongodb.org/mongo-driver v1.16.1 h1:rIVLL3q0IHM39dvE+z2ulZLp9ENZKThVfuvN/IiN4l8= +go.mongodb.org/mongo-driver v1.16.1/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 h1:9G6E0TXzGFVfTnawRzrPl83iHOAV7L8NJiR8RSGYV1g= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0/go.mod h1:azvtTADFQJA8mX80jIH/akaE7h+dbm/sVuaHqN13w74= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= -go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= -go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= -go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= -go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0 h1:hCq2hNMwsegUvPzI7sPOvtO9cqyy5GbWt/Ybp2xrx8Q= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0/go.mod h1:LqaApwGx/oUmzsbqxkzuBvyoPpkxk3JQWnqfVrJ3wCA= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 h1:ZIg3ZT/aQ7AfKqdwp7ECpOK6vHqquXXuyTjIO8ZdmPs= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0/go.mod h1:DQAwmETtZV00skUwgD6+0U89g80NKsJE3DCKeLLPQMI= +go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= +go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= +go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= +go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= -go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= +go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= +go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= @@ -512,19 +490,21 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w= -golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -539,8 +519,9 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -565,13 +546,15 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -580,6 +563,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -610,20 +596,25 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -632,12 +623,14 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -656,8 +649,10 @@ golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpd golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= +golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -668,27 +663,27 @@ gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJ gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -google.golang.org/api v0.187.0 h1:Mxs7VATVC2v7CY+7Xwm4ndkX71hpElcvx0D1Ji/p1eo= -google.golang.org/api v0.187.0/go.mod h1:KIHlTc4x7N7gKKuVsdmfBXN13yEEWXWFURWY6SBp2gk= +google.golang.org/api v0.197.0 h1:x6CwqQLsFiA5JKAiGyGBjc2bNtHtLddhJCE2IKuhhcQ= +google.golang.org/api v0.197.0/go.mod h1:AuOuo20GoQ331nq7DquGHlU6d+2wN2fZ8O0ta60nRNw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20240701130421-f6361c86f094 h1:6whtk83KtD3FkGrVb2hFXuQ+ZMbCNdakARIn/aHMmG8= -google.golang.org/genproto v0.0.0-20240701130421-f6361c86f094/go.mod h1:Zs4wYw8z1zr6RNF4cwYb31mvN/EGaKAdQjNCF3DW6K4= -google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0= -google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 h1:BulPr26Jqjnd4eYDVe+YvyR7Yc2vJGkO5/0UxD0/jZU= +google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:hL97c3SYopEHblzpxRL4lSs523++l8DYxGM1FQiYmb4= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc= +google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= -google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= +google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -736,17 +731,17 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= -k8s.io/api v0.31.0 h1:b9LiSjR2ym/SzTOlfMHm1tr7/21aD7fSkqgD/CVJBCo= -k8s.io/api v0.31.0/go.mod h1:0YiFF+JfFxMM6+1hQei8FY8M7s1Mth+z/q7eF1aJkTE= -k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= -k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= +k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU= +k8s.io/api v0.31.1/go.mod h1:sbN1g6eY6XVLeqNsZGLnI5FwVseTrZX7Fv3O26rhAaI= +k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/N6E40= +k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.31.0 h1:m9jOiSr3FoSSL5WO9bjm1n6B9KROYYgNZOb4tyZ1lBc= -k8s.io/apimachinery v0.31.0/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= -k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8= -k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU= -k8s.io/cloud-provider v0.31.0 h1:qNOs78I2/7zQmyStfDtY2M7EdilUl9fCSYMcqBju/tA= -k8s.io/cloud-provider v0.31.0/go.mod h1:QgUPqLoL6aXhLlrNg1U4IrJk/PvvxgeOnT2ixkgnqT0= +k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= +k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0= +k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg= +k8s.io/cloud-provider v0.31.1 h1:40b6AgDizwm5eWratZbqubTHMob25VWr6NX2Ei5TwZA= +k8s.io/cloud-provider v0.31.1/go.mod h1:xAdkE7fdZdu9rKLuOZUMBfagu7bM+bas3iPux/2nLGg= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= @@ -760,18 +755,18 @@ k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-openapi v0.0.0-20240703190633-0aa61b46e8c2 h1:T5TEV4a+pEjc+j9Xui3MGGeoDLIN6uzZrx8NYotFMgQ= -k8s.io/kube-openapi v0.0.0-20240703190633-0aa61b46e8c2/go.mod h1:UxDHUPsUwTOOxSU+oXURfFBcAS6JwiRXTYqYwfuGowc= -k8s.io/kubectl v0.31.0 h1:kANwAAPVY02r4U4jARP/C+Q1sssCcN/1p9Nk+7BQKVg= -k8s.io/kubectl v0.31.0/go.mod h1:pB47hhFypGsaHAPjlwrNbvhXgmuAr01ZBvAIIUaI8d4= +k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 h1:1dWzkmJrrprYvjGwh9kEUxmcUV/CtNU8QM7h1FLWQOo= +k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA= +k8s.io/kubectl v0.31.1 h1:ih4JQJHxsEggFqDJEHSOdJ69ZxZftgeZvYo7M/cpp24= +k8s.io/kubectl v0.31.1/go.mod h1:aNuQoR43W6MLAtXQ/Bu4GDmoHlbhHKuyD49lmTC8eJM= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -kubevirt.io/api v1.2.2 h1:PeA937vsZawmKAsiiDQZJ/BbGH4OhEWsIzWrCNfmYXk= -kubevirt.io/api v1.2.2/go.mod h1:SbeR9ma4EwnaOZEUkh/lNz0kzYm5LPpEDE30vKXC5Zg= -kubevirt.io/containerized-data-importer-api v1.59.0 h1:GdDt9BlR0qHejpMaPfASbsG8JWDmBf1s7xZBj5W9qn0= -kubevirt.io/containerized-data-importer-api v1.59.0/go.mod h1:4yOGtCE7HvgKp7wftZZ3TBvDJ0x9d6N6KaRjRYcUFpE= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +kubevirt.io/api v1.3.1 h1:MoTNo/zvDlZ44c2ocXLPln8XTaQOeUodiYbEKrTCqv4= +kubevirt.io/api v1.3.1/go.mod h1:tCn7VAZktEvymk490iPSMPCmKM9UjbbfH2OsFR/IOLU= +kubevirt.io/containerized-data-importer-api v1.60.3 h1:kQEXi7scpzUa0RPf3/3MKk1Kmem0ZlqqiuK3kDF5L2I= +kubevirt.io/containerized-data-importer-api v1.60.3/go.mod h1:8mwrkZIdy8j/LmCyKt2wFXbiMavLUIqDaegaIF67CZs= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 h1:fZYvD3/Vnitfkx6IJxjLAk8ugnZQ7CXVYcRfkSKmuZY= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index b14cd3911..bc079af4c 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.23-node-20-0 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.23-node-20-3 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index a511a46cc..c32c40110 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.23-node-20-0 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.23-node-20-3 containerize ./hack/verify-licenses.sh go mod vendor diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 13bd1fca1..fa385cf17 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -853,7 +853,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string, annotations m AccessModes: []corev1.PersistentVolumeAccessMode{ config.StorageAccessType, }, - Resources: corev1.ResourceRequirements{ + Resources: corev1.VolumeResourceRequirements{ Requests: pvcRequest, }, } From cb8d1977ff47f1a03dd0af07335fe36f4984f9ed Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Thu, 26 Sep 2024 10:21:30 +0200 Subject: [PATCH 427/489] Support topology aware zones and regions in KubeVirt (#1860) * support topology aware zones and regions * adding tests * remove dead code --- .../provider/kubevirt/provider.go | 19 +++++ .../provider/kubevirt/provider_test.go | 17 ++++ .../location-zone-and-region-aware.yaml | 83 +++++++++++++++++++ .../provider/kubevirt/types/types.go | 7 ++ 4 files changed, 126 insertions(+) create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/location-zone-and-region-aware.yaml diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index fa385cf17..b57b833a6 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -76,6 +76,10 @@ const ( registrySource imageSource = "registry" // pvcSource defines the pvc source type for VM Disk Image. pvcSource imageSource = "pvc" + // topologyRegionKey and topologyZoneKey on PVC is a topology-aware volume provisioners will automatically set + // node affinity constraints on a PersistentVolume. + topologyRegionKey = "topology.kubernetes.io/region" + topologyZoneKey = "topology.kubernetes.io/zone" ) type provider struct { @@ -106,6 +110,8 @@ type Config struct { SecondaryDisks []SecondaryDisks NodeAffinityPreset NodeAffinityPreset TopologySpreadConstraints []corev1.TopologySpreadConstraint + Region string + Zone string } // StorageTarget represents targeted storage definition that will be used to provision VirtualMachine volumes. Currently, @@ -331,6 +337,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf(`failed to parse "topologySpreadConstraints" field: %w`, err) } + if rawConfig.VirtualMachine.Location != nil { + config.Zone = rawConfig.VirtualMachine.Location.Zone + config.Region = rawConfig.VirtualMachine.Location.Region + } + return &config, pconfig, nil } @@ -649,6 +660,14 @@ func (p *provider) newVirtualMachine(_ context.Context, c *Config, pc *providerc labels["cluster.x-k8s.io/cluster-name"] = c.ClusterName labels["cluster.x-k8s.io/role"] = "worker" + if c.Region != "" { + labels[topologyRegionKey] = c.Region + } + + if c.Zone != "" { + labels[topologyZoneKey] = c.Zone + } + var ( dataVolumeName = machine.Name annotations = map[string]string{} diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index a1ef8545a..d4ab2bc58 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -21,6 +21,7 @@ import ( "context" "embed" "html/template" + "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/types" "path" "reflect" "testing" @@ -68,6 +69,7 @@ type kubevirtProviderSpecConf struct { OsImageSource imageSource OsImageSourceURL string PullMethod cdiv1beta1.RegistryPullMethod + Location *types.Location } func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { @@ -101,6 +103,12 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { }, {{- end }} "virtualMachine": { + {{- if .Location }} + "location": { + "zone": "{{ .Location.Zone }}", + "region": "{{ .Location.Region }}" + }, + {{- end }} {{- if .Instancetype }} "instancetype": { "name": "{{ .Instancetype.Name }}", @@ -202,6 +210,15 @@ func TestNewVirtualMachine(t *testing.T) { }, }, }, + { + name: "location-zone-and-region-aware", + specConf: kubevirtProviderSpecConf{ + Location: &types.Location{ + Region: "europe-central", + Zone: "hh", + }, + }, + }, { name: "topologyspreadconstraints", specConf: kubevirtProviderSpecConf{TopologySpreadConstraint: true}, diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/location-zone-and-region-aware.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/location-zone-and-region-aware.yaml new file mode 100644 index 000000000..66c747353 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/location-zone-and-region-aware.yaml @@ -0,0 +1,83 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: location-zone-and-region-aware + topology.kubernetes.io/region: europe-central + topology.kubernetes.io/zone: hh + md: md-name + name: location-zone-and-region-aware + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: location-zone-and-region-aware + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + runStrategy: Once + template: + metadata: + creationTimestamp: null + annotations: + "ovn.kubernetes.io/allow_live_migration": "true" + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: location-zone-and-region-aware + md: md-name + topology.kubernetes.io/region: europe-central + topology.kubernetes.io/zone: hh + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: location-zone-and-region-aware + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index 52a2c2e63..e5505e714 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -57,6 +57,7 @@ type VirtualMachine struct { Template Template `json:"template,omitempty"` DNSPolicy providerconfigtypes.ConfigVarString `json:"dnsPolicy,omitempty"` DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` + Location *Location `json:"location,omitempty"` } // Flavor. @@ -125,6 +126,12 @@ type TopologySpreadConstraint struct { WhenUnsatisfiable providerconfigtypes.ConfigVarString `json:"whenUnsatisfiable,omitempty"` } +// Location describes the region and zone where the machines are created at and where the deployed resources will reside. +type Location struct { + Region string `json:"region,omitempty"` + Zone string `json:"zone,omitempty"` +} + func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { rawConfig := &RawConfig{} From 3ebc8529b89844557f2f3d3a9e2fdcf9480bf8f2 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Fri, 4 Oct 2024 10:29:17 +0200 Subject: [PATCH 428/489] Support Zone-Aware Topology in KubeVirt (#1861) * refactor kv labels * skip adding topology labels on multi zones/regions * fix tests * remove location test as it is not part of the VMCreation function * refactor newVirtualMachine func --- .../provider/kubevirt/provider.go | 89 ++++++++++++++++--- .../provider/kubevirt/provider_test.go | 20 +---- .../location-zone-and-region-aware.yaml | 83 ----------------- 3 files changed, 79 insertions(+), 113 deletions(-) delete mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/location-zone-and-region-aware.yaml diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index b57b833a6..ef5fd3901 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -42,6 +42,7 @@ import ( providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -561,6 +562,15 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus } func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { + c, _, err := p.getConfig(spec.ProviderSpec) + if err != nil { + return spec, err + } + + if err := appendTopologiesLabels(context.TODO(), c, spec.Labels); err != nil { + return spec, err + } + return spec, nil } @@ -605,8 +615,12 @@ func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *cl } userDataSecretName := fmt.Sprintf("userdata-%s-%s", machine.Name, strconv.Itoa(int(time.Now().Unix()))) + labels := map[string]string{} + if err := appendTopologiesLabels(ctx, c, labels); err != nil { + return nil, fmt.Errorf("failed to append labels: %w", err) + } - virtualMachine, err := p.newVirtualMachine(ctx, c, pc, machine, userDataSecretName, userdata, + virtualMachine, err := p.newVirtualMachine(c, pc, machine, labels, userDataSecretName, userdata, machineDeploymentNameAndRevisionForMachineGetter(ctx, machine, data.Client)) if err != nil { return nil, fmt.Errorf("could not create a VirtualMachine manifest %w", err) @@ -630,8 +644,8 @@ func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *cl return &kubeVirtServer{}, nil } -func (p *provider) newVirtualMachine(_ context.Context, c *Config, pc *providerconfigtypes.Config, machine *clusterv1alpha1.Machine, - userdataSecretName, userdata string, mdNameGetter machineDeploymentNameGetter) (*kubevirtv1.VirtualMachine, error) { +func (p *provider) newVirtualMachine(c *Config, pc *providerconfigtypes.Config, machine *clusterv1alpha1.Machine, + labels map[string]string, userdataSecretName, userdata string, mdNameGetter machineDeploymentNameGetter) (*kubevirtv1.VirtualMachine, error) { // We add the timestamp because the secret name must be different when we recreate the VMI // because its pod got deleted // The secret has an ownerRef on the VMI so garbace collection will take care of cleaning up. @@ -640,7 +654,7 @@ func (p *provider) newVirtualMachine(_ context.Context, c *Config, pc *providerc evictionStrategy := kubevirtv1.EvictionStrategyExternal resourceRequirements := kubevirtv1.ResourceRequirements{} - labels := map[string]string{"kubevirt.io/vm": machine.Name} + labels["kubevirt.io/vm"] = machine.Name //Add a common label to all VirtualMachines spawned by the same MachineDeployment (= MachineDeployment name). if mdName, err := mdNameGetter(); err == nil { labels[machineDeploymentLabelKey] = mdName @@ -660,14 +674,6 @@ func (p *provider) newVirtualMachine(_ context.Context, c *Config, pc *providerc labels["cluster.x-k8s.io/cluster-name"] = c.ClusterName labels["cluster.x-k8s.io/role"] = "worker" - if c.Region != "" { - labels[topologyRegionKey] = c.Region - } - - if c.Zone != "" { - labels[topologyZoneKey] = c.Zone - } - var ( dataVolumeName = machine.Name annotations = map[string]string{} @@ -977,3 +983,62 @@ func getTopologySpreadConstraints(config *Config, matchLabels map[string]string) }, } } + +func appendTopologiesLabels(ctx context.Context, c *Config, labels map[string]string) error { + if labels == nil { + labels = map[string]string{} + } + // trying to get region and zone from the storage class + err := getStorageTopologies(ctx, c.StorageClassName, c, labels) + if err != nil { + return fmt.Errorf("failed to get storage topologies: %w", err) + } + + // if regions are explicitly set then we read them from the configs + if c.Region != "" { + labels[topologyRegionKey] = c.Region + } + + if c.Zone != "" { + labels[topologyZoneKey] = c.Zone + } + + return nil +} + +func getStorageTopologies(ctx context.Context, storageClasName string, c *Config, labels map[string]string) error { + kubeClient, err := client.New(c.RestConfig, client.Options{}) + if err != nil { + return fmt.Errorf("failed to get kubevirt client: %w", err) + } + + sc := &storagev1.StorageClass{} + if err := kubeClient.Get(ctx, types.NamespacedName{Name: storageClasName}, sc); err != nil { + return err + } + + for _, topology := range sc.AllowedTopologies { + for _, exp := range topology.MatchLabelExpressions { + if exp.Key == topologyRegionKey { + if exp.Values == nil || len(exp.Values) != 1 { + // found multiple or no regions available. One zone/region is allowed + return nil + } + + labels[topologyRegionKey] = exp.Values[0] + continue + } + + if exp.Key == topologyZoneKey { + if exp.Values == nil || len(exp.Values) != 1 { + // found multiple or no zones available. One zone/region is allowed + return nil + } + + labels[topologyZoneKey] = exp.Values[0] + } + } + } + + return nil +} diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index d4ab2bc58..b5d867abc 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -21,7 +21,6 @@ import ( "context" "embed" "html/template" - "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/types" "path" "reflect" "testing" @@ -69,7 +68,6 @@ type kubevirtProviderSpecConf struct { OsImageSource imageSource OsImageSourceURL string PullMethod cdiv1beta1.RegistryPullMethod - Location *types.Location } func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { @@ -103,12 +101,6 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { }, {{- end }} "virtualMachine": { - {{- if .Location }} - "location": { - "zone": "{{ .Location.Zone }}", - "region": "{{ .Location.Region }}" - }, - {{- end }} {{- if .Instancetype }} "instancetype": { "name": "{{ .Instancetype.Name }}", @@ -210,15 +202,6 @@ func TestNewVirtualMachine(t *testing.T) { }, }, }, - { - name: "location-zone-and-region-aware", - specConf: kubevirtProviderSpecConf{ - Location: &types.Location{ - Region: "europe-central", - Zone: "hh", - }, - }, - }, { name: "topologyspreadconstraints", specConf: kubevirtProviderSpecConf{TopologySpreadConstraint: true}, @@ -279,9 +262,10 @@ func TestNewVirtualMachine(t *testing.T) { } // Do not rely on POD_NAMESPACE env variable, force to known value c.Namespace = testNamespace + labels := map[string]string{} // Check the created VirtualMachine - vm, _ := p.newVirtualMachine(context.TODO(), c, pc, machine, "udsn", userdata, fakeMachineDeploymentNameAndRevisionForMachineGetter()) + vm, _ := p.newVirtualMachine(c, pc, machine, labels, "udsn", userdata, fakeMachineDeploymentNameAndRevisionForMachineGetter()) vm.TypeMeta.APIVersion, vm.TypeMeta.Kind = kubevirtv1.VirtualMachineGroupVersionKind.ToAPIVersionAndKind() if !equality.Semantic.DeepEqual(vm, expectedVms[tt.name]) { diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/location-zone-and-region-aware.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/location-zone-and-region-aware.yaml deleted file mode 100644 index 66c747353..000000000 --- a/pkg/cloudprovider/provider/kubevirt/testdata/location-zone-and-region-aware.yaml +++ /dev/null @@ -1,83 +0,0 @@ -apiVersion: kubevirt.io/v1 -kind: VirtualMachine -metadata: - annotations: - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: location-zone-and-region-aware - topology.kubernetes.io/region: europe-central - topology.kubernetes.io/zone: hh - md: md-name - name: location-zone-and-region-aware - namespace: test-namespace -spec: - dataVolumeTemplates: - - metadata: - name: location-zone-and-region-aware - spec: - pvc: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 10Gi - storageClassName: longhorn - source: - http: - url: http://x.y.z.t/ubuntu.img - runStrategy: Once - template: - metadata: - creationTimestamp: null - annotations: - "ovn.kubernetes.io/allow_live_migration": "true" - "kubevirt.io/allow-pod-bridge-network-live-migration": "true" - labels: - cluster.x-k8s.io/cluster-name: cluster-name - cluster.x-k8s.io/role: worker - kubevirt.io/vm: location-zone-and-region-aware - md: md-name - topology.kubernetes.io/region: europe-central - topology.kubernetes.io/zone: hh - spec: - affinity: {} - domain: - devices: - disks: - - disk: - bus: virtio - name: datavolumedisk - - disk: - bus: virtio - name: cloudinitdisk - interfaces: - - name: default - bridge: {} - resources: - limits: - cpu: "2" - memory: 2Gi - requests: - cpu: "2" - memory: 2Gi - networks: - - name: default - pod: {} - terminationGracePeriodSeconds: 30 - topologyspreadconstraints: - - maxskew: 1 - topologykey: kubernetes.io/hostname - whenunsatisfiable: ScheduleAnyway - labelselector: - matchlabels: - md: md-name - volumes: - - dataVolume: - name: location-zone-and-region-aware - name: datavolumedisk - - cloudInitNoCloud: - secretRef: - name: udsn - name: cloudinitdisk - evictionStrategy: External From 6a9e720c35e5cebfaa4a39c7db2f48eb8669afe4 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Fri, 4 Oct 2024 18:49:06 +0200 Subject: [PATCH 429/489] Support KubeVirt Provider Network (#1862) * add provider network support * add provider network vpc support * support subnetGatewayIP defaulting * refactor kubevirt tests * address PR reviews * adding tests for kubevirt provider network --- .../provider/kubevirt/provider.go | 65 ++++++++++++++- .../provider/kubevirt/provider_test.go | 22 +++++ .../kubevirt/testdata/affinity-no-values.yaml | 1 - .../provider/kubevirt/testdata/affinity.yaml | 1 - .../kubevirt/testdata/custom-local-disk.yaml | 1 - .../kubevirt/testdata/http-image-source.yaml | 1 - .../instancetype-preference-custom.yaml | 1 - .../instancetype-preference-standard.yaml | 1 - .../testdata/kubeovn-provider-network.yaml | 81 +++++++++++++++++++ .../kubevirt/testdata/nominal-case.yaml | 1 - .../kubevirt/testdata/pvc-image-source.yaml | 1 - .../testdata/registry-image-source-pod.yaml | 1 - .../testdata/registry-image-source.yaml | 1 - .../kubevirt/testdata/secondary-disks.yaml | 1 - .../testdata/topologyspreadconstraints.yaml | 1 - .../use-storage-as-storage-target.yaml | 1 - .../provider/kubevirt/types/types.go | 31 +++++-- 17 files changed, 193 insertions(+), 19 deletions(-) create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/kubeovn-provider-network.yaml diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index ef5fd3901..4a2d0f608 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -19,8 +19,10 @@ package kubevirt import ( "context" "encoding/base64" + "encoding/json" "errors" "fmt" + "net" "net/url" "os" "strconv" @@ -113,6 +115,11 @@ type Config struct { TopologySpreadConstraints []corev1.TopologySpreadConstraint Region string Zone string + + ProviderNetworkName string + SubnetName string + SubnetCIDRBlock string + SubnetGatewayIP string } // StorageTarget represents targeted storage definition that will be used to provision VirtualMachine volumes. Currently, @@ -343,6 +350,15 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p config.Region = rawConfig.VirtualMachine.Location.Region } + if rawConfig.VirtualMachine.ProviderNetwork != nil { + config.ProviderNetworkName = rawConfig.VirtualMachine.ProviderNetwork.Name + if rawConfig.VirtualMachine.ProviderNetwork.VPC.Subnet != nil { + config.SubnetName = rawConfig.VirtualMachine.ProviderNetwork.VPC.Subnet.Name + config.SubnetCIDRBlock = rawConfig.VirtualMachine.ProviderNetwork.VPC.Subnet.CIDRBlock + config.SubnetGatewayIP = rawConfig.VirtualMachine.ProviderNetwork.VPC.Subnet.GatewayIP + } + } + return &config, pconfig, nil } @@ -686,9 +702,14 @@ func (p *provider) newVirtualMachine(c *Config, pc *providerconfigtypes.Config, annotations["kubevirt.io/ignitiondata"] = userdata } - annotations["ovn.kubernetes.io/allow_live_migration"] = "true" annotations["kubevirt.io/allow-pod-bridge-network-live-migration"] = "true" + if strings.ToLower(c.ProviderNetworkName) == "kubeovn" { + if err := setOVNAnnotations(c, annotations); err != nil { + return nil, fmt.Errorf("failed to set OVN annotations: %w", err) + } + } + for k, v := range machine.Annotations { if strings.HasPrefix(k, "cdi.kubevirt.io") { dvAnnotations[k] = v @@ -1042,3 +1063,45 @@ func getStorageTopologies(ctx context.Context, storageClasName string, c *Config return nil } + +func setOVNAnnotations(c *Config, annotations map[string]string) error { + annotations["ovn.kubernetes.io/allow_live_migration"] = "true" + + if c.SubnetName != "" { + annotations["ovn.kubernetes.io/logical_switch"] = c.SubnetName + } + + var subnetGatewayIP string + if c.SubnetGatewayIP == "" { + _, ipNet, err := net.ParseCIDR(c.SubnetCIDRBlock) + if err != nil { + return err + } + + firstIP := ipNet.IP.To4() + if firstIP == nil { + return errors.New("invalid IPv4 address") + } + + firstIP[3]++ + subnetGatewayIP = firstIP.String() + } else { + subnetGatewayIP = c.SubnetGatewayIP + } + + routes := []struct { + Gw string `json:"gw"` + }{ + { + Gw: subnetGatewayIP, + }, + } + marshalledRoutes, err := json.Marshal(routes) + if err != nil { + return err + } + + annotations["ovn.kubernetes.io/routes"] = string(marshalledRoutes) + + return nil +} diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index b5d867abc..8695e21f0 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -21,6 +21,7 @@ import ( "context" "embed" "html/template" + "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/types" "path" "reflect" "testing" @@ -68,6 +69,7 @@ type kubevirtProviderSpecConf struct { OsImageSource imageSource OsImageSourceURL string PullMethod cdiv1beta1.RegistryPullMethod + ProviderNetwork *types.ProviderNetwork } func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { @@ -101,6 +103,18 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { }, {{- end }} "virtualMachine": { + {{- if .ProviderNetwork }} + "providerNetwork": { + "name": "kubeovn", + "vpc": { + "name": "test-vpc", + "subnet": { + "name": "test-subnet", + "cidrBlock": "10.10.0.0/16" + } + } + }, + {{- end }} {{- if .Instancetype }} "instancetype": { "name": "{{ .Instancetype.Name }}", @@ -202,6 +216,14 @@ func TestNewVirtualMachine(t *testing.T) { }, }, }, + { + name: "kubeovn-provider-network", + specConf: kubevirtProviderSpecConf{ + ProviderNetwork: &types.ProviderNetwork{Name: "KubeOVN", VPC: types.VPC{Name: "test-vpc", Subnet: &types.Subnet{ + Name: "test-subnet", + CIDRBlock: "10.10.0.0/24", + }}}}, + }, { name: "topologyspreadconstraints", specConf: kubevirtProviderSpecConf{TopologySpreadConstraint: true}, diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml index 5d55f2071..8ff6120fe 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml @@ -29,7 +29,6 @@ spec: template: metadata: annotations: - "ovn.kubernetes.io/allow_live_migration": "true" "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml index 8d206aa0c..b694f8230 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml @@ -29,7 +29,6 @@ spec: template: metadata: annotations: - "ovn.kubernetes.io/allow_live_migration": "true" "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml index 767303e29..0f70140c0 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml @@ -30,7 +30,6 @@ spec: metadata: creationTimestamp: null annotations: - "ovn.kubernetes.io/allow_live_migration": "true" "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml index 93fe25186..548f617e1 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml @@ -29,7 +29,6 @@ spec: metadata: creationTimestamp: null annotations: - "ovn.kubernetes.io/allow_live_migration": "true" "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: kubevirt.io/vm: http-image-source diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml index 4830ac6c6..de113ac3c 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml @@ -35,7 +35,6 @@ spec: template: metadata: annotations: - "ovn.kubernetes.io/allow_live_migration": "true" "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml index 20ddfe91e..7aa94bfb7 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml @@ -35,7 +35,6 @@ spec: metadata: creationTimestamp: null annotations: - "ovn.kubernetes.io/allow_live_migration": "true" "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/kubeovn-provider-network.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/kubeovn-provider-network.yaml new file mode 100644 index 000000000..520d074b0 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/kubeovn-provider-network.yaml @@ -0,0 +1,81 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: kubeovn-provider-network + md: md-name + name: kubeovn-provider-network + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: kubeovn-provider-network + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + runStrategy: Once + template: + metadata: + creationTimestamp: null + annotations: + ovn.kubernetes.io/allow_live_migration: "true" + ovn.kubernetes.io/logical_switch: test-subnet + ovn.kubernetes.io/routes: '[{"gw":"10.10.0.1"}]' + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: kubeovn-provider-network + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: kubeovn-provider-network + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml index c679f0da1..3aa314504 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml @@ -29,7 +29,6 @@ spec: metadata: creationTimestamp: null annotations: - "ovn.kubernetes.io/allow_live_migration": "true" "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml index 7caf5b201..713dbeddd 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml @@ -30,7 +30,6 @@ spec: metadata: creationTimestamp: null annotations: - "ovn.kubernetes.io/allow_live_migration": "true" "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: kubevirt.io/vm: pvc-image-source diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml index f632c05f8..17bc96142 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml @@ -30,7 +30,6 @@ spec: metadata: creationTimestamp: null annotations: - "ovn.kubernetes.io/allow_live_migration": "true" "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: kubevirt.io/vm: registry-image-source-pod diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml index 3bfcd1a68..4a950ead1 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml @@ -30,7 +30,6 @@ spec: metadata: creationTimestamp: null annotations: - "ovn.kubernetes.io/allow_live_migration": "true" "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: kubevirt.io/vm: registry-image-source diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml index d563ea7eb..2ea6e2703 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml @@ -55,7 +55,6 @@ spec: metadata: creationTimestamp: null annotations: - "ovn.kubernetes.io/allow_live_migration": "true" "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml index ebd161b71..670150045 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml @@ -29,7 +29,6 @@ spec: metadata: creationTimestamp: null annotations: - "ovn.kubernetes.io/allow_live_migration": "true" "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml index 7ba8c9cef..33fbae826 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml @@ -30,7 +30,6 @@ spec: metadata: creationTimestamp: null annotations: - "ovn.kubernetes.io/allow_live_migration": "true" "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index e5505e714..e1e26c787 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -53,11 +53,12 @@ type VirtualMachine struct { // Instancetype is optional. Instancetype *kubevirtv1.InstancetypeMatcher `json:"instancetype,omitempty"` // Preference is optional. - Preference *kubevirtv1.PreferenceMatcher `json:"preference,omitempty"` - Template Template `json:"template,omitempty"` - DNSPolicy providerconfigtypes.ConfigVarString `json:"dnsPolicy,omitempty"` - DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` - Location *Location `json:"location,omitempty"` + Preference *kubevirtv1.PreferenceMatcher `json:"preference,omitempty"` + Template Template `json:"template,omitempty"` + DNSPolicy providerconfigtypes.ConfigVarString `json:"dnsPolicy,omitempty"` + DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` + Location *Location `json:"location,omitempty"` + ProviderNetwork *ProviderNetwork `json:"providerNetwork,omitempty"` } // Flavor. @@ -132,6 +133,26 @@ type Location struct { Zone string `json:"zone,omitempty"` } +// ProviderNetwork describes the infra cluster network fabric that is being used. +type ProviderNetwork struct { + Name string `json:"name"` + VPC VPC `json:"vpc"` +} + +// VPC is a virtual network dedicated to a single tenant within a KubeVirt, where the resources in the VPC +// is isolated from any other resources within the KubeVirt infra cluster. +type VPC struct { + Name string `json:"name"` + Subnet *Subnet `json:"subnet,omitempty"` +} + +// Subnet a smaller, segmented portion of a larger network, like a Virtual Private Cloud (VPC). +type Subnet struct { + Name string `json:"name"` + CIDRBlock string `json:"cidrBlock"` + GatewayIP string `json:"gatewayIP,omitempty"` +} + func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { rawConfig := &RawConfig{} From 0fb8b75ceb46509f8a656494090cf0802c75c052 Mon Sep 17 00:00:00 2001 From: Mohamed Rafraf <81432497+mohamed-rafraf@users.noreply.github.com> Date: Tue, 8 Oct 2024 13:47:09 +0200 Subject: [PATCH 430/489] fix template bug and add reboot action (#1864) --- .../plugins/tinkerbell/client/template.go | 38 ++++++++++++++----- .../plugins/tinkerbell/client/workflow.go | 2 +- .../baremetal/plugins/tinkerbell/driver.go | 2 +- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go index cc1078ef6..924c1a38c 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go @@ -55,10 +55,10 @@ type Template struct { } const ( - fsType = "ext4" - defaultInterpreter = "/bin/sh -c" - hardwareDisk1 = "{{ index .Hardware.Disks 0 }}" - + fsType = "ext4" + defaultInterpreter = "/bin/sh -c" + hardwareDisk1 = "{{ index .Hardware.Disks 0 }}" + hardwareName = "{{.hardware_name}}" ProvisionWorkerNodeTemplate = "provision-worker-node" ) @@ -90,14 +90,14 @@ func (t *TemplateClient) Delete(ctx context.Context, namespacedName types.Namesp } // CreateTemplate creates a Tinkerbell Template in the Kubernetes cluster. -func (t *TemplateClient) CreateTemplate(ctx context.Context, hardware *tinkv1alpha1.Hardware, namespace, osImageURL string) error { +func (t *TemplateClient) CreateTemplate(ctx context.Context, namespace, osImageURL string) error { template := &tinkv1alpha1.Template{} if err := t.tinkclient.Get(ctx, types.NamespacedName{ Name: ProvisionWorkerNodeTemplate, Namespace: namespace, }, template); err != nil { if kerrors.IsNotFound(err) { - data, err := getTemplate(hardware, osImageURL) + data, err := getTemplate(osImageURL) if err != nil { return err } @@ -122,14 +122,15 @@ func (t *TemplateClient) CreateTemplate(ctx context.Context, hardware *tinkv1alp return nil } -func getTemplate(hardware *tinkv1alpha1.Hardware, osImageURL string) (string, error) { +func getTemplate(osImageURL string) (string, error) { actions := []Action{ createWipeDiskAction(), createStreamUbuntuImageAction(hardwareDisk1, osImageURL), createGrowPartitionAction(hardwareDisk1), createNetworkConfigAction(), createCloudInitConfigAction(), - decodeCloudInitFile(hardware.Name), + decodeCloudInitFile(hardwareName), + createRebootAction(), } task := Task{ @@ -244,7 +245,7 @@ func createCloudInitConfigAction() Action { Environment: map[string]string{ "DEST_DISK": "{{ index .Hardware.Disks 0 }}3", "FS_TYPE": fsType, - "DEST_PATH": "{{.dst_path}}", + "DEST_PATH": fmt.Sprintf("/tmp/%s-bootstrap-config", hardwareName), "CONTENTS": "{{.cloud_init_script}}", "UID": "0", "GID": "0", @@ -264,7 +265,24 @@ func decodeCloudInitFile(hardwareName string) Action { "FS_TYPE": fsType, "CHROOT": "y", "DEFAULT_INTERPRETER": "/bin/sh -c", - "CMD_LINE": fmt.Sprintf("cat /tmp/%s-bootstrap-config | base64 -d > /etc/cloud/cloud.cfg.d/%s-cloud-init.cfg", hardwareName, hardwareName), + "CMD_LINE": fmt.Sprintf("cat /tmp/%s-bootstrap-config | base64 -d > '/etc/cloud/cloud.cfg.d/%s-cloud-init.cfg'", hardwareName, hardwareName), + }, + } +} + +func createRebootAction() Action { + return Action{ + Name: "reboot-action", + Image: "ghcr.io/jacobweinstock/waitdaemon:0.1.1", + Pid: "host", + Timeout: 90, + Command: []string{"reboot"}, + Environment: map[string]string{ + "IMAGE": "alpine", + "WAIT_SECONDS": "10", + }, + Volumes: []string{ + "/var/run/docker.sock:/var/run/docker.sock", }, } } diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go index 06c0e9a55..da3d4bd3c 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go @@ -67,7 +67,7 @@ func (w *WorkflowClient) CreateWorkflow(ctx context.Context, userData, templateR HardwareRef: hardware.GetName(), HardwareMap: map[string]string{ "device_1": hardware.GetMACAddress(), - "dst_path": fmt.Sprintf("/tmp/%s-bootstrap-config", hardware.Name), + "hardware_name": hardware.GetName(), "cloud_init_script": base64.StdEncoding.EncodeToString([]byte(userData)), "interface_name": ifaceConfig.IfaceName, "cidr": convertNetmaskToCIDR(ifaceConfig.IP), diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go index ea95e78a6..3461497b2 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go @@ -123,7 +123,7 @@ func (d *driver) ProvisionServer(ctx context.Context, _ *zap.SugaredLogger, meta } // Create template if it doesn't exist - err = d.TemplateClient.CreateTemplate(ctx, hardware, d.HardwareRef.Namespace, d.OSImageURL) + err = d.TemplateClient.CreateTemplate(ctx, d.HardwareRef.Namespace, d.OSImageURL) if err != nil { return nil, err } From cda075d98012771d8013eae0c142dbc7cab6c304 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 8 Oct 2024 14:54:10 +0200 Subject: [PATCH 431/489] Support KubeOVN VPC and Subnet (#1863) * support kubeovn provider network * kubeovn go mod imports * register kubeovn resource subnet and vpc * support kubeovn subnet fetch * refactor provider networks * fix provider networks boilerplate * refactor kubevirt provider --- cmd/machine-controller/main.go | 4 + go.mod | 15 +- go.sum | 190 +++++++++++++++--- .../provider/kubevirt/provider.go | 41 +++- .../providernetworks/kubeovn/provider.go | 60 ++++++ .../providernetworks/provider_networks.go | 48 +++++ 6 files changed, 319 insertions(+), 39 deletions(-) create mode 100644 pkg/cloudprovider/provider/kubevirt/providernetworks/kubeovn/provider.go create mode 100644 pkg/cloudprovider/provider/kubevirt/providernetworks/provider_networks.go diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 6e1d25a90..16cc72f43 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -27,6 +27,7 @@ import ( "time" "github.com/go-logr/zapr" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" @@ -227,6 +228,9 @@ func main() { if err := clusterv1alpha1.AddToScheme(scheme.Scheme); err != nil { log.Fatalw("Failed to add api to scheme", "api", clusterv1alpha1.SchemeGroupVersion, zap.Error(err)) } + if err := kubeovnv1.AddToScheme(scheme.Scheme); err != nil { + log.Fatalw("Failed to add kubeovn api to scheme", "api", clusterv1alpha1.SchemeGroupVersion, zap.Error(err)) + } cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) if err != nil { diff --git a/go.mod b/go.mod index 0385bc162..46ac4bf25 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module k8c.io/machine-controller -go 1.22.3 +go 1.22.7 toolchain go1.23.1 @@ -28,6 +28,7 @@ require ( github.com/gophercloud/gophercloud v1.14.0 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb github.com/hetznercloud/hcloud-go/v2 v2.13.1 + github.com/kubeovn/kube-ovn v1.12.26 github.com/linode/linodego v1.40.0 github.com/nutanix-cloud-native/prism-go-client v0.5.1 github.com/packethost/packngo v0.31.0 @@ -52,7 +53,7 @@ require ( k8s.io/api v0.31.1 k8s.io/apiextensions-apiserver v0.31.1 k8s.io/apimachinery v0.31.1 - k8s.io/client-go v0.31.1 + k8s.io/client-go v1.5.2 k8s.io/cloud-provider v0.31.1 k8s.io/klog v1.0.0 k8s.io/kubectl v0.31.1 @@ -114,7 +115,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect @@ -146,6 +147,7 @@ require ( github.com/onsi/gomega v1.34.1 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect + github.com/ovn-org/libovsdb v0.7.0 // indirect github.com/peterhellberg/link v1.2.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.59.1 // indirect @@ -181,8 +183,13 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect + k8s.io/kube-openapi v0.30.0 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) + +replace ( + k8s.io/client-go v1.5.2 => k8s.io/client-go v0.31.1 + k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f +) diff --git a/go.sum b/go.sum index 5f7cef617..73250b583 100644 --- a/go.sum +++ b/go.sum @@ -49,7 +49,7 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240905143811-b2ab5b7c9c14 h1:9uqKGeUuok/9Q5B5DzDM+bVgyEZVruzaflXw8WiaZ+Y= github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240905143811-b2ab5b7c9c14/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= @@ -65,7 +65,6 @@ github.com/aliyun/alibaba-cloud-sdk-go v1.63.15 h1:r2uwBUQhLhcPzaWz9tRJqc8MjYwHb github.com/aliyun/alibaba-cloud-sdk-go v1.63.15/go.mod h1:SOSDHfe1kX91v3W5QiBsWSLqeLxImobbMX1mxrFHsVQ= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= -github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g= @@ -101,9 +100,15 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= +github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -121,7 +126,6 @@ github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= @@ -148,13 +152,14 @@ github.com/fullstorydev/grpcurl v1.8.7 h1:xJWosq3BQovQ4QrdPO72OrPiWuGgEsxY8ldYsJ github.com/fullstorydev/grpcurl v1.8.7/go.mod h1:pVtM4qe3CMoLaIzYS8uvTuDj2jVYmXqMUkZeijnXp/E= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= -github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -166,10 +171,8 @@ github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= @@ -196,10 +199,14 @@ github.com/go-playground/validator/v10 v10.10.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4 github.com/go-resty/resty/v2 v2.14.0 h1:/rhkzsAqGQkozwfKS5aFAbb6TyKd3zyFRWcdRXLPCAU= github.com/go-resty/resty/v2 v2.14.0/go.mod h1:IW6mekUOsElt9C7oWr0XRt9BNSD6D5rr9mhk6NjmNHg= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= @@ -225,10 +232,12 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 h1:0VpGH+cDhbDtdcweoyCVsF3fhN8kejK6rFe/2FFX2nU= +github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49/go.mod h1:BkkQ4L1KS1xMt2aWSPStnn55ChGC0DPOn2FQYj+f25M= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -239,6 +248,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -249,6 +259,7 @@ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k= github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= @@ -261,11 +272,9 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gT github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= -github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gophercloud/gophercloud v1.14.0 h1:Bt9zQDhPrbd4qX7EILGmy+i7GP35cc+AAL2+wIJpUE8= github.com/gophercloud/gophercloud v1.14.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= -github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -282,6 +291,7 @@ github.com/hetznercloud/hcloud-go/v2 v2.13.1 h1:jq0GP4QaYE5d8xR/Zw17s9qoaESRJMXf github.com/hetznercloud/hcloud-go/v2 v2.13.1/go.mod h1:dhix40Br3fDiBhwaSG/zgaYOFFddpfBm/6R1Zz0IiF0= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= @@ -311,14 +321,18 @@ github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJ github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubeovn/kube-ovn v1.12.26 h1:pp4PfEIee9DZ/3vGQqIR9xVWucbqF9HHn0DRzkyEMLs= +github.com/kubeovn/kube-ovn v1.12.26/go.mod h1:L0PL79dsqsnA65Z1rjTFwQMap59yc3e7kIqGO53HAlA= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/linode/linodego v1.40.0 h1:7ESY0PwK94hoggoCtIroT1Xk6b1flrFBNZ6KwqbTqlI= @@ -337,7 +351,6 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -350,7 +363,6 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= -github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= @@ -361,25 +373,62 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= +github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= +github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc= +github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk= +github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= +github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= +github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= +github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= +github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/ginkgo/v2 v2.17.2/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= +github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= +github.com/onsi/ginkgo/v2 v2.20.0/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= github.com/onsi/ginkgo/v2 v2.20.1 h1:YlVIbqct+ZmnEph770q9Q7NVAz4wwIiVNahee6JyUzo= github.com/onsi/ginkgo/v2 v2.20.1/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= -github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= +github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw= +github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw= +github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4= +github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= +github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= +github.com/onsi/gomega v1.33.0/go.mod h1:+925n5YtiFsLzzafLUHzVMBpvvRAzrydIBiSIxjX3wY= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/ovn-org/libovsdb v0.7.0 h1:owk3MHhaJ0gs0dWvTBtj7lPGEzbcyPrDYerEFqPXO/Y= +github.com/ovn-org/libovsdb v0.7.0/go.mod h1:dJbxEaalQl83nn904K32FaMjlH/qOObZ0bj4ejQ78AI= github.com/packethost/packngo v0.31.0 h1:LLH90ardhULWbagBIc3I3nl2uU75io0a7AwY6hyi0S4= github.com/packethost/packngo v0.31.0/go.mod h1:Io6VJqzkiqmIEQbpOjeIw9v8q9PfcTEq8TEY/tMQsfw= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= @@ -405,6 +454,7 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= @@ -422,6 +472,7 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -430,6 +481,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tinkerbell/tink v0.10.1 h1:mxdPQf7n4nB/AVdjbqCm5c98vsITU35g7Yw5cdOWmCw= @@ -448,7 +500,6 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -468,8 +519,8 @@ go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= +go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= +go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -489,12 +540,21 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -503,6 +563,7 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= @@ -517,11 +578,19 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -535,7 +604,6 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -543,13 +611,29 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -563,7 +647,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= @@ -582,51 +668,84 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= @@ -644,13 +763,25 @@ golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= +golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -697,6 +828,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= @@ -715,7 +849,6 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -745,22 +878,22 @@ k8s.io/cloud-provider v0.31.1/go.mod h1:xAdkE7fdZdu9rKLuOZUMBfagu7bM+bas3iPux/2n k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo/v2 v2.0.0-20240812201722-3b05ca7b6e59/go.mod h1:VH3AT8AaQOqiGjMF9p0/IM1Dj+82ZwjfxUP1IxaHE+8= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 h1:1dWzkmJrrprYvjGwh9kEUxmcUV/CtNU8QM7h1FLWQOo= -k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA= +k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f h1:bnWtxXWdAl5bVOCEPoNdvMkyj6cTW3zxHuwKIakuV9w= +k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f/go.mod h1:G0W3eI9gG219NHRq3h5uQaRBl4pj4ZpwzRP5ti8y770= k8s.io/kubectl v0.31.1 h1:ih4JQJHxsEggFqDJEHSOdJ69ZxZftgeZvYo7M/cpp24= k8s.io/kubectl v0.31.1/go.mod h1:aNuQoR43W6MLAtXQ/Bu4GDmoHlbhHKuyD49lmTC8eJM= -k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v1.3.1 h1:MoTNo/zvDlZ44c2ocXLPln8XTaQOeUodiYbEKrTCqv4= @@ -775,7 +908,6 @@ sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hw sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 4a2d0f608..11e890a88 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -37,6 +37,8 @@ import ( clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" + "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/providernetworks" + "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/providernetworks/kubeovn" kubevirttypes "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" controllerutil "k8c.io/machine-controller/pkg/controller/util" @@ -351,11 +353,8 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } if rawConfig.VirtualMachine.ProviderNetwork != nil { - config.ProviderNetworkName = rawConfig.VirtualMachine.ProviderNetwork.Name - if rawConfig.VirtualMachine.ProviderNetwork.VPC.Subnet != nil { - config.SubnetName = rawConfig.VirtualMachine.ProviderNetwork.VPC.Subnet.Name - config.SubnetCIDRBlock = rawConfig.VirtualMachine.ProviderNetwork.VPC.Subnet.CIDRBlock - config.SubnetGatewayIP = rawConfig.VirtualMachine.ProviderNetwork.VPC.Subnet.GatewayIP + if err := validateProviderNetwork(&config, rawConfig.VirtualMachine.ProviderNetwork); err != nil { + return nil, nil, fmt.Errorf(`failed to validate "providerNetwork": %w`, err) } } @@ -704,7 +703,7 @@ func (p *provider) newVirtualMachine(c *Config, pc *providerconfigtypes.Config, annotations["kubevirt.io/allow-pod-bridge-network-live-migration"] = "true" - if strings.ToLower(c.ProviderNetworkName) == "kubeovn" { + if strings.ToLower(c.ProviderNetworkName) == string(providernetworks.KubeOVN) { if err := setOVNAnnotations(c, annotations); err != nil { return nil, fmt.Errorf("failed to set OVN annotations: %w", err) } @@ -1105,3 +1104,33 @@ func setOVNAnnotations(c *Config, annotations map[string]string) error { return nil } + +func validateProviderNetwork(config *Config, providerNetwork *kubevirttypes.ProviderNetwork) error { + config.ProviderNetworkName = providerNetwork.Name + if providerNetwork.VPC.Subnet != nil { + config.SubnetName = providerNetwork.VPC.Subnet.Name + kvClient, err := client.New(config.RestConfig, client.Options{}) + if err != nil { + return fmt.Errorf("failed to create kubevirt client: %w", err) + } + + providerNetworks, err := kubeovn.New(kvClient) + if err != nil { + return fmt.Errorf("failed to create kubeovn providerNetworks: %w", err) + } + + config.SubnetCIDRBlock = providerNetwork.VPC.Subnet.CIDRBlock + if config.SubnetCIDRBlock == "" { + subnet, err := providerNetworks.GetVPCSubnet(context.Background(), config.SubnetName) + if err != nil { + return fmt.Errorf("failed to get vpcSubnet: %w", err) + } + + config.SubnetCIDRBlock = subnet.CIDRBlock + } + + config.SubnetGatewayIP = providerNetwork.VPC.Subnet.GatewayIP + } + + return nil +} diff --git a/pkg/cloudprovider/provider/kubevirt/providernetworks/kubeovn/provider.go b/pkg/cloudprovider/provider/kubevirt/providernetworks/kubeovn/provider.go new file mode 100644 index 000000000..cac26d9b5 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/providernetworks/kubeovn/provider.go @@ -0,0 +1,60 @@ +/* +Copyright 2024 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubeovn + +import ( + "context" + "fmt" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + + "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/providernetworks" + + "k8s.io/apimachinery/pkg/types" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +type kubeOVNProviderNetwork struct { + client ctrlruntimeclient.Client +} + +func New(client ctrlruntimeclient.Client) (providernetworks.ProviderNetwork, error) { + return &kubeOVNProviderNetwork{client: client}, nil +} + +func (k *kubeOVNProviderNetwork) GetVPC(ctx context.Context, vpcName string) (*providernetworks.VPC, error) { + vpc := &kubeovnv1.Vpc{} + if err := k.client.Get(ctx, types.NamespacedName{Name: vpcName}, vpc); err != nil { + return nil, fmt.Errorf("failed to get VPC %s: %w", vpcName, err) + } + + return &providernetworks.VPC{ + Name: vpc.Name, + }, nil +} + +func (k *kubeOVNProviderNetwork) GetVPCSubnet(ctx context.Context, subnetName string) (*providernetworks.Subnet, error) { + vpcSubnet := &kubeovnv1.Subnet{} + if err := k.client.Get(ctx, types.NamespacedName{Name: subnetName}, vpcSubnet); err != nil { + return nil, fmt.Errorf("failed to get VPC subnet %s: %w", subnetName, err) + } + + return &providernetworks.Subnet{ + Name: vpcSubnet.Name, + CIDRBlock: vpcSubnet.Spec.CIDRBlock, + }, nil +} diff --git a/pkg/cloudprovider/provider/kubevirt/providernetworks/provider_networks.go b/pkg/cloudprovider/provider/kubevirt/providernetworks/provider_networks.go new file mode 100644 index 000000000..abb724913 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/providernetworks/provider_networks.go @@ -0,0 +1,48 @@ +/* +Copyright 2024 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package providernetworks + +import ( + "context" +) + +type SupportedProviderNetworks string + +const ( + KubeOVN SupportedProviderNetworks = "kubeovn" +) + +// ProviderNetwork describes the infra cluster network fabric that is being used. These fabrics could be a as simple cni +// specific features up to full-blown networking components such as VPCs and Subnets. +type ProviderNetwork interface { + GetVPC(ctx context.Context, vpcName string) (*VPC, error) + GetVPCSubnet(ctx context.Context, subnetName string) (*Subnet, error) +} + +// VPC is a virtual network dedicated to a single tenant within a KubeVirt, where the resources in the VPC +// is isolated from any other resources within the KubeVirt infra cluster. +type VPC struct { + Name string `json:"name"` +} + +// Subnet a smaller, segmented portion of a larger network, like a Virtual Private Cloud (VPC). +type Subnet struct { + Name string `json:"name"` + CIDRBlock string `json:"cidrBlock"` + GatewayIP string `json:"gatewayIP,omitempty"` + ExcludeIPs []string `json:"excludeIP,omitempty"` +} From 889019806083b17d6afb2d755936c92cac29a1f3 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Wed, 9 Oct 2024 16:25:10 +0200 Subject: [PATCH 432/489] Revert Changes to KubeVirt Provider Networks (#1865) * refactor kv provider networks * fix kubevirt tests * refactor kubevirt tests * adjust kubevirt testing files --- cmd/machine-controller/main.go | 4 - go.mod | 15 +- go.sum | 190 +++--------------- .../provider/kubevirt/provider.go | 80 +------- .../provider/kubevirt/provider_test.go | 6 +- .../providernetworks/kubeovn/provider.go | 60 ------ .../providernetworks/provider_networks.go | 48 ----- .../kubevirt/testdata/affinity-no-values.yaml | 1 + .../provider/kubevirt/testdata/affinity.yaml | 1 + .../kubevirt/testdata/custom-local-disk.yaml | 1 + .../kubevirt/testdata/http-image-source.yaml | 1 + .../instancetype-preference-custom.yaml | 1 + .../instancetype-preference-standard.yaml | 1 + .../testdata/kubeovn-provider-network.yaml | 1 - .../kubevirt/testdata/nominal-case.yaml | 1 + .../kubevirt/testdata/pvc-image-source.yaml | 1 + .../testdata/registry-image-source-pod.yaml | 1 + .../testdata/registry-image-source.yaml | 1 + .../kubevirt/testdata/secondary-disks.yaml | 1 + .../testdata/topologyspreadconstraints.yaml | 1 + .../use-storage-as-storage-target.yaml | 1 + .../provider/kubevirt/types/types.go | 4 +- 22 files changed, 54 insertions(+), 367 deletions(-) delete mode 100644 pkg/cloudprovider/provider/kubevirt/providernetworks/kubeovn/provider.go delete mode 100644 pkg/cloudprovider/provider/kubevirt/providernetworks/provider_networks.go diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 16cc72f43..6e1d25a90 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -27,7 +27,6 @@ import ( "time" "github.com/go-logr/zapr" - kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" @@ -228,9 +227,6 @@ func main() { if err := clusterv1alpha1.AddToScheme(scheme.Scheme); err != nil { log.Fatalw("Failed to add api to scheme", "api", clusterv1alpha1.SchemeGroupVersion, zap.Error(err)) } - if err := kubeovnv1.AddToScheme(scheme.Scheme); err != nil { - log.Fatalw("Failed to add kubeovn api to scheme", "api", clusterv1alpha1.SchemeGroupVersion, zap.Error(err)) - } cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) if err != nil { diff --git a/go.mod b/go.mod index 46ac4bf25..0385bc162 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module k8c.io/machine-controller -go 1.22.7 +go 1.22.3 toolchain go1.23.1 @@ -28,7 +28,6 @@ require ( github.com/gophercloud/gophercloud v1.14.0 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb github.com/hetznercloud/hcloud-go/v2 v2.13.1 - github.com/kubeovn/kube-ovn v1.12.26 github.com/linode/linodego v1.40.0 github.com/nutanix-cloud-native/prism-go-client v0.5.1 github.com/packethost/packngo v0.31.0 @@ -53,7 +52,7 @@ require ( k8s.io/api v0.31.1 k8s.io/apiextensions-apiserver v0.31.1 k8s.io/apimachinery v0.31.1 - k8s.io/client-go v1.5.2 + k8s.io/client-go v0.31.1 k8s.io/cloud-provider v0.31.1 k8s.io/klog v1.0.0 k8s.io/kubectl v0.31.1 @@ -115,7 +114,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect @@ -147,7 +146,6 @@ require ( github.com/onsi/gomega v1.34.1 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect - github.com/ovn-org/libovsdb v0.7.0 // indirect github.com/peterhellberg/link v1.2.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.59.1 // indirect @@ -183,13 +181,8 @@ require ( gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.30.0 // indirect + k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) - -replace ( - k8s.io/client-go v1.5.2 => k8s.io/client-go v0.31.1 - k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f -) diff --git a/go.sum b/go.sum index 73250b583..5f7cef617 100644 --- a/go.sum +++ b/go.sum @@ -49,7 +49,7 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240905143811-b2ab5b7c9c14 h1:9uqKGeUuok/9Q5B5DzDM+bVgyEZVruzaflXw8WiaZ+Y= github.com/OpenNebula/one/src/oca/go/src/goca v0.0.0-20240905143811-b2ab5b7c9c14/go.mod h1:dvAwZi1Aol7eu6BENzHtl8ztGBkacB9t/fJj+fYk+Xg= github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= @@ -65,6 +65,7 @@ github.com/aliyun/alibaba-cloud-sdk-go v1.63.15 h1:r2uwBUQhLhcPzaWz9tRJqc8MjYwHb github.com/aliyun/alibaba-cloud-sdk-go v1.63.15/go.mod h1:SOSDHfe1kX91v3W5QiBsWSLqeLxImobbMX1mxrFHsVQ= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go-v2 v1.30.5 h1:mWSRTwQAb0aLE17dSzztCVJWI9+cRMgqebndjwDyK0g= @@ -100,15 +101,9 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= -github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= -github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -126,6 +121,7 @@ github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= @@ -152,14 +148,13 @@ github.com/fullstorydev/grpcurl v1.8.7 h1:xJWosq3BQovQ4QrdPO72OrPiWuGgEsxY8ldYsJ github.com/fullstorydev/grpcurl v1.8.7/go.mod h1:pVtM4qe3CMoLaIzYS8uvTuDj2jVYmXqMUkZeijnXp/E= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -171,8 +166,10 @@ github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= @@ -199,14 +196,10 @@ github.com/go-playground/validator/v10 v10.10.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4 github.com/go-resty/resty/v2 v2.14.0 h1:/rhkzsAqGQkozwfKS5aFAbb6TyKd3zyFRWcdRXLPCAU= github.com/go-resty/resty/v2 v2.14.0/go.mod h1:IW6mekUOsElt9C7oWr0XRt9BNSD6D5rr9mhk6NjmNHg= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= -github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= -github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= @@ -232,12 +225,10 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= -github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 h1:0VpGH+cDhbDtdcweoyCVsF3fhN8kejK6rFe/2FFX2nU= -github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49/go.mod h1:BkkQ4L1KS1xMt2aWSPStnn55ChGC0DPOn2FQYj+f25M= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -248,7 +239,6 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -259,7 +249,6 @@ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k= github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= @@ -272,9 +261,11 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gT github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= github.com/gophercloud/gophercloud v1.14.0 h1:Bt9zQDhPrbd4qX7EILGmy+i7GP35cc+AAL2+wIJpUE8= github.com/gophercloud/gophercloud v1.14.0/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -291,7 +282,6 @@ github.com/hetznercloud/hcloud-go/v2 v2.13.1 h1:jq0GP4QaYE5d8xR/Zw17s9qoaESRJMXf github.com/hetznercloud/hcloud-go/v2 v2.13.1/go.mod h1:dhix40Br3fDiBhwaSG/zgaYOFFddpfBm/6R1Zz0IiF0= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20240312041847-bd984b5ce465/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= @@ -321,18 +311,14 @@ github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJ github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubeovn/kube-ovn v1.12.26 h1:pp4PfEIee9DZ/3vGQqIR9xVWucbqF9HHn0DRzkyEMLs= -github.com/kubeovn/kube-ovn v1.12.26/go.mod h1:L0PL79dsqsnA65Z1rjTFwQMap59yc3e7kIqGO53HAlA= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/linode/linodego v1.40.0 h1:7ESY0PwK94hoggoCtIroT1Xk6b1flrFBNZ6KwqbTqlI= @@ -351,6 +337,7 @@ github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa1 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= @@ -363,6 +350,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= @@ -373,62 +361,25 @@ github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= -github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= -github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= -github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= -github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= -github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo= -github.com/onsi/ginkgo/v2 v2.8.1/go.mod h1:N1/NbDngAFcSLdyZ+/aYTYGSlq9qMCS/cNKGJjy+csc= -github.com/onsi/ginkgo/v2 v2.9.0/go.mod h1:4xkjoL/tZv4SMWeww56BU5kAt19mVB47gTWxmrTcxyk= -github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= -github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= -github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= -github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= -github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= -github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= -github.com/onsi/ginkgo/v2 v2.17.2/go.mod h1:nP2DPOQoNsQmsVyv5rDA8JkXQoCs6goXIvr/PRJ1eCc= -github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= -github.com/onsi/ginkgo/v2 v2.20.0/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= github.com/onsi/ginkgo/v2 v2.20.1 h1:YlVIbqct+ZmnEph770q9Q7NVAz4wwIiVNahee6JyUzo= github.com/onsi/ginkgo/v2 v2.20.1/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= -github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= -github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= -github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= -github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= -github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= -github.com/onsi/gomega v1.27.1/go.mod h1:aHX5xOykVYzWOV4WqQy0sy8BQptgukenXpCXfadcIAw= -github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw= -github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= -github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= -github.com/onsi/gomega v1.27.7/go.mod h1:1p8OOlwo2iUUDsHnOrjE5UKYJ+e3W8eQ3qSlRahPmr4= -github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= -github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= -github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= -github.com/onsi/gomega v1.33.0/go.mod h1:+925n5YtiFsLzzafLUHzVMBpvvRAzrydIBiSIxjX3wY= -github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= -github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= -github.com/ovn-org/libovsdb v0.7.0 h1:owk3MHhaJ0gs0dWvTBtj7lPGEzbcyPrDYerEFqPXO/Y= -github.com/ovn-org/libovsdb v0.7.0/go.mod h1:dJbxEaalQl83nn904K32FaMjlH/qOObZ0bj4ejQ78AI= github.com/packethost/packngo v0.31.0 h1:LLH90ardhULWbagBIc3I3nl2uU75io0a7AwY6hyi0S4= github.com/packethost/packngo v0.31.0/go.mod h1:Io6VJqzkiqmIEQbpOjeIw9v8q9PfcTEq8TEY/tMQsfw= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= @@ -454,7 +405,6 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= @@ -472,7 +422,6 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -481,7 +430,6 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tinkerbell/tink v0.10.1 h1:mxdPQf7n4nB/AVdjbqCm5c98vsITU35g7Yw5cdOWmCw= @@ -500,6 +448,7 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -519,8 +468,8 @@ go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= -go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE= -go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg= +go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= +go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -540,21 +489,12 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -563,7 +503,6 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= @@ -578,19 +517,11 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= -golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= -golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.20.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -604,6 +535,7 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -611,29 +543,13 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -647,9 +563,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= @@ -668,84 +582,51 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= -golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= -golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= -golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= @@ -763,25 +644,13 @@ golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= -golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= -golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= -golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= -golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= -golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= -golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= -golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= -golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -828,9 +697,6 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= @@ -849,6 +715,7 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -878,22 +745,22 @@ k8s.io/cloud-provider v0.31.1/go.mod h1:xAdkE7fdZdu9rKLuOZUMBfagu7bM+bas3iPux/2n k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= -k8s.io/gengo/v2 v2.0.0-20240812201722-3b05ca7b6e59/go.mod h1:VH3AT8AaQOqiGjMF9p0/IM1Dj+82ZwjfxUP1IxaHE+8= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f h1:bnWtxXWdAl5bVOCEPoNdvMkyj6cTW3zxHuwKIakuV9w= -k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f/go.mod h1:G0W3eI9gG219NHRq3h5uQaRBl4pj4ZpwzRP5ti8y770= +k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 h1:1dWzkmJrrprYvjGwh9kEUxmcUV/CtNU8QM7h1FLWQOo= +k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA= k8s.io/kubectl v0.31.1 h1:ih4JQJHxsEggFqDJEHSOdJ69ZxZftgeZvYo7M/cpp24= k8s.io/kubectl v0.31.1/go.mod h1:aNuQoR43W6MLAtXQ/Bu4GDmoHlbhHKuyD49lmTC8eJM= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v1.3.1 h1:MoTNo/zvDlZ44c2ocXLPln8XTaQOeUodiYbEKrTCqv4= @@ -908,6 +775,7 @@ sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hw sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 11e890a88..c08eb1891 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -19,10 +19,8 @@ package kubevirt import ( "context" "encoding/base64" - "encoding/json" "errors" "fmt" - "net" "net/url" "os" "strconv" @@ -37,8 +35,6 @@ import ( clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/providernetworks" - "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/providernetworks/kubeovn" kubevirttypes "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" controllerutil "k8c.io/machine-controller/pkg/controller/util" @@ -120,8 +116,6 @@ type Config struct { ProviderNetworkName string SubnetName string - SubnetCIDRBlock string - SubnetGatewayIP string } // StorageTarget represents targeted storage definition that will be used to provision VirtualMachine volumes. Currently, @@ -353,8 +347,9 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } if rawConfig.VirtualMachine.ProviderNetwork != nil { - if err := validateProviderNetwork(&config, rawConfig.VirtualMachine.ProviderNetwork); err != nil { - return nil, nil, fmt.Errorf(`failed to validate "providerNetwork": %w`, err) + config.ProviderNetworkName = rawConfig.VirtualMachine.ProviderNetwork.Name + if rawConfig.VirtualMachine.ProviderNetwork.VPC.Subnet != nil { + config.SubnetName = rawConfig.VirtualMachine.ProviderNetwork.VPC.Subnet.Name } } @@ -703,10 +698,8 @@ func (p *provider) newVirtualMachine(c *Config, pc *providerconfigtypes.Config, annotations["kubevirt.io/allow-pod-bridge-network-live-migration"] = "true" - if strings.ToLower(c.ProviderNetworkName) == string(providernetworks.KubeOVN) { - if err := setOVNAnnotations(c, annotations); err != nil { - return nil, fmt.Errorf("failed to set OVN annotations: %w", err) - } + if err := setOVNAnnotations(c, annotations); err != nil { + return nil, fmt.Errorf("failed to set OVN annotations: %w", err) } for k, v := range machine.Annotations { @@ -1065,72 +1058,9 @@ func getStorageTopologies(ctx context.Context, storageClasName string, c *Config func setOVNAnnotations(c *Config, annotations map[string]string) error { annotations["ovn.kubernetes.io/allow_live_migration"] = "true" - if c.SubnetName != "" { annotations["ovn.kubernetes.io/logical_switch"] = c.SubnetName } - var subnetGatewayIP string - if c.SubnetGatewayIP == "" { - _, ipNet, err := net.ParseCIDR(c.SubnetCIDRBlock) - if err != nil { - return err - } - - firstIP := ipNet.IP.To4() - if firstIP == nil { - return errors.New("invalid IPv4 address") - } - - firstIP[3]++ - subnetGatewayIP = firstIP.String() - } else { - subnetGatewayIP = c.SubnetGatewayIP - } - - routes := []struct { - Gw string `json:"gw"` - }{ - { - Gw: subnetGatewayIP, - }, - } - marshalledRoutes, err := json.Marshal(routes) - if err != nil { - return err - } - - annotations["ovn.kubernetes.io/routes"] = string(marshalledRoutes) - - return nil -} - -func validateProviderNetwork(config *Config, providerNetwork *kubevirttypes.ProviderNetwork) error { - config.ProviderNetworkName = providerNetwork.Name - if providerNetwork.VPC.Subnet != nil { - config.SubnetName = providerNetwork.VPC.Subnet.Name - kvClient, err := client.New(config.RestConfig, client.Options{}) - if err != nil { - return fmt.Errorf("failed to create kubevirt client: %w", err) - } - - providerNetworks, err := kubeovn.New(kvClient) - if err != nil { - return fmt.Errorf("failed to create kubeovn providerNetworks: %w", err) - } - - config.SubnetCIDRBlock = providerNetwork.VPC.Subnet.CIDRBlock - if config.SubnetCIDRBlock == "" { - subnet, err := providerNetworks.GetVPCSubnet(context.Background(), config.SubnetName) - if err != nil { - return fmt.Errorf("failed to get vpcSubnet: %w", err) - } - - config.SubnetCIDRBlock = subnet.CIDRBlock - } - - config.SubnetGatewayIP = providerNetwork.VPC.Subnet.GatewayIP - } - return nil } diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index 8695e21f0..991d5f13a 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -109,8 +109,7 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { "vpc": { "name": "test-vpc", "subnet": { - "name": "test-subnet", - "cidrBlock": "10.10.0.0/16" + "name": "test-subnet" } } }, @@ -220,8 +219,7 @@ func TestNewVirtualMachine(t *testing.T) { name: "kubeovn-provider-network", specConf: kubevirtProviderSpecConf{ ProviderNetwork: &types.ProviderNetwork{Name: "KubeOVN", VPC: types.VPC{Name: "test-vpc", Subnet: &types.Subnet{ - Name: "test-subnet", - CIDRBlock: "10.10.0.0/24", + Name: "test-subnet", }}}}, }, { diff --git a/pkg/cloudprovider/provider/kubevirt/providernetworks/kubeovn/provider.go b/pkg/cloudprovider/provider/kubevirt/providernetworks/kubeovn/provider.go deleted file mode 100644 index cac26d9b5..000000000 --- a/pkg/cloudprovider/provider/kubevirt/providernetworks/kubeovn/provider.go +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2024 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package kubeovn - -import ( - "context" - "fmt" - - kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" - - "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/providernetworks" - - "k8s.io/apimachinery/pkg/types" - ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" -) - -type kubeOVNProviderNetwork struct { - client ctrlruntimeclient.Client -} - -func New(client ctrlruntimeclient.Client) (providernetworks.ProviderNetwork, error) { - return &kubeOVNProviderNetwork{client: client}, nil -} - -func (k *kubeOVNProviderNetwork) GetVPC(ctx context.Context, vpcName string) (*providernetworks.VPC, error) { - vpc := &kubeovnv1.Vpc{} - if err := k.client.Get(ctx, types.NamespacedName{Name: vpcName}, vpc); err != nil { - return nil, fmt.Errorf("failed to get VPC %s: %w", vpcName, err) - } - - return &providernetworks.VPC{ - Name: vpc.Name, - }, nil -} - -func (k *kubeOVNProviderNetwork) GetVPCSubnet(ctx context.Context, subnetName string) (*providernetworks.Subnet, error) { - vpcSubnet := &kubeovnv1.Subnet{} - if err := k.client.Get(ctx, types.NamespacedName{Name: subnetName}, vpcSubnet); err != nil { - return nil, fmt.Errorf("failed to get VPC subnet %s: %w", subnetName, err) - } - - return &providernetworks.Subnet{ - Name: vpcSubnet.Name, - CIDRBlock: vpcSubnet.Spec.CIDRBlock, - }, nil -} diff --git a/pkg/cloudprovider/provider/kubevirt/providernetworks/provider_networks.go b/pkg/cloudprovider/provider/kubevirt/providernetworks/provider_networks.go deleted file mode 100644 index abb724913..000000000 --- a/pkg/cloudprovider/provider/kubevirt/providernetworks/provider_networks.go +++ /dev/null @@ -1,48 +0,0 @@ -/* -Copyright 2024 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package providernetworks - -import ( - "context" -) - -type SupportedProviderNetworks string - -const ( - KubeOVN SupportedProviderNetworks = "kubeovn" -) - -// ProviderNetwork describes the infra cluster network fabric that is being used. These fabrics could be a as simple cni -// specific features up to full-blown networking components such as VPCs and Subnets. -type ProviderNetwork interface { - GetVPC(ctx context.Context, vpcName string) (*VPC, error) - GetVPCSubnet(ctx context.Context, subnetName string) (*Subnet, error) -} - -// VPC is a virtual network dedicated to a single tenant within a KubeVirt, where the resources in the VPC -// is isolated from any other resources within the KubeVirt infra cluster. -type VPC struct { - Name string `json:"name"` -} - -// Subnet a smaller, segmented portion of a larger network, like a Virtual Private Cloud (VPC). -type Subnet struct { - Name string `json:"name"` - CIDRBlock string `json:"cidrBlock"` - GatewayIP string `json:"gatewayIP,omitempty"` - ExcludeIPs []string `json:"excludeIP,omitempty"` -} diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml index 8ff6120fe..48dbb7d0c 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml @@ -30,6 +30,7 @@ spec: metadata: annotations: "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml index b694f8230..2460731de 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml @@ -30,6 +30,7 @@ spec: metadata: annotations: "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml index 0f70140c0..32d50db24 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml @@ -31,6 +31,7 @@ spec: creationTimestamp: null annotations: "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml index 548f617e1..289549edb 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml @@ -30,6 +30,7 @@ spec: creationTimestamp: null annotations: "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + "ovn.kubernetes.io/allow_live_migration": "true" labels: kubevirt.io/vm: http-image-source cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml index de113ac3c..6df4ec46f 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml @@ -36,6 +36,7 @@ spec: metadata: annotations: "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml index 7aa94bfb7..12f591b51 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml @@ -36,6 +36,7 @@ spec: creationTimestamp: null annotations: "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/kubeovn-provider-network.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/kubeovn-provider-network.yaml index 520d074b0..9f0f8948a 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/kubeovn-provider-network.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/kubeovn-provider-network.yaml @@ -31,7 +31,6 @@ spec: annotations: ovn.kubernetes.io/allow_live_migration: "true" ovn.kubernetes.io/logical_switch: test-subnet - ovn.kubernetes.io/routes: '[{"gw":"10.10.0.1"}]' "kubevirt.io/allow-pod-bridge-network-live-migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml index 3aa314504..816517740 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml @@ -30,6 +30,7 @@ spec: creationTimestamp: null annotations: "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml index 713dbeddd..07159c1a1 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml @@ -31,6 +31,7 @@ spec: creationTimestamp: null annotations: "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + "ovn.kubernetes.io/allow_live_migration": "true" labels: kubevirt.io/vm: pvc-image-source cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml index 17bc96142..5e197b77f 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml @@ -31,6 +31,7 @@ spec: creationTimestamp: null annotations: "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + "ovn.kubernetes.io/allow_live_migration": "true" labels: kubevirt.io/vm: registry-image-source-pod cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml index 4a950ead1..7ea3bdfd5 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml @@ -31,6 +31,7 @@ spec: creationTimestamp: null annotations: "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + "ovn.kubernetes.io/allow_live_migration": "true" labels: kubevirt.io/vm: registry-image-source cluster.x-k8s.io/cluster-name: cluster-name diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml index 2ea6e2703..2976b746c 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml @@ -56,6 +56,7 @@ spec: creationTimestamp: null annotations: "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml index 670150045..71828981e 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml @@ -30,6 +30,7 @@ spec: creationTimestamp: null annotations: "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml index 33fbae826..813faf8f6 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml @@ -31,6 +31,7 @@ spec: creationTimestamp: null annotations: "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + "ovn.kubernetes.io/allow_live_migration": "true" labels: cluster.x-k8s.io/cluster-name: cluster-name cluster.x-k8s.io/role: worker diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index e1e26c787..27641b130 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -148,9 +148,7 @@ type VPC struct { // Subnet a smaller, segmented portion of a larger network, like a Virtual Private Cloud (VPC). type Subnet struct { - Name string `json:"name"` - CIDRBlock string `json:"cidrBlock"` - GatewayIP string `json:"gatewayIP,omitempty"` + Name string `json:"name"` } func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { From 713b23c97a6c7e921d7d6c5e4083f2b83cb2bdb6 Mon Sep 17 00:00:00 2001 From: Mohamed Rafraf <81432497+mohamed-rafraf@users.noreply.github.com> Date: Fri, 11 Oct 2024 07:59:03 +0200 Subject: [PATCH 433/489] Fix: Configure Cloud-Init Script to execute automatically (#1866) * fix cloud-init in baremetal * lint :) --- .../plugins/tinkerbell/client/template.go | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go index 924c1a38c..bc5db5deb 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go @@ -128,7 +128,7 @@ func getTemplate(osImageURL string) (string, error) { createStreamUbuntuImageAction(hardwareDisk1, osImageURL), createGrowPartitionAction(hardwareDisk1), createNetworkConfigAction(), - createCloudInitConfigAction(), + configureCloudInitAction(), decodeCloudInitFile(hardwareName), createRebootAction(), } @@ -237,20 +237,24 @@ network: } } -func createCloudInitConfigAction() Action { +func configureCloudInitAction() Action { + commands := `mkdir -p /var/lib/cloud/seed/nocloud && chmod 755 /var/lib/cloud/seed/nocloud +echo 'datasource_list: [ NoCloud ]' > /etc/cloud/cloud.cfg.d/01_ds-identify.cfg +echo '{{.cloud_init_script}}' > /tmp/{{.hardware_name}}-bootstrap-config +echo 'instance-id: {{.hardware_name}}' > /var/lib/cloud/seed/nocloud/meta-data +echo 'local-hostname: {{.hardware_name}}' >> /var/lib/cloud/seed/nocloud/meta-data +` + return Action{ - Name: "add-cloud-init-config", - Image: "quay.io/tinkerbell-actions/writefile:v1.0.0", + Name: "configure-cloud-init", + Image: "quay.io/tinkerbell-actions/cexec:v1.0.0", Timeout: 90, Environment: map[string]string{ - "DEST_DISK": "{{ index .Hardware.Disks 0 }}3", - "FS_TYPE": fsType, - "DEST_PATH": fmt.Sprintf("/tmp/%s-bootstrap-config", hardwareName), - "CONTENTS": "{{.cloud_init_script}}", - "UID": "0", - "GID": "0", - "MODE": "0644", - "DIRMODE": "0755", + "BLOCK_DEVICE": "{{ index .Hardware.Disks 0 }}3", + "FS_TYPE": fsType, + "CHROOT": "y", + "DEFAULT_INTERPRETER": defaultInterpreter, + "CMD_LINE": commands, }, } } @@ -265,7 +269,7 @@ func decodeCloudInitFile(hardwareName string) Action { "FS_TYPE": fsType, "CHROOT": "y", "DEFAULT_INTERPRETER": "/bin/sh -c", - "CMD_LINE": fmt.Sprintf("cat /tmp/%s-bootstrap-config | base64 -d > '/etc/cloud/cloud.cfg.d/%s-cloud-init.cfg'", hardwareName, hardwareName), + "CMD_LINE": fmt.Sprintf("cat /tmp/%s-bootstrap-config | base64 -d > ''/var/lib/cloud/seed/nocloud/user-data'", hardwareName), }, } } From 4465175f4891bb3866663b6a4b302bf40c503d79 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 11 Oct 2024 16:27:03 +0500 Subject: [PATCH 434/489] Upgrade to Go 1.23.2 (#1867) Signed-off-by: Waleed Malik --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 12 ++++++------ .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 10 +++++----- .prow/verify.yaml | 16 ++++++++-------- Dockerfile | 2 +- Makefile | 2 +- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- 22 files changed, 45 insertions(+), 45 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index d4cd4d85c..d958d6b2b 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -35,7 +35,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -65,7 +65,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -95,7 +95,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -123,7 +123,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 744398c45..6f772d882 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -27,7 +27,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - /bin/bash - -c @@ -56,7 +56,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index 19956b730..4139c0186 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 7bedcd160..6f02370e7 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -32,7 +32,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index b2c27d6bd..d30e4de2d 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -96,7 +96,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -130,7 +130,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -162,7 +162,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -194,7 +194,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 91d22f336..0c7d66053 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -96,7 +96,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 4f11b59f1..9a135b35b 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index a7f6b3657..d14dd0120 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 6921900b6..ce0ea7650 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 51db5d389..0b4f55dc6 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index fa2dd7e6b..5fc091859 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index c2485844a..f9eba462b 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index cfc486b64..15b9f2f66 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 47808b1f7..97102b6cd 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -64,7 +64,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 49b8bbd50..960019489 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index b43201c06..216993262 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -32,7 +32,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index f1217a51a..ee71eac20 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -95,7 +95,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -128,7 +128,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -161,7 +161,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 7afdc9774..04c9cfcbb 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -22,7 +22,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-4 command: - make args: @@ -44,7 +44,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-4 command: - make args: @@ -66,7 +66,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-4 command: - make args: @@ -87,7 +87,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-4 command: - make args: @@ -107,7 +107,7 @@ presubmits: path_alias: k8c.io/machine-controller spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-4 command: - "/usr/local/bin/shfmt" args: @@ -136,7 +136,7 @@ presubmits: path_alias: k8c.io/machine-controller spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-4 command: - "./hack/verify-boilerplate.sh" resources: @@ -156,7 +156,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-4 command: - ./hack/verify-licenses.sh resources: @@ -173,7 +173,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-3 + - image: quay.io/kubermatic/build:go-1.23-node-20-4 command: - make args: diff --git a/Dockerfile b/Dockerfile index 04bae0024..0a1bdd85e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.23.1 +ARG GO_VERSION=1.23.2 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/k8c.io/machine-controller COPY . . diff --git a/Makefile b/Makefile index be85e3dff..098c867be 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.23.1 +GO_VERSION ?= 1.23.2 GOOS ?= $(shell go env GOOS) diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index bc079af4c..b0fa24681 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.23-node-20-3 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.23-node-20-4 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index c32c40110..778ba0bd8 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.23-node-20-3 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.23-node-20-4 containerize ./hack/verify-licenses.sh go mod vendor From 95ee8c03ff733f3d0392ade70173cdd615e9036d Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 11 Oct 2024 16:51:04 +0500 Subject: [PATCH 435/489] Update k8s versions to latest patch (#1868) Signed-off-by: Waleed Malik --- examples/alibaba-machinedeployment.yaml | 2 +- examples/anexia-machinedeployment.yaml | 4 ++-- examples/aws-machinedeployment.yaml | 2 +- examples/azure-machinedeployment.yaml | 2 +- examples/digitalocean-machinedeployment.yaml | 2 +- examples/equinixmetal-machinedeployment.yaml | 2 +- examples/gce-machinedeployment.yaml | 2 +- examples/hetzner-machinedeployment.yaml | 2 +- examples/kubevirt-machinedeployment.yaml | 2 +- examples/linode-machinedeployment.yaml | 2 +- examples/nutanix-machinedeployment.yaml | 2 +- examples/opennebula-machinedeployment.yaml | 2 +- examples/openstack-machinedeployment.yaml | 2 +- examples/scaleway-machinedeployment.yaml | 2 +- ...ware-cloud-director-machinedeployment.yaml | 2 +- ...e-datastore-cluster-machinedeployment.yaml | 2 +- examples/vsphere-machinedeployment.yaml | 2 +- examples/vultr-machinedeployment.yaml | 2 +- test/e2e/provisioning/all_e2e_test.go | 20 +++++++++---------- test/e2e/provisioning/helper.go | 8 ++++---- ...hinedeployment-azure-redhat-satellite.yaml | 2 +- 21 files changed, 34 insertions(+), 34 deletions(-) diff --git a/examples/alibaba-machinedeployment.yaml b/examples/alibaba-machinedeployment.yaml index 0b00f70d6..6fb621c98 100644 --- a/examples/alibaba-machinedeployment.yaml +++ b/examples/alibaba-machinedeployment.yaml @@ -62,4 +62,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index 0b875b4d8..da31dd1d7 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -70,7 +70,7 @@ spec: # least one Prefix, because we have to know (and thus, reserve) # at least one IP address for each Machine. prefixes: - - "<< ANEXIA_PREFIX_ID >>" + - "<< ANEXIA_PREFIX_ID >>" # You may have this old disk config attribute in your config - please migrate to the disks attribute. # For now it is still recognized though. @@ -83,4 +83,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/aws-machinedeployment.yaml b/examples/aws-machinedeployment.yaml index 4881ad415..d111744ed 100644 --- a/examples/aws-machinedeployment.yaml +++ b/examples/aws-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/azure-machinedeployment.yaml b/examples/azure-machinedeployment.yaml index b2d22d9e2..11d40775a 100644 --- a/examples/azure-machinedeployment.yaml +++ b/examples/azure-machinedeployment.yaml @@ -91,4 +91,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/digitalocean-machinedeployment.yaml b/examples/digitalocean-machinedeployment.yaml index a0e5646a3..043787ebd 100644 --- a/examples/digitalocean-machinedeployment.yaml +++ b/examples/digitalocean-machinedeployment.yaml @@ -57,4 +57,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/equinixmetal-machinedeployment.yaml b/examples/equinixmetal-machinedeployment.yaml index 9246f24ec..66f519943 100644 --- a/examples/equinixmetal-machinedeployment.yaml +++ b/examples/equinixmetal-machinedeployment.yaml @@ -51,4 +51,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index eaffe350b..fa8a06404 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -90,4 +90,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/hetzner-machinedeployment.yaml b/examples/hetzner-machinedeployment.yaml index 7ef07fe16..8f3c80bf7 100644 --- a/examples/hetzner-machinedeployment.yaml +++ b/examples/hetzner-machinedeployment.yaml @@ -67,4 +67,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index 4436edf8a..96bddfe6a 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -69,4 +69,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/linode-machinedeployment.yaml b/examples/linode-machinedeployment.yaml index 52b5ab7ef..4df4da709 100644 --- a/examples/linode-machinedeployment.yaml +++ b/examples/linode-machinedeployment.yaml @@ -53,4 +53,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/nutanix-machinedeployment.yaml b/examples/nutanix-machinedeployment.yaml index 41482e82d..06c6235db 100644 --- a/examples/nutanix-machinedeployment.yaml +++ b/examples/nutanix-machinedeployment.yaml @@ -82,4 +82,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/opennebula-machinedeployment.yaml b/examples/opennebula-machinedeployment.yaml index 1b20c86c6..e0a48b9bd 100644 --- a/examples/opennebula-machinedeployment.yaml +++ b/examples/opennebula-machinedeployment.yaml @@ -67,4 +67,4 @@ spec: # use cloud-init for flatcar as ignition doesn't know anything about OpenNebula yet provisioningUtility: "cloud-init" versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index 2f2becfd4..ffc93c2f8 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -166,4 +166,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/scaleway-machinedeployment.yaml b/examples/scaleway-machinedeployment.yaml index f21bc387a..9b7f7ca7c 100644 --- a/examples/scaleway-machinedeployment.yaml +++ b/examples/scaleway-machinedeployment.yaml @@ -61,4 +61,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/vmware-cloud-director-machinedeployment.yaml b/examples/vmware-cloud-director-machinedeployment.yaml index b326cb494..00a882308 100644 --- a/examples/vmware-cloud-director-machinedeployment.yaml +++ b/examples/vmware-cloud-director-machinedeployment.yaml @@ -89,4 +89,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml index d2590254f..4e42ac34c 100644 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ b/examples/vsphere-datastore-cluster-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index 2e3efd019..85930ca1f 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -81,4 +81,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/examples/vultr-machinedeployment.yaml b/examples/vultr-machinedeployment.yaml index 47d9dc51a..b1e2554b9 100644 --- a/examples/vultr-machinedeployment.yaml +++ b/examples/vultr-machinedeployment.yaml @@ -70,4 +70,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.30.4 + kubelet: 1.30.5 diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 0aa013ed6..e66b2d04c 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -85,7 +85,7 @@ const ( ) const ( - defaultKubernetesVersion = "1.29.8" + defaultKubernetesVersion = "1.29.9" awsDefaultKubernetesVersion = "1.26.12" defaultContainerRuntime = "containerd" ) @@ -348,7 +348,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.26. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.28.12", "1.29.8", "1.30.4", "1.31.0"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.28.14", "1.29.9", "1.30.5", "1.31.1"))) runScenarios(context.Background(), t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -424,7 +424,7 @@ func TestAWSProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := Not(VersionSelector("1.28.12", "1.29.8", "1.30.4", "1.31.0")) + selector := Not(VersionSelector("1.28.14", "1.29.9", "1.30.5", "1.31.1")) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -478,7 +478,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.28.12", "1.29.8", "1.30.4", "1.31.0"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.28.14", "1.29.9", "1.30.5", "1.31.1"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -500,7 +500,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.28.12", "1.29.8", "1.30.4", "1.31.0"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.28.14", "1.29.9", "1.30.5", "1.31.1"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -573,7 +573,7 @@ func TestAzureProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.30.4", "1.31.0"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.30.5", "1.31.1"))) // act params := []string{ @@ -602,7 +602,7 @@ func TestAzureCustomImageReferenceProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.4", "1.31.0"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.5", "1.31.1"))) // act params := []string{ fmt.Sprintf("<< AZURE_TENANT_ID >>=%s", azureTenantID), @@ -816,7 +816,7 @@ func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.4", "1.31.0"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.5", "1.31.1"))) params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) @@ -828,7 +828,7 @@ func TestVsphereMultipleNICProvisioningE2E(t *testing.T) { t.Parallel() // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.4", "1.31.0"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.5", "1.31.1"))) params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereMultipleNICManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) @@ -857,7 +857,7 @@ func TestVsphereDatastoreClusterProvisioningE2E(t *testing.T) { t.Parallel() // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(OsSelector("ubuntu", "rhel", "flatcar"), Not(VersionSelector("1.30.4", "1.31.0"))) + selector := And(OsSelector("ubuntu", "rhel", "flatcar"), Not(VersionSelector("1.30.5", "1.31.1"))) params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereDSCManifest, fmt.Sprintf("vs-dsc-%s", *testRunIdentifier)) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index ca42162e8..9f5de9aba 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -34,10 +34,10 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.28.12"), - semver.MustParse("v1.29.8"), - semver.MustParse("v1.30.4"), - semver.MustParse("v1.31.0"), + semver.MustParse("v1.28.14"), + semver.MustParse("v1.29.9"), + semver.MustParse("v1.30.5"), + semver.MustParse("v1.31.1"), } operatingSystems = []providerconfigtypes.OperatingSystem{ diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml index 6f67ec0f7..03c47442a 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml @@ -55,4 +55,4 @@ spec: rhelOrganizationName: "" rhelActivationKey: "" versions: - kubelet: 1.29.8 + kubelet: 1.29.9 From 46cfdfe3a668d596f99e65850e38991dada968a7 Mon Sep 17 00:00:00 2001 From: Mohamed Rafraf <81432497+mohamed-rafraf@users.noreply.github.com> Date: Fri, 11 Oct 2024 15:59:04 +0200 Subject: [PATCH 436/489] Fix typo in template.go (#1869) Signed-off-by: Mohamed Rafraf <81432497+mohamed-rafraf@users.noreply.github.com> --- .../provider/baremetal/plugins/tinkerbell/client/template.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go index bc5db5deb..c5baf8ad1 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go @@ -269,7 +269,7 @@ func decodeCloudInitFile(hardwareName string) Action { "FS_TYPE": fsType, "CHROOT": "y", "DEFAULT_INTERPRETER": "/bin/sh -c", - "CMD_LINE": fmt.Sprintf("cat /tmp/%s-bootstrap-config | base64 -d > ''/var/lib/cloud/seed/nocloud/user-data'", hardwareName), + "CMD_LINE": fmt.Sprintf("cat /tmp/%s-bootstrap-config | base64 -d > '/var/lib/cloud/seed/nocloud/user-data'", hardwareName), }, } } From abbb86c338554e8e377b95b4b5c235ea9450ec82 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Mon, 14 Oct 2024 11:43:09 +0200 Subject: [PATCH 437/489] support ubuntu nobel (#1858) --- pkg/cloudprovider/provider/aws/provider.go | 4 ++-- pkg/cloudprovider/provider/azure/provider.go | 4 ++-- pkg/cloudprovider/provider/digitalocean/provider.go | 2 +- pkg/cloudprovider/provider/equinixmetal/provider.go | 2 +- pkg/cloudprovider/provider/gce/config.go | 2 +- pkg/cloudprovider/provider/hetzner/provider.go | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 0da17a53b..2527e301a 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -145,13 +145,13 @@ var ( providerconfigtypes.OperatingSystemUbuntu: { awstypes.CPUArchitectureX86_64: { // Be as precise as possible - otherwise we might get a nightly dev build - description: "Canonical, Ubuntu, 22.04 LTS, amd64 jammy image build on ????-??-??", + description: "Canonical, Ubuntu, 24.04 LTS, amd64 noble image build on ????-??-??", // The AWS marketplace ID from Canonical owner: "099720109477", }, awstypes.CPUArchitectureARM64: { // Be as precise as possible - otherwise we might get a nightly dev build - description: "Canonical, Ubuntu, 22.04 LTS, arm64 jammy image build on ????-??-??", + description: "Canonical, Ubuntu, 24.04 LTS, arm64 noble image build on 2024-05-29 ????-??-??", // The AWS marketplace ID from Canonical owner: "099720109477", }, diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 1289c3aa2..dc6200a4b 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -149,8 +149,8 @@ var imageReferences = map[providerconfigtypes.OperatingSystem]compute.ImageRefer }, providerconfigtypes.OperatingSystemUbuntu: { Publisher: to.StringPtr("Canonical"), - Offer: to.StringPtr("0001-com-ubuntu-server-jammy"), - Sku: to.StringPtr("22_04-lts"), + Offer: to.StringPtr("ubuntu-24_04-lts"), + Sku: to.StringPtr("server-gen1"), Version: to.StringPtr("latest"), }, providerconfigtypes.OperatingSystemRHEL: { diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index af82ff6e3..6cb24fc70 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -85,7 +85,7 @@ func (t *TokenSource) Token() (*oauth2.Token, error) { func getSlugForOS(os providerconfigtypes.OperatingSystem) (string, error) { switch os { case providerconfigtypes.OperatingSystemUbuntu: - return "ubuntu-22-04-x64", nil + return "ubuntu-24-04-x64", nil case providerconfigtypes.OperatingSystemCentOS: return "centos-7-x64", nil case providerconfigtypes.OperatingSystemRockyLinux: diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index 445551d99..5e4946c30 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -466,7 +466,7 @@ func getDeviceByTag(client *packngo.Client, projectID, tag string) (*packngo.Dev func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { switch os { case providerconfigtypes.OperatingSystemUbuntu: - return "ubuntu_22_04", nil + return "ubuntu_24_04", nil case providerconfigtypes.OperatingSystemCentOS: return "centos_7", nil case providerconfigtypes.OperatingSystemFlatcar: diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index 739587d44..c3faaeb56 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -51,7 +51,7 @@ var imageProjects = map[providerconfigtypes.OperatingSystem]string{ // imageFamilies maps the OS to the Google Cloud image projects. var imageFamilies = map[providerconfigtypes.OperatingSystem]string{ - providerconfigtypes.OperatingSystemUbuntu: "ubuntu-2204-lts", + providerconfigtypes.OperatingSystemUbuntu: "ubuntu-2404-lts-amd64", providerconfigtypes.OperatingSystemFlatcar: "flatcar-stable", } diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 8b613b007..3f353f87b 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -72,7 +72,7 @@ type Config struct { func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { switch os { case providerconfigtypes.OperatingSystemUbuntu: - return "ubuntu-22.04", nil + return "ubuntu-24.04", nil case providerconfigtypes.OperatingSystemCentOS: return "centos-7", nil case providerconfigtypes.OperatingSystemRockyLinux: From 22c1174dadda99c68ec401a48d91da15b35acce0 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Fri, 18 Oct 2024 13:33:17 +0500 Subject: [PATCH 438/489] Drop support for Kubernetes v1.28 (#1870) Signed-off-by: Waleed Malik --- README.md | 1 - test/e2e/provisioning/all_e2e_test.go | 8 ++++---- test/e2e/provisioning/helper.go | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b835c768d..c46afab47 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,6 @@ Currently supported K8S versions are: - 1.31 - 1.30 - 1.29 -- 1.28 ### Community Providers diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index e66b2d04c..3cb7c613d 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -348,7 +348,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.26. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.28.14", "1.29.9", "1.30.5", "1.31.1"))) + selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.29.9", "1.30.5", "1.31.1"))) runScenarios(context.Background(), t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -424,7 +424,7 @@ func TestAWSProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := Not(VersionSelector("1.28.14", "1.29.9", "1.30.5", "1.31.1")) + selector := Not(VersionSelector("1.29.9", "1.30.5", "1.31.1")) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -478,7 +478,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.28.14", "1.29.9", "1.30.5", "1.31.1"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.29.9", "1.30.5", "1.31.1"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -500,7 +500,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.28.14", "1.29.9", "1.30.5", "1.31.1"))) + selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.29.9", "1.30.5", "1.31.1"))) // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 9f5de9aba..f2f8d52a9 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -34,7 +34,6 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.28.14"), semver.MustParse("v1.29.9"), semver.MustParse("v1.30.5"), semver.MustParse("v1.31.1"), From fd8ed7696cf8e1512d3c7a5c095f04f5f400738e Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 22 Oct 2024 08:54:22 +0200 Subject: [PATCH 439/489] Refactor kubervirt provider storage (#1872) * refactor kubervirt provider storage * refactor kubervirt provider storage * fix failing tests --- .../provider/kubevirt/provider.go | 111 +++++++++++++----- .../provider/kubevirt/provider_test.go | 3 + .../kubevirt/testdata/affinity-no-values.yaml | 4 +- .../provider/kubevirt/testdata/affinity.yaml | 4 +- .../kubevirt/testdata/custom-local-disk.yaml | 4 +- .../kubevirt/testdata/http-image-source.yaml | 4 +- .../instancetype-preference-custom.yaml | 4 +- .../instancetype-preference-standard.yaml | 4 +- .../testdata/kubeovn-provider-network.yaml | 6 +- .../kubevirt/testdata/nominal-case.yaml | 4 +- .../kubevirt/testdata/pvc-image-source.yaml | 4 +- .../testdata/registry-image-source-pod.yaml | 4 +- .../testdata/registry-image-source.yaml | 4 +- .../kubevirt/testdata/secondary-disks.yaml | 8 +- .../testdata/topologyspreadconstraints.yaml | 4 +- .../use-storage-as-storage-target.yaml | 2 +- 16 files changed, 113 insertions(+), 61 deletions(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index c08eb1891..723cbef5f 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -124,6 +124,7 @@ type StorageTarget string const ( Storage StorageTarget = "storage" + PVC StorageTarget = "pvc" ) type AffinityType string @@ -308,30 +309,14 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if rawConfig.VirtualMachine.DNSConfig != nil { config.DNSConfig = rawConfig.VirtualMachine.DNSConfig } - config.SecondaryDisks = make([]SecondaryDisks, 0, len(rawConfig.VirtualMachine.Template.SecondaryDisks)) - for i, sd := range rawConfig.VirtualMachine.Template.SecondaryDisks { - sdSizeString, err := p.configVarResolver.GetConfigVarStringValue(sd.Size) - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "secondaryDisks.size" field: %w`, err) - } - pvc, err := resource.ParseQuantity(sdSizeString) - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.size" field: %w`, err) - } - - scString, err := p.configVarResolver.GetConfigVarStringValue(sd.StorageClassName) - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.storageClass" field: %w`, err) - } - config.SecondaryDisks = append(config.SecondaryDisks, SecondaryDisks{ - Name: fmt.Sprintf("secondarydisk%d", i), - Size: pvc, - StorageClassName: scString, - StorageAccessType: p.getStorageAccessType(sd.StorageAccessType), - }) + infraClient, err := client.New(config.RestConfig, client.Options{}) + if err != nil { + return nil, nil, fmt.Errorf("failed to get kubevirt client: %w", err) + } + config.StorageAccessType, config.SecondaryDisks, err = p.configureStorage(infraClient, rawConfig.VirtualMachine.Template) + if err != nil { + return nil, nil, fmt.Errorf(`failed to configure storage: %w`, err) } - config.StorageAccessType = p.getStorageAccessType(rawConfig.VirtualMachine.Template.PrimaryDisk.StorageAccessType) - config.NodeAffinityPreset, err = p.parseNodeAffinityPreset(rawConfig.Affinity.NodeAffinityPreset) if err != nil { return nil, nil, fmt.Errorf(`failed to parse "nodeAffinityPreset" field: %w`, err) @@ -356,12 +341,29 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return &config, pconfig, nil } -func (p *provider) getStorageAccessType(accessType providerconfigtypes.ConfigVarString) corev1.PersistentVolumeAccessMode { +func (p *provider) getStorageAccessType(ctx context.Context, accessType providerconfigtypes.ConfigVarString, + infraClient client.Client, storageClassName string) (corev1.PersistentVolumeAccessMode, error) { at, _ := p.configVarResolver.GetConfigVarStringValue(accessType) if at == "" { - return corev1.ReadWriteOnce + sp := &cdiv1beta1.StorageProfile{} + if err := infraClient.Get(ctx, types.NamespacedName{Name: storageClassName}, sp); err != nil { + return "", fmt.Errorf(`failed to get cdi storageprofile: %w`, err) + } + + // choose RWO as a default access mode and if RWX is supported then choose it instead. + accessMode := corev1.ReadWriteOnce + for _, claimProperty := range sp.Status.ClaimPropertySets { + for _, am := range claimProperty.AccessModes { + if am == corev1.ReadWriteMany { + accessMode = corev1.ReadWriteMany + } + } + } + + return accessMode, nil } - return corev1.PersistentVolumeAccessMode(at) + + return corev1.PersistentVolumeAccessMode(at), nil } func (p *provider) parseNodeAffinityPreset(nodeAffinityPreset kubevirttypes.NodeAffinityPreset) (NodeAffinityPreset, error) { @@ -712,7 +714,13 @@ func (p *provider) newVirtualMachine(c *Config, pc *providerconfigtypes.Config, } defaultBridgeNetwork := defaultBridgeNetwork() - runStrategyOnce := kubevirtv1.RunStrategyOnce + runStrategy := kubevirtv1.RunStrategyOnce + // currently we only support KubeOvn as a ProviderNetwork and KubeOvn has the ability to pin the IP of the VM(static ip) + // even if the VMi was stopped or deleted thus we can have the VM always running and in the events of VM restarts the + // ip address of the VMI will not change. + if c.SubnetName != "" { + runStrategy = kubevirtv1.RunStrategyAlways + } virtualMachine := &kubevirtv1.VirtualMachine{ ObjectMeta: metav1.ObjectMeta{ @@ -721,7 +729,7 @@ func (p *provider) newVirtualMachine(c *Config, pc *providerconfigtypes.Config, Labels: labels, }, Spec: kubevirtv1.VirtualMachineSpec{ - RunStrategy: &runStrategyOnce, + RunStrategy: &runStrategy, Instancetype: c.Instancetype, Preference: c.Preference, Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ @@ -885,8 +893,8 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string, annotations m } switch config.StorageTarget { - case Storage: - dataVolumeTemplates[0].Spec.Storage = &cdiv1beta1.StorageSpec{ + case PVC: + dataVolumeTemplates[0].Spec.PVC = &corev1.PersistentVolumeClaimSpec{ StorageClassName: ptr.To(config.StorageClassName), AccessModes: []corev1.PersistentVolumeAccessMode{ config.StorageAccessType, @@ -896,7 +904,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string, annotations m }, } default: - dataVolumeTemplates[0].Spec.PVC = &corev1.PersistentVolumeClaimSpec{ + dataVolumeTemplates[0].Spec.Storage = &cdiv1beta1.StorageSpec{ StorageClassName: ptr.To(config.StorageClassName), AccessModes: []corev1.PersistentVolumeAccessMode{ config.StorageAccessType, @@ -1064,3 +1072,44 @@ func setOVNAnnotations(c *Config, annotations map[string]string) error { return nil } + +func (p *provider) configureStorage(infraClient client.Client, template kubevirttypes.Template) (corev1.PersistentVolumeAccessMode, []SecondaryDisks, error) { + secondaryDisks := make([]SecondaryDisks, 0, len(template.SecondaryDisks)) + for i, sd := range template.SecondaryDisks { + sdSizeString, err := p.configVarResolver.GetConfigVarStringValue(sd.Size) + if err != nil { + return "", nil, fmt.Errorf(`failed to parse "secondaryDisks.size" field: %w`, err) + } + pvc, err := resource.ParseQuantity(sdSizeString) + if err != nil { + return "", nil, fmt.Errorf(`failed to parse value of "secondaryDisks.size" field: %w`, err) + } + + scString, err := p.configVarResolver.GetConfigVarStringValue(sd.StorageClassName) + if err != nil { + return "", nil, fmt.Errorf(`failed to parse value of "secondaryDisks.storageClass" field: %w`, err) + } + storageAccessMode, err := p.getStorageAccessType(context.TODO(), sd.StorageAccessType, infraClient, scString) + if err != nil { + return "", nil, fmt.Errorf(`failed to get value of storageAccessMode: %w`, err) + } + secondaryDisks = append(secondaryDisks, SecondaryDisks{ + Name: fmt.Sprintf("secondarydisk%d", i), + Size: pvc, + StorageClassName: scString, + StorageAccessType: storageAccessMode, + }) + } + scString, err := p.configVarResolver.GetConfigVarStringValue(template.PrimaryDisk.StorageClassName) + if err != nil { + return "", nil, fmt.Errorf(`failed to parse value of "primaryDisk.storageClass" field: %w`, err) + } + + primaryDisk, err := p.getStorageAccessType(context.TODO(), template.PrimaryDisk.StorageAccessType, + infraClient, scString) + if err != nil { + return "", nil, fmt.Errorf(`failed to get value of primaryDiskstorageAccessType: %w`, err) + } + + return primaryDisk, secondaryDisks, nil +} diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index 991d5f13a..c58412147 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -132,11 +132,14 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { {{- if .SecondaryDisks }} "secondaryDisks": [{ "size": "20Gi", + "storageAccessType": "ReadWriteMany", "storageClassName": "longhorn2"},{ "size": "30Gi", + "storageAccessType": "ReadWriteMany", "storageClassName": "longhorn3"}], {{- end }} "primaryDisk": { + "storageAccessType": "ReadWriteMany", {{- if .StorageTarget }} "storageTarget": "{{ .StorageTarget }}", {{- end }} diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml index 48dbb7d0c..a8382d8ee 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml @@ -15,9 +15,9 @@ spec: creationTimestamp: null name: affinity-no-values spec: - pvc: + storage: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 10Gi diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml index 2460731de..7c0a3e9d8 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml @@ -15,9 +15,9 @@ spec: creationTimestamp: null name: affinity spec: - pvc: + storage: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 10Gi diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml index 32d50db24..b83d4b01b 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml @@ -14,9 +14,9 @@ spec: - metadata: name: custom-local-disk spec: - pvc: + storage: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 10Gi diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml index 289549edb..f36d5392b 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml @@ -14,9 +14,9 @@ spec: - metadata: name: http-image-source spec: - pvc: + storage: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 10Gi diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml index 6df4ec46f..9154f71c1 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml @@ -15,9 +15,9 @@ spec: creationTimestamp: null name: instancetype-preference-custom spec: - pvc: + storage: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 10Gi diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml index 12f591b51..c890481cb 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml @@ -14,9 +14,9 @@ spec: - metadata: name: instancetype-preference-standard spec: - pvc: + storage: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 10Gi diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/kubeovn-provider-network.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/kubeovn-provider-network.yaml index 9f0f8948a..aa8dc5b6f 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/kubeovn-provider-network.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/kubeovn-provider-network.yaml @@ -14,9 +14,9 @@ spec: - metadata: name: kubeovn-provider-network spec: - pvc: + storage: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 10Gi @@ -24,7 +24,7 @@ spec: source: http: url: http://x.y.z.t/ubuntu.img - runStrategy: Once + runStrategy: Always template: metadata: creationTimestamp: null diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml index 816517740..58f1ae771 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml @@ -14,9 +14,9 @@ spec: - metadata: name: nominal-case spec: - pvc: + storage: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 10Gi diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml index 07159c1a1..61884708e 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml @@ -14,9 +14,9 @@ spec: - metadata: name: pvc-image-source spec: - pvc: + storage: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 10Gi diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml index 5e197b77f..69aaaac70 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml @@ -14,9 +14,9 @@ spec: - metadata: name: registry-image-source-pod spec: - pvc: + storage: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 10Gi diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml index 7ea3bdfd5..f4608a12d 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml @@ -14,9 +14,9 @@ spec: - metadata: name: registry-image-source spec: - pvc: + storage: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 10Gi diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml index 2976b746c..8abda0e51 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml @@ -14,9 +14,9 @@ spec: - metadata: name: secondary-disks spec: - pvc: + storage: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 10Gi @@ -29,7 +29,7 @@ spec: spec: pvc: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 20Gi @@ -42,7 +42,7 @@ spec: spec: pvc: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 30Gi diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml index 71828981e..141091668 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml @@ -14,9 +14,9 @@ spec: - metadata: name: topologyspreadconstraints spec: - pvc: + storage: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 10Gi diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml index 813faf8f6..b441d5533 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml @@ -20,7 +20,7 @@ spec: url: "http://x.y.z.t/ubuntu.img" storage: accessModes: - - ReadWriteOnce + - ReadWriteMany resources: requests: storage: 10Gi From 36a68f1ae069a3584886b1b01d8ebfccfa47a897 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Tue, 5 Nov 2024 00:16:31 +0100 Subject: [PATCH 440/489] replace packethost/packngo with equinix SDK (#1871) --- go.mod | 2 +- go.sum | 7 +- .../provider/equinixmetal/provider.go | 249 ++++++++++-------- 3 files changed, 137 insertions(+), 121 deletions(-) diff --git a/go.mod b/go.mod index 0385bc162..fe6441475 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/aws/smithy-go v1.20.4 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc github.com/digitalocean/godo v1.124.0 + github.com/equinix/equinix-sdk-go v0.46.0 github.com/go-logr/logr v1.4.2 github.com/go-logr/zapr v1.3.0 github.com/go-test/deep v1.1.0 @@ -30,7 +31,6 @@ require ( github.com/hetznercloud/hcloud-go/v2 v2.13.1 github.com/linode/linodego v1.40.0 github.com/nutanix-cloud-native/prism-go-client v0.5.1 - github.com/packethost/packngo v0.31.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 5f7cef617..ee7d9e6c9 100644 --- a/go.sum +++ b/go.sum @@ -130,6 +130,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/equinix/equinix-sdk-go v0.46.0 h1:ldQo4GtXNr+0XsThQJf/pUdx5wcLFe9QpLFtAwonqH8= +github.com/equinix/equinix-sdk-go v0.46.0/go.mod h1:hEb3XLaedz7xhl/dpPIS6eOIiXNPeqNiVoyDrT6paIg= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -238,7 +240,6 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -349,7 +350,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -380,8 +380,6 @@ github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPf github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= -github.com/packethost/packngo v0.31.0 h1:LLH90ardhULWbagBIc3I3nl2uU75io0a7AwY6hyi0S4= -github.com/packethost/packngo v0.31.0/go.mod h1:Io6VJqzkiqmIEQbpOjeIw9v8q9PfcTEq8TEY/tMQsfw= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= @@ -484,7 +482,6 @@ go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index 5e4946c30..d7a271377 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -21,10 +21,11 @@ import ( "encoding/json" "errors" "fmt" - "reflect" + "net/http" + "slices" "strings" - "github.com/packethost/packngo" + "github.com/equinix/equinix-sdk-go/services/metalv1" "go.uber.org/zap" "k8c.io/machine-controller/pkg/apis/cluster/common" @@ -36,9 +37,10 @@ import ( "k8c.io/machine-controller/pkg/providerconfig" providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" ) const ( @@ -148,7 +150,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *e return &c, rawConfig, pconfig, err } -func (p *provider) getMetalDevice(machine *clusterv1alpha1.Machine) (*packngo.Device, *packngo.Client, error) { +func (p *provider) getMetalDevice(ctx context.Context, machine *clusterv1alpha1.Machine) (*metalv1.Device, *metalv1.APIClient, error) { c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, nil, cloudprovidererrors.TerminalError{ @@ -158,14 +160,14 @@ func (p *provider) getMetalDevice(machine *clusterv1alpha1.Machine) (*packngo.De } client := getClient(c.Token) - device, err := getDeviceByTag(client, c.ProjectID, generateTag(string(machine.UID))) + device, err := getDeviceByTag(ctx, client, c.ProjectID, generateTag(string(machine.UID))) if err != nil { return nil, nil, err } return device, client, nil } -func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { c, _, pc, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -194,49 +196,65 @@ func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec cluste if c.Facilities != nil && (len(c.Facilities) > 0 || c.Facilities[0] != "") { // get all valid facilities - facilities, _, err := client.Facilities.List(nil) + request := client.FacilitiesApi.FindFacilitiesByProject(ctx, c.ProjectID) + facilities, resp, err := client.FacilitiesApi.FindFacilitiesByProjectExecute(request) if err != nil { return fmt.Errorf("failed to list facilities: %w", err) } + resp.Body.Close() + + expectedFacilities := sets.New(c.Facilities...) + availableFacilities := sets.New[string]() + for _, facility := range facilities.Facilities { + availableFacilities.Insert(*facility.Code) + } + // ensure our requested facilities are in those facilities - if missingFacilities := itemsNotInList(facilityProp(facilities, "Code"), c.Facilities); len(missingFacilities) > 0 { - return fmt.Errorf("unknown facilities: %s", strings.Join(missingFacilities, ",")) + if diff := expectedFacilities.Difference(availableFacilities); diff.Len() > 0 { + return fmt.Errorf("unknown facilities: %v", sets.List(diff)) } } if c.Metro != "" { - metros, _, err := client.Metros.List(nil) + request := client.MetrosApi.FindMetros(ctx) + metros, resp, err := client.MetrosApi.FindMetrosExecute(request) if err != nil { return fmt.Errorf("failed to list metros: %w", err) } + resp.Body.Close() - var metroExists bool - for _, metro := range metros { - if strings.EqualFold(metro.Code, c.Metro) { - metroExists = true - } - } + metroExists := slices.ContainsFunc(metros.Metros, func(m metalv1.Metro) bool { + return strings.EqualFold(*m.Code, c.Metro) + }) if !metroExists { - return fmt.Errorf("unknown metro: %s", c.Metro) + return fmt.Errorf("unknown metro %q", c.Metro) } } // get all valid plans a.k.a. instance types - plans, _, err := client.Plans.List(nil) + request := client.PlansApi.FindPlansByProject(ctx, c.ProjectID) + plans, resp, err := client.PlansApi.FindPlansByProjectExecute(request) if err != nil { return fmt.Errorf("failed to list instance types / plans: %w", err) } + resp.Body.Close() + // ensure our requested plan is in those plans - validPlanNames := planProp(plans, "Name") - if missingPlans := itemsNotInList(validPlanNames, []string{c.InstanceType}); len(missingPlans) > 0 { - return fmt.Errorf("unknown instance type / plan: %s, acceptable plans: %s", strings.Join(missingPlans, ","), strings.Join(validPlanNames, ",")) + expectedPlans := sets.New(c.InstanceType) + availablePlans := sets.New[string]() + for _, plan := range plans.Plans { + availablePlans.Insert(*plan.Name) + } + + if diff := expectedPlans.Difference(availablePlans); diff.Len() > 0 { + return fmt.Errorf("unknown instance type / plan: %s, acceptable plans: %v", c.InstanceType, sets.List(availablePlans)) } return nil } -func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, _, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -246,6 +264,7 @@ func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, machine *clus } client := getClient(c.Token) + request := client.DevicesApi.CreateDevice(ctx, c.ProjectID) imageName, err := getNameForOS(pc.OperatingSystem) if err != nil { @@ -255,24 +274,43 @@ func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, machine *clus } } - serverCreateOpts := &packngo.DeviceCreateRequest{ - Hostname: machine.Spec.Name, - UserData: userdata, - ProjectID: c.ProjectID, - Facility: c.Facilities, - Metro: c.Metro, - BillingCycle: c.BillingCycle, - Plan: c.InstanceType, - OS: imageName, - Tags: []string{ - generateTag(string(machine.UID)), - }, - } + billingCycle := metalv1.DeviceCreateInputBillingCycle(c.BillingCycle) - device, res, err := client.Devices.Create(serverCreateOpts) + if c.Metro != "" { + request = request.CreateDeviceRequest(metalv1.CreateDeviceRequest{ + DeviceCreateInMetroInput: &metalv1.DeviceCreateInMetroInput{ + Hostname: &machine.Spec.Name, + Userdata: &userdata, + Metro: c.Metro, + BillingCycle: &billingCycle, + Plan: c.InstanceType, + OperatingSystem: imageName, + Tags: []string{ + generateTag(string(machine.UID)), + }, + }, + }) + } else { + request = request.CreateDeviceRequest(metalv1.CreateDeviceRequest{ + DeviceCreateInFacilityInput: &metalv1.DeviceCreateInFacilityInput{ + Hostname: &machine.Spec.Name, + Userdata: &userdata, + Facility: c.Facilities, + BillingCycle: &billingCycle, + Plan: c.InstanceType, + OperatingSystem: imageName, + Tags: []string{ + generateTag(string(machine.UID)), + }, + }, + }) + } + + device, resp, err := client.DevicesApi.CreateDeviceExecute(request) if err != nil { - return nil, metalErrorToTerminalError(err, res, "failed to create server") + return nil, metalErrorToTerminalError(err, resp, "failed to create server") } + resp.Body.Close() return &metalDevice{device: device}, nil } @@ -295,10 +333,13 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine } client := getClient(c.Token) - res, err := client.Devices.Delete(instance.(*metalDevice).device.ID, false) + request := client.DevicesApi.DeleteDevice(ctx, *instance.(*metalDevice).device.Id) + + resp, err := client.DevicesApi.DeleteDeviceExecute(request) if err != nil { - return false, metalErrorToTerminalError(err, res, "failed to delete the server") + return false, metalErrorToTerminalError(err, resp, "failed to delete the server") } + resp.Body.Close() return false, nil } @@ -316,8 +357,8 @@ func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.Machin return spec, nil } -func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - device, _, err := p.getMetalDevice(machine) +func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { + device, _, err := p.getMetalDevice(ctx, machine) if err != nil { return nil, err } @@ -328,8 +369,8 @@ func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *cluster return nil, cloudprovidererrors.ErrInstanceNotFound } -func (p *provider) MigrateUID(_ context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newID types.UID) error { - device, client, err := p.getMetalDevice(machine) +func (p *provider) MigrateUID(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, newID types.UID) error { + device, client, err := p.getMetalDevice(ctx, machine) if err != nil { return err } @@ -351,13 +392,19 @@ func (p *provider) MigrateUID(_ context.Context, log *zap.SugaredLogger, machine tags = append(tags, generateTag(string(newID))) log.Info("Setting UID label for machine") - dur := &packngo.DeviceUpdateRequest{ - Tags: &tags, - } - _, response, err := client.Devices.Update(device.ID, dur) + + dur := client.DevicesApi. + UpdateDevice(ctx, *device.Id). + DeviceUpdateInput(metalv1.DeviceUpdateInput{ + Tags: tags, + }) + + _, response, err := client.DevicesApi.UpdateDeviceExecute(dur) if err != nil { return metalErrorToTerminalError(err, response, "failed to update UID label") } + response.Body.Close() + log.Info("Successfully set UID label for machine") return nil @@ -380,43 +427,48 @@ func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { } type metalDevice struct { - device *packngo.Device + device *metalv1.Device } func (s *metalDevice) Name() string { - return s.device.Hostname + return *s.device.Hostname } func (s *metalDevice) ID() string { - return s.device.ID + return *s.device.Id } func (s *metalDevice) ProviderID() string { - if s.device == nil || s.device.ID == "" { + if s.device == nil || *s.device.Id == "" { return "" } - return "equinixmetal://" + s.device.ID + return "equinixmetal://" + *s.device.Id } -func (s *metalDevice) Addresses() map[string]v1.NodeAddressType { - // returns addresses in CIDR format - addresses := map[string]v1.NodeAddressType{} - for _, ip := range s.device.Network { - if ip.Public { - addresses[ip.Address] = v1.NodeExternalIP - continue +// Addresses returns addresses in CIDR format. +func (s *metalDevice) Addresses() map[string]corev1.NodeAddressType { + addresses := map[string]corev1.NodeAddressType{} + for _, ip := range s.device.IpAddresses { + kind := corev1.NodeInternalIP + if *ip.Public { + kind = corev1.NodeExternalIP } - addresses[ip.Address] = v1.NodeInternalIP + + addresses[*ip.Address] = kind } return addresses } func (s *metalDevice) Status() instance.Status { - switch s.device.State { - case "provisioning": + if s.device.State == nil { + return instance.StatusUnknown + } + + switch *s.device.State { + case metalv1.DEVICESTATE_PROVISIONING: return instance.StatusCreating - case "active": + case metalv1.DEVICESTATE_ACTIVE: return instance.StatusRunning default: return instance.StatusUnknown @@ -448,17 +500,23 @@ func setProviderSpec(rawConfig equinixmetaltypes.RawConfig, s clusterv1alpha1.Pr return &runtime.RawExtension{Raw: rawPconfig}, nil } -func getDeviceByTag(client *packngo.Client, projectID, tag string) (*packngo.Device, error) { - devices, response, err := client.Devices.List(projectID, nil) +func getDeviceByTag(ctx context.Context, client *metalv1.APIClient, projectID, tag string) (*metalv1.Device, error) { + request := client.DevicesApi. + FindProjectDevices(ctx, projectID). + Tag(tag) + + devices, response, err := client.DevicesApi.FindProjectDevicesExecute(request) if err != nil { return nil, metalErrorToTerminalError(err, response, "failed to list devices") } + response.Body.Close() - for _, device := range devices { - if itemInList(device.Tags, tag) { + for _, device := range devices.Devices { + if slices.Contains(device.Tags, tag) { return &device, nil } } + return nil, nil } @@ -477,10 +535,12 @@ func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { return "", providerconfigtypes.ErrOSNotSupported } -func getClient(apiKey string) *packngo.Client { - client := packngo.NewClientWithAuth("kubermatic", apiKey, nil) - client.UserAgent = fmt.Sprintf("kubermatic/machine-controller %s", client.UserAgent) - return client +func getClient(apiKey string) *metalv1.APIClient { + configuration := metalv1.NewConfiguration() + configuration.UserAgent = fmt.Sprintf("kubermatic/machine-controller %s", configuration.UserAgent) + configuration.AddDefaultHeader("X-Auth-Token", apiKey) + + return metalv1.NewAPIClient(configuration) } func generateTag(ID string) string { @@ -499,13 +559,13 @@ func getTagUID(tag string) (string, error) { // can be qualified as a "terminal" error, for more info see v1alpha1.MachineStatus // // if the given error doesn't qualify the error passed as an argument will be returned. -func metalErrorToTerminalError(err error, response *packngo.Response, msg string) error { +func metalErrorToTerminalError(err error, response *http.Response, msg string) error { prepareAndReturnError := func() error { - return fmt.Errorf("%s, due to %w", msg, err) + return fmt.Errorf("%s: %w", msg, err) } if err != nil { - if response != nil && response.Response != nil && response.Response.StatusCode == 403 { + if response != nil && response.StatusCode == http.StatusForbidden { // authorization primitives come from MachineSpec // thus we are setting InvalidConfigurationMachineError return cloudprovidererrors.TerminalError{ @@ -519,44 +579,3 @@ func metalErrorToTerminalError(err error, response *packngo.Response, msg string return err } - -func itemInList(list []string, item string) bool { - for _, elm := range list { - if elm == item { - return true - } - } - return false -} - -func itemsNotInList(list, items []string) []string { - listMap := make(map[string]bool) - missing := make([]string, 0) - for _, item := range list { - listMap[item] = true - } - for _, item := range items { - if _, ok := listMap[item]; !ok { - missing = append(missing, item) - } - } - return missing -} - -func facilityProp(vs []packngo.Facility, field string) []string { - vsm := make([]string, len(vs)) - for i, v := range vs { - val := reflect.ValueOf(v) - vsm[i] = val.FieldByName(field).String() - } - return vsm -} - -func planProp(vs []packngo.Plan, field string) []string { - vsm := make([]string, len(vs)) - for i, v := range vs { - val := reflect.ValueOf(v) - vsm[i] = val.FieldByName(field).String() - } - return vsm -} From b5a4e9101c15bb168d584a85af85eeff69734fac Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Fri, 8 Nov 2024 10:04:35 +0100 Subject: [PATCH 441/489] Support VM Network Multi-Queue in KubeVirt Provider (#1874) * support enableNetworkMultiQueue in KV VMs * adjust the tests --- pkg/cloudprovider/provider/kubevirt/provider.go | 16 ++++++++++++++-- .../kubevirt/testdata/affinity-no-values.yaml | 1 + .../provider/kubevirt/testdata/affinity.yaml | 1 + .../kubevirt/testdata/custom-local-disk.yaml | 1 + .../kubevirt/testdata/http-image-source.yaml | 1 + .../testdata/instancetype-preference-custom.yaml | 1 + .../instancetype-preference-standard.yaml | 1 + .../testdata/kubeovn-provider-network.yaml | 1 + .../provider/kubevirt/testdata/nominal-case.yaml | 1 + .../kubevirt/testdata/pvc-image-source.yaml | 1 + .../testdata/registry-image-source-pod.yaml | 1 + .../kubevirt/testdata/registry-image-source.yaml | 1 + .../kubevirt/testdata/secondary-disks.yaml | 1 + .../testdata/topologyspreadconstraints.yaml | 1 + .../testdata/use-storage-as-storage-target.yaml | 1 + .../provider/kubevirt/types/types.go | 13 +++++++------ 16 files changed, 35 insertions(+), 8 deletions(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 723cbef5f..71202ad8f 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -113,6 +113,7 @@ type Config struct { TopologySpreadConstraints []corev1.TopologySpreadConstraint Region string Zone string + EnableNetworkMultiQueue bool ProviderNetworkName string SubnetName string @@ -249,6 +250,16 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } } + var enableNetworkMultiQueueSet bool + config.EnableNetworkMultiQueue, enableNetworkMultiQueueSet, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.VirtualMachine.EnableNetworkMultiQueue) + if err != nil { + return nil, nil, fmt.Errorf(`failed to get value of "enableNetworkMultiQueue" field: %w`, err) + } + + if !enableNetworkMultiQueueSet { + config.EnableNetworkMultiQueue = true + } + config.ClusterName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ClusterName) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "clusterName" field: %w`, err) @@ -744,8 +755,9 @@ func (p *provider) newVirtualMachine(c *Config, pc *providerconfigtypes.Config, }, Domain: kubevirtv1.DomainSpec{ Devices: kubevirtv1.Devices{ - Interfaces: []kubevirtv1.Interface{*defaultBridgeNetwork}, - Disks: getVMDisks(c), + Interfaces: []kubevirtv1.Interface{*defaultBridgeNetwork}, + Disks: getVMDisks(c), + NetworkInterfaceMultiQueue: ptr.To(c.EnableNetworkMultiQueue), }, Resources: resourceRequirements, }, diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml index a8382d8ee..4e1986e86 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml @@ -56,6 +56,7 @@ spec: interfaces: - name: default bridge: {} + networkInterfaceMultiqueue: true resources: limits: cpu: "2" diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml index 7c0a3e9d8..61d9f81ac 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml @@ -59,6 +59,7 @@ spec: interfaces: - name: default bridge: {} + networkInterfaceMultiqueue: true resources: limits: cpu: "2" diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml index b83d4b01b..a783434ab 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml @@ -51,6 +51,7 @@ spec: interfaces: - name: default bridge: {} + networkInterfaceMultiqueue: true resources: limits: cpu: "2" diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml index f36d5392b..1f17d4d96 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml @@ -50,6 +50,7 @@ spec: interfaces: - name: default bridge: {} + networkInterfaceMultiqueue: true resources: limits: cpu: "2" diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml index 9154f71c1..fa8d2d933 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml @@ -56,6 +56,7 @@ spec: interfaces: - name: default bridge: {} + networkInterfaceMultiqueue: true networks: - name: default pod: {} diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml index c890481cb..1478d6539 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml @@ -56,6 +56,7 @@ spec: interfaces: - name: default bridge: {} + networkInterfaceMultiqueue: true networks: - name: default pod: {} diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/kubeovn-provider-network.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/kubeovn-provider-network.yaml index aa8dc5b6f..9bb367d33 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/kubeovn-provider-network.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/kubeovn-provider-network.yaml @@ -51,6 +51,7 @@ spec: interfaces: - name: default bridge: {} + networkInterfaceMultiqueue: true resources: limits: cpu: "2" diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml index 58f1ae771..c6eb221ad 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml @@ -50,6 +50,7 @@ spec: interfaces: - name: default bridge: {} + networkInterfaceMultiqueue: true resources: limits: cpu: "2" diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml index 61884708e..1eb84320d 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml @@ -51,6 +51,7 @@ spec: interfaces: - name: default bridge: {} + networkInterfaceMultiqueue: true resources: limits: cpu: "2" diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml index 69aaaac70..99e5546e4 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source-pod.yaml @@ -51,6 +51,7 @@ spec: interfaces: - name: default bridge: {} + networkInterfaceMultiqueue: true resources: limits: cpu: "2" diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml index f4608a12d..b1da847a3 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/registry-image-source.yaml @@ -51,6 +51,7 @@ spec: interfaces: - name: default bridge: {} + networkInterfaceMultiqueue: true resources: limits: cpu: "2" diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml index 8abda0e51..2be84ebf3 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml @@ -82,6 +82,7 @@ spec: interfaces: - name: default bridge: {} + networkInterfaceMultiqueue: true resources: limits: cpu: "2" diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml index 141091668..c12f06ef6 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml @@ -50,6 +50,7 @@ spec: interfaces: - name: default bridge: {} + networkInterfaceMultiqueue: true resources: limits: cpu: "2" diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml index b441d5533..09124dfa0 100644 --- a/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml +++ b/pkg/cloudprovider/provider/kubevirt/testdata/use-storage-as-storage-target.yaml @@ -51,6 +51,7 @@ spec: interfaces: - name: default bridge: {} + networkInterfaceMultiqueue: true resources: limits: cpu: "2" diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index 27641b130..804248c92 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -53,12 +53,13 @@ type VirtualMachine struct { // Instancetype is optional. Instancetype *kubevirtv1.InstancetypeMatcher `json:"instancetype,omitempty"` // Preference is optional. - Preference *kubevirtv1.PreferenceMatcher `json:"preference,omitempty"` - Template Template `json:"template,omitempty"` - DNSPolicy providerconfigtypes.ConfigVarString `json:"dnsPolicy,omitempty"` - DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` - Location *Location `json:"location,omitempty"` - ProviderNetwork *ProviderNetwork `json:"providerNetwork,omitempty"` + Preference *kubevirtv1.PreferenceMatcher `json:"preference,omitempty"` + Template Template `json:"template,omitempty"` + DNSPolicy providerconfigtypes.ConfigVarString `json:"dnsPolicy,omitempty"` + DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` + Location *Location `json:"location,omitempty"` + ProviderNetwork *ProviderNetwork `json:"providerNetwork,omitempty"` + EnableNetworkMultiQueue providerconfigtypes.ConfigVarBool `json:"enableNetworkMultiQueue,omitempty"` } // Flavor. From 51cf731d5c560ef4f3acdd7c771c31f0cd5ac95b Mon Sep 17 00:00:00 2001 From: Adonis Murati Date: Fri, 15 Nov 2024 14:05:01 +0100 Subject: [PATCH 442/489] Fix bug with handling server types and add new Hetzner server types (#1877) * test hetzner e2e test Signed-off-by: Adonis Murati * fix bug with serverType and add new Hetzner server types Signed-off-by: Adonis Murati * separate error handling with nil checking for clarity Signed-off-by: Adonis Murati --------- Signed-off-by: Adonis Murati --- docs/cloud-provider.md | 2 +- .../clusterv1alpha1machineWithProviderConfig/hetzner.yaml | 2 +- .../conversions/testdata/machinesv1alpha1machine/hetzner.yaml | 2 +- .../testdata/migrated_clusterv1alpha1machine/hetzner.yaml | 2 +- .../hetzner.yaml | 2 +- pkg/cloudprovider/provider/hetzner/provider.go | 4 ++++ test/e2e/provisioning/testdata/machine-invalid.yaml | 2 +- test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml | 2 +- 8 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index 24979afd7..18478c133 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -224,7 +224,7 @@ labels: ### machine.spec.providerConfig.cloudProviderSpec ```yaml token: "<< HETZNER_API_TOKEN >>" -serverType: "cx11" +serverType: "cx22" datacenter: "" location: "fsn1" # Optional: network IDs or names diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/hetzner.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/hetzner.yaml index 78e599569..fbf9753ab 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/hetzner.yaml +++ b/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/hetzner.yaml @@ -10,7 +10,7 @@ spec: cloudProviderSpec: datacenter: '' location: fsn1 - serverType: cx11 + serverType: cx22 token: << HETZNER_TOKEN >> operatingSystem: << OS_NAME >> operatingSystemSpec: diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/hetzner.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/hetzner.yaml index 62fa47ccc..68b92438b 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/hetzner.yaml +++ b/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/hetzner.yaml @@ -13,7 +13,7 @@ spec: namespace: kube-system name: machine-controller-hetzner key: token - serverType: "cx11" + serverType: "cx22" datacenter: "" location: "fsn1" operatingSystem: "ubuntu" diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/hetzner.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/hetzner.yaml index e5a817e69..a60736114 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/hetzner.yaml +++ b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/hetzner.yaml @@ -11,7 +11,7 @@ spec: cloudProviderSpec: datacenter: "" location: fsn1 - serverType: cx11 + serverType: cx22 token: secretKeyRef: key: token diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/hetzner.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/hetzner.yaml index e25b81872..767c358df 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/hetzner.yaml +++ b/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/hetzner.yaml @@ -13,7 +13,7 @@ spec: cloudProviderSpec: datacenter: "" location: fsn1 - serverType: cx11 + serverType: cx22 token: << HETZNER_TOKEN >> operatingSystem: << OS_NAME >> operatingSystemSpec: diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 3f353f87b..39db5e726 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -223,6 +223,10 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus return fmt.Errorf("failed to get server type: %w", err) } + if serverType == nil { + return fmt.Errorf("server type %q not found", c.ServerType) + } + image := c.Image if image == "" { image, err = getNameForOS(pc.OperatingSystem) diff --git a/test/e2e/provisioning/testdata/machine-invalid.yaml b/test/e2e/provisioning/testdata/machine-invalid.yaml index 17a100569..1fc9b1d9b 100644 --- a/test/e2e/provisioning/testdata/machine-invalid.yaml +++ b/test/e2e/provisioning/testdata/machine-invalid.yaml @@ -12,7 +12,7 @@ spec: cloudProvider: "hetzner" cloudProviderSpec: token: << HETZNER_TOKEN >> - serverType: "cx11" + serverType: "cx22" datacenter: "" location: "fsn1" operatingSystem: "<< OS_NAME >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml b/test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml index 66a5cc2ee..85e7f36b3 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml @@ -27,7 +27,7 @@ spec: cloudProvider: "hetzner" cloudProviderSpec: token: << HETZNER_TOKEN >> - serverType: "cx11" + serverType: "cx22" datacenter: "" location: "nbg1" networks: From 4f251c4bc5a387d48c95c44c851867898c8b9cb3 Mon Sep 17 00:00:00 2001 From: Adonis Murati Date: Mon, 18 Nov 2024 08:33:03 +0100 Subject: [PATCH 443/489] remove centos support from machine controller (#1876) Signed-off-by: Adonis Murati --- Makefile | 2 +- README.md | 2 +- docs/howto-provider.md | 2 +- docs/openstack-images.md | 2 +- docs/operating-system.md | 28 ++++----- docs/vsphere.md | 8 +-- examples/digitalocean-machinedeployment.yaml | 1 - examples/kubevirt-machinedeployment.yaml | 2 +- examples/openstack-machinedeployment.yaml | 1 - examples/operating-system-manager.yaml | 2 - examples/scaleway-machinedeployment.yaml | 1 - hack/build-kubevirt-images.sh | 2 +- hack/e2e-setup-openstack-images.sh | 1 - hack/kubevirt_dockerfiles/dockerfile.centos | 3 - hack/setup-openstack-images.sh | 12 ---- image-builder/README.md | 6 +- image-builder/build.sh | 35 +---------- .../machinesv1alpha1machine/openstack.yaml | 1 - .../provider/alibaba/provider.go | 3 - pkg/cloudprovider/provider/aws/provider.go | 15 ----- pkg/cloudprovider/provider/azure/provider.go | 6 -- .../provider/digitalocean/provider.go | 2 - .../provider/equinixmetal/provider.go | 2 - .../provider/hetzner/provider.go | 2 - .../provider/kubevirt/types/types.go | 1 - pkg/cloudprovider/provider/linode/provider.go | 6 -- .../provider/scaleway/provider.go | 2 - pkg/cloudprovider/provider/vultr/provider.go | 3 - pkg/providerconfig/types.go | 3 - pkg/providerconfig/types/types.go | 2 - pkg/userdata/centos/centos.go | 59 ------------------- 31 files changed, 25 insertions(+), 192 deletions(-) delete mode 100644 hack/kubevirt_dockerfiles/dockerfile.centos delete mode 100644 pkg/userdata/centos/centos.go diff --git a/Makefile b/Makefile index 098c867be..e338ed2ec 100644 --- a/Makefile +++ b/Makefile @@ -33,7 +33,7 @@ IMAGE_TAG = \ $(shell echo $$(git rev-parse HEAD && if [[ -n $$(git status --porcelain) ]]; then echo '-dirty'; fi)|tr -d ' ') IMAGE_NAME ?= $(REGISTRY)/$(REGISTRY_NAMESPACE)/machine-controller:$(IMAGE_TAG) -OS = amzn2 centos ubuntu rhel flatcar rockylinux +OS = amzn2 ubuntu rhel flatcar rockylinux BASE64_ENC = \ $(shell if base64 -w0 <(echo "") &> /dev/null; then echo "base64 -w0"; else echo "base64 -b0"; fi) diff --git a/README.md b/README.md index c46afab47..858bc3aac 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ ### What Works - Creation of worker nodes on AWS, Digitalocean, Openstack, Azure, Google Cloud Platform, Nutanix, VMWare Cloud Director, VMWare vSphere, Hetzner Cloud and Kubevirt -- Using Ubuntu, Flatcar, CentOS 7 or Rocky Linux 8 distributions ([not all distributions work on all providers](/docs/operating-system.md)) +- Using Ubuntu, Flatcar, or Rocky Linux 8 distributions ([not all distributions work on all providers](/docs/operating-system.md)) ### Supported Kubernetes Versions diff --git a/docs/howto-provider.md b/docs/howto-provider.md index b6bc583d1..3f72a5a84 100644 --- a/docs/howto-provider.md +++ b/docs/howto-provider.md @@ -65,7 +65,7 @@ Provider implementations are located in individual packages in `k8c.io/machine-c When retrieving the individual configuration from the provider specification a type for unmarshalling is needed. Here first the provider configuration is read and based on it the individual values of the configuration are retrieved. Typically the access data (token, ID/key combination, document with all information) alternatively can be passed via an environment variable. According methods of the used `providerconfig.ConfigVarResolver` do support this. -For creation of new machines the support of the possible information has to be checked. The machine controller supports _CentOS_, _Flatcar_ and _Ubuntu_. In case one or more aren't supported by the cloud infrastructure the error `providerconfig.ErrOSNotSupported` has to be returned. +For creation of new machines the support of the possible information has to be checked. The machine controller supports _Flatcar_ and _Ubuntu_. In case one or more aren't supported by the cloud infrastructure the error `providerconfig.ErrOSNotSupported` has to be returned. ## Integrate provider into the Machine Controller diff --git a/docs/openstack-images.md b/docs/openstack-images.md index a1979885c..8a89250e9 100644 --- a/docs/openstack-images.md +++ b/docs/openstack-images.md @@ -10,5 +10,5 @@ There is a script to upload all supported image to OpenStack. By default all images will be named `machine-controller-${OS_NAME}`. The image names can be overwritten using environment variables: ```bash -UBUNTU_IMAGE_NAME="ubuntu" CENTOS_IMAGE_NAME="centos" ./hack/setup-openstack-images.sh +UBUNTU_IMAGE_NAME="ubuntu" ./hack/setup-openstack-images.sh ``` diff --git a/docs/operating-system.md b/docs/operating-system.md index fd6994ece..c4b7692f6 100644 --- a/docs/operating-system.md +++ b/docs/operating-system.md @@ -4,19 +4,19 @@ ### Cloud provider -| | Ubuntu | CentOS | Flatcar | RHEL | Amazon Linux 2 | Rocky Linux | -|---|---|---|---|---|---|---| -| AWS | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Azure | ✓ | ✓ | ✓ | ✓ | x | ✓ | -| Digitalocean | ✓ | ✓ | x | x | x | ✓ | -| Equinix Metal | ✓ | ✓ | ✓ | x | x | ✓ | -| Google Cloud Platform | ✓ | x | ✓ | x | x | x | -| Hetzner | ✓ | x | x | x | x | ✓ | -| KubeVirt | ✓ | ✓ | ✓ | ✓ | x | ✓ | -| Nutanix | ✓ | ✓ | x | x | x | x | -| Openstack | ✓ | ✓ | ✓ | ✓ | x | ✓ | -| VMware Cloud Director | ✓ | x | x | x | x | x | -| VSphere | ✓ | ✓ | ✓ | ✓ | x | ✓ | +| | Ubuntu | Flatcar | RHEL | Amazon Linux 2 | Rocky Linux | +|---|---|---|---|---|---| +| AWS | ✓ | ✓ | ✓ | ✓ | ✓ | +| Azure | ✓ | ✓ | ✓ | x | ✓ | +| Digitalocean | ✓ | x | x | x | ✓ | +| Equinix Metal | ✓ | ✓ | x | x | ✓ | +| Google Cloud Platform | ✓ | ✓ | x | x | x | +| Hetzner | ✓ | x | x | x | ✓ | +| KubeVirt | ✓ | ✓ | ✓ | x | ✓ | +| Nutanix | ✓ | x | x | x | x | +| Openstack | ✓ | ✓ | ✓ | x | ✓ | +| VMware Cloud Director | ✓ | x | x | x | x | +| VSphere | ✓ | ✓ | ✓ | x | ✓ | ## Configuring a operating system @@ -24,7 +24,6 @@ The operating system to use can be set via `machine.spec.providerConfig.operatin Allowed values: - `amzn2` -- `centos` - `flatcar` - `rhel` - `rockylinux` @@ -40,7 +39,6 @@ Machine controller may work with other OS versions that are not listed in the ta | | Versions | |---|---| | AmazonLinux2 | 2.x | -| CentOS | 7.4.x, 7.6.x, 7.7.x | | RHEL | 8.x | | Rocky Linux | 8.5 | | Ubuntu | 20.04 LTS, 22.04 LTS | diff --git a/docs/vsphere.md b/docs/vsphere.md index ab6962c94..0a299cb2a 100644 --- a/docs/vsphere.md +++ b/docs/vsphere.md @@ -154,7 +154,7 @@ Procedure: ``` # The URL below is just an example - image_url="https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2" + image_url="https://cloud-images.ubuntu.com/releases/20.04/release/ubuntu-20.04-server-cloudimg-amd64.img" image_name="$(basename -- "${image_url}" | sed 's/.qcow2$//g')" curl -sL "${image_url}" -O . ``` @@ -203,12 +203,6 @@ Red Hat Enterprise Linux 8.x KVM Guest Image can be found at [Red Hat Customer P Follow [qcow2](#create-template-vm-from-qcow2) template VM creation guide. -#### CentOS - -CentOS 7 image can be found at the following link: . - -Follow [qcow2](#create-template-vm-from-qcow2) template VM creation guide. - ## Provider configuration VSphere provider accepts the following configuration parameters: diff --git a/examples/digitalocean-machinedeployment.yaml b/examples/digitalocean-machinedeployment.yaml index 043787ebd..50793d6c5 100644 --- a/examples/digitalocean-machinedeployment.yaml +++ b/examples/digitalocean-machinedeployment.yaml @@ -52,7 +52,6 @@ spec: monitoring: false tags: - "machine-controller" - # Can be 'ubuntu' or 'centos' operatingSystem: "ubuntu" operatingSystemSpec: disableAutoUpdate: true diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index 96bddfe6a..567c29f2c 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -56,7 +56,7 @@ spec: - maxSkew: "1" topologyKey: "kubernetes.io/hostname" whenUnsatisfiable: "" # Allowed values: "DoNotSchedule", "ScheduleAnyway" - # Can also be `centos`, must align with he configured registryImage above + # Must align with the configured registryImage above operatingSystem: "ubuntu" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index ffc93c2f8..5b9dec66e 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -153,7 +153,6 @@ spec: # the list of metadata you would like to attach to the instance tags: tagKey: tagValue - # Can be 'ubuntu' or 'centos' operatingSystem: "ubuntu" operatingSystemSpec: distUpgradeOnBoot: true diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index ce1648bb6..da5a8b5e2 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -233,7 +233,6 @@ spec: enum: - flatcar - rhel - - centos - ubuntu - amzn2 - rockylinux @@ -683,7 +682,6 @@ spec: enum: - flatcar - rhel - - centos - ubuntu - amzn2 - rockylinux diff --git a/examples/scaleway-machinedeployment.yaml b/examples/scaleway-machinedeployment.yaml index 9b7f7ca7c..1f5b8d163 100644 --- a/examples/scaleway-machinedeployment.yaml +++ b/examples/scaleway-machinedeployment.yaml @@ -56,7 +56,6 @@ spec: ipv6: false tags: - "machine-controller" - # Can be 'ubuntu' or 'centos' operatingSystem: "ubuntu" operatingSystemSpec: disableAutoUpdate: true diff --git a/hack/build-kubevirt-images.sh b/hack/build-kubevirt-images.sh index 440a44217..2028e2151 100755 --- a/hack/build-kubevirt-images.sh +++ b/hack/build-kubevirt-images.sh @@ -20,7 +20,7 @@ BUILD_NUM=2 cd $(dirname $0)/kubevirt_dockerfiles -for flavor in ubuntu centos; do +for flavor in ubuntu; do docker build \ -t quay.io/kubermatic/machine-controller-kubevirt:$flavor-$BUILD_NUM \ -f dockerfile.$flavor . diff --git a/hack/e2e-setup-openstack-images.sh b/hack/e2e-setup-openstack-images.sh index d67c3825f..309b9bf04 100755 --- a/hack/e2e-setup-openstack-images.sh +++ b/hack/e2e-setup-openstack-images.sh @@ -20,6 +20,5 @@ set -o pipefail cd $(dirname $0)/ export UBUNTU_IMAGE_NAME="machine-controller-e2e-ubuntu" -export CENTOS_IMAGE_NAME="machine-controller-e2e-centos" ./setup-openstack-images.sh diff --git a/hack/kubevirt_dockerfiles/dockerfile.centos b/hack/kubevirt_dockerfiles/dockerfile.centos deleted file mode 100644 index c26389521..000000000 --- a/hack/kubevirt_dockerfiles/dockerfile.centos +++ /dev/null @@ -1,3 +0,0 @@ -FROM kubevirt/registry-disk-v1alpha:v0.10.0 - -RUN curl -L -o /disk/centos7.img https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2 diff --git a/hack/setup-openstack-images.sh b/hack/setup-openstack-images.sh index 52ab771ae..28d51b005 100755 --- a/hack/setup-openstack-images.sh +++ b/hack/setup-openstack-images.sh @@ -18,7 +18,6 @@ set -o nounset set -o pipefail UBUNTU_IMAGE_NAME=${UBUNTU_IMAGE_NAME:-"machine-controller-ubuntu"} -CENTOS_IMAGE_NAME=${CENTOS_IMAGE_NAME:-"machine-controller-centos"} echo "Downloading Ubuntu 18.04 image from upstream..." curl -L -o ubuntu.img http://cloud-images.ubuntu.com/bionic/current/bionic-server-cloudimg-amd64.img @@ -30,14 +29,3 @@ openstack image create \ ${UBUNTU_IMAGE_NAME} rm ubuntu.img echo "Successfully uploaded ${UBUNTU_IMAGE_NAME} to OpenStack..." - -echo "Downloading CentOS 7 image from upstream..." -curl -L -o centos.qcow2 http://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud.qcow2 -echo "Uploading CentOS 7 image to OpenStack..." -openstack image create \ - --disk-format qcow2 \ - --container-format bare \ - --file centos.qcow2 \ - ${CENTOS_IMAGE_NAME} -rm centos.qcow2 -echo "Successfully uploaded ${CENTOS_IMAGE_NAME} to OpenStack..." diff --git a/image-builder/README.md b/image-builder/README.md index 613132bac..743cc7adc 100644 --- a/image-builder/README.md +++ b/image-builder/README.md @@ -4,19 +4,17 @@ Currently supported operating systems: * RedHat CoreOS - * CentOS 7 * Debian 9 ### Usage -`./build.sh --target-os centos7|debian9 [--release K8S-RELEASE]` +`./build.sh --target-os debian9 [--release K8S-RELEASE]` Parameters: * `--target-os` is mandatory and specifies the Linux distribution image to be built. Possible values: - * `centos7` * `debian9` * `--release` specifies the Kubernetes release to be added to the image, e.g. `v1.10.2`. If not provided, the script will look up the latest stable release and use that. ### Output -The script will generate a VMDK disk image with the filename `TARGET_OS-output.vmdk`, e.g. `centos7-output.vmdk`. +The script will generate a VMDK disk image with the filename `TARGET_OS-output.vmdk`. diff --git a/image-builder/build.sh b/image-builder/build.sh index c8784a55c..97bff8688 100755 --- a/image-builder/build.sh +++ b/image-builder/build.sh @@ -23,7 +23,7 @@ TARGET_OS="" usage() { echo -e "usage:" - echo -e "\t$0 --target-os centos7|debian9|ubuntu-xenial|ubuntu-bionic [--release K8S-RELEASE]" + echo -e "\t$0 --target-os debian9|ubuntu-xenial|ubuntu-bionic [--release K8S-RELEASE]" } while [ $# -gt 0 ]; do @@ -34,7 +34,7 @@ while [ $# -gt 0 ]; do ;; --target-os) if [[ -z $2 ]]; then - echo "You must specify target OS. Currently 'centos7' is supported." + echo "You must specify target OS." exit 1 fi TARGET_OS="$2" @@ -80,29 +80,6 @@ mkdir -p "$TARGETFS" "$SCRIPT_DIR/downloads" # on failure unmount target filesystem (if mounted) and delete the temporary directory trap "sudo mountpoint --quiet $TARGETFS && sudo umount --recursive $TARGETFS; rm -rf $TEMPDIR" EXIT SIGINT -get_centos7_image() { - CENTOS7_BUILD="1802" - echo " * Downloading vanilla CentOS image." - wget "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-$CENTOS7_BUILD.qcow2.xz" -P "$TEMPDIR" - - echo " * Verifying GPG signature" - wget --quiet "https://cloud.centos.org/centos/7/images/sha256sum.txt.asc" -O "$TEMPDIR/centos7-sha256sum.txt.asc" - gpg2 --quiet --import "$SCRIPT_DIR/RPM-GPG-KEY-CentOS-7" - gpg2 "$TEMPDIR/centos7-sha256sum.txt.asc" - - echo " * Verifying SHA256 digest" - EXPECTED_SHA256="$(grep "CentOS-7-x86_64-GenericCloud-$CENTOS7_BUILD.qcow2.xz$" <"$TEMPDIR/centos7-sha256sum.txt" | cut -f1 -d ' ')" - CALCULATED_SHA256="$(sha256sum "$TEMPDIR/CentOS-7-x86_64-GenericCloud-$CENTOS7_BUILD.qcow2.xz" | cut -f1 -d ' ')" - if [[ $CALCULATED_SHA256 != "$EXPECTED_SHA256" ]]; then - echo " * SHA256 digest verification failed. '$CALCULATED_SHA256' != '$EXPECTED_SHA256'" - exit 1 - fi - - echo " * Decompressing" - unxz --keep "$TEMPDIR/CentOS-7-x86_64-GenericCloud-$CENTOS7_BUILD.qcow2.xz" - mv "$TEMPDIR/CentOS-7-x86_64-GenericCloud-$CENTOS7_BUILD.qcow2" "$SCRIPT_DIR/downloads/CentOS-7-x86_64-GenericCloud.qcow2" -} - get_debian9_image() { DEBIAN_CD_SIGNING_KEY_FINGERPRINT="DF9B9C49EAA9298432589D76DA87E80D6294BE9B" @@ -167,7 +144,7 @@ mount_rootfs() { local IMAGE="$1" local FOLDER="$2" case $TARGET_OS in - debian9 | centos7 | ubuntu-*) + debian9 | ubuntu-*) echo " * /" sudo guestmount -a "$IMAGE" -m "/dev/sda1" "$TARGETFS" ;; @@ -180,12 +157,6 @@ mount_rootfs() { } case $TARGET_OS in -centos7) - CLEAN_IMAGE="$SCRIPT_DIR/downloads/CentOS-7-x86_64-GenericCloud.qcow2" - if [[ ! -f $CLEAN_IMAGE ]]; then - get_centos7_image - fi - ;; debian9) CLEAN_IMAGE="$SCRIPT_DIR/downloads/debian-9-openstack-amd64.qcow2" if [[ ! -f $CLEAN_IMAGE ]]; then diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml b/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml index 157d535d4..9c0b49dda 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml +++ b/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml @@ -65,7 +65,6 @@ spec: # the list of tags you would like to attach to the instance tags: tagKey: tagValue - # Can be 'ubuntu' or 'centos' operatingSystem: "ubuntu" operatingSystemSpec: distUpgradeOnBoot: true diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index 9f7f6a5bc..e46ce5d1f 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -44,7 +44,6 @@ import ( const ( machineUIDTag = "machine_uid" - centosImageName = "CentOS 7.9 64 bit" ubuntuImageName = "Ubuntu 22.04 64 bit" finalizerInstance = "kubermatic.io/cleanup-alibaba-instance" @@ -456,8 +455,6 @@ func (p *provider) getImageIDForOS(machineSpec clusterv1alpha1.MachineSpec, os p switch image.OSNameEn { case ubuntuImageName: availableImage[providerconfigtypes.OperatingSystemUbuntu] = image.ImageId - case centosImageName: - availableImage[providerconfigtypes.OperatingSystemCentOS] = image.ImageId } } diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 2527e301a..4dba90c62 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -105,19 +105,6 @@ var ( } amiFilters = map[providerconfigtypes.OperatingSystem]map[awstypes.CPUArchitecture]amiFilter{ - // Source: https://wiki.centos.org/Cloud/AWS - providerconfigtypes.OperatingSystemCentOS: { - awstypes.CPUArchitectureX86_64: { - description: "CentOS Linux 7* x86_64*", - // The AWS marketplace ID from CentOS Community Platform Engineering (CPE) - owner: "125523088429", - }, - awstypes.CPUArchitectureARM64: { - description: "CentOS Linux 7* aarch64*", - // The AWS marketplace ID from CentOS Community Platform Engineering (CPE) - owner: "125523088429", - }, - }, providerconfigtypes.OperatingSystemRockyLinux: { awstypes.CPUArchitectureX86_64: { description: "*Rocky-8-EC2-*.x86_64", @@ -337,8 +324,6 @@ func getDefaultRootDevicePath(os providerconfigtypes.OperatingSystem) (string, e switch os { case providerconfigtypes.OperatingSystemUbuntu: return rootDevicePathSDA, nil - case providerconfigtypes.OperatingSystemCentOS: - return rootDevicePathSDA, nil case providerconfigtypes.OperatingSystemRockyLinux: return rootDevicePathSDA, nil case providerconfigtypes.OperatingSystemRHEL: diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index dc6200a4b..96532d88e 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -141,12 +141,6 @@ func (vm *azureVM) Status() instance.Status { } var imageReferences = map[providerconfigtypes.OperatingSystem]compute.ImageReference{ - providerconfigtypes.OperatingSystemCentOS: { - Publisher: to.StringPtr("OpenLogic"), - Offer: to.StringPtr("CentOS"), - Sku: to.StringPtr("7_9"), // https://docs.microsoft.com/en-us/azure/virtual-machines/linux/using-cloud-init - Version: to.StringPtr("latest"), - }, providerconfigtypes.OperatingSystemUbuntu: { Publisher: to.StringPtr("Canonical"), Offer: to.StringPtr("ubuntu-24_04-lts"), diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index 6cb24fc70..e8d03129e 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -86,8 +86,6 @@ func getSlugForOS(os providerconfigtypes.OperatingSystem) (string, error) { switch os { case providerconfigtypes.OperatingSystemUbuntu: return "ubuntu-24-04-x64", nil - case providerconfigtypes.OperatingSystemCentOS: - return "centos-7-x64", nil case providerconfigtypes.OperatingSystemRockyLinux: return "rockylinux-8-x64", nil } diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index d7a271377..faaba8db7 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -525,8 +525,6 @@ func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { switch os { case providerconfigtypes.OperatingSystemUbuntu: return "ubuntu_24_04", nil - case providerconfigtypes.OperatingSystemCentOS: - return "centos_7", nil case providerconfigtypes.OperatingSystemFlatcar: return "flatcar_stable", nil case providerconfigtypes.OperatingSystemRockyLinux: diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 39db5e726..93dc87383 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -73,8 +73,6 @@ func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { switch os { case providerconfigtypes.OperatingSystemUbuntu: return "ubuntu-24.04", nil - case providerconfigtypes.OperatingSystemCentOS: - return "centos-7", nil case providerconfigtypes.OperatingSystemRockyLinux: return "rocky-8", nil } diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index 804248c92..4cac85ec0 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -26,7 +26,6 @@ import ( ) var SupportedOS = map[providerconfigtypes.OperatingSystem]*struct{}{ - providerconfigtypes.OperatingSystemCentOS: nil, providerconfigtypes.OperatingSystemUbuntu: nil, providerconfigtypes.OperatingSystemRHEL: nil, providerconfigtypes.OperatingSystemFlatcar: nil, diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index f38e77948..2a8c98acf 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -85,12 +85,6 @@ func getSlugForOS(os providerconfigtypes.OperatingSystem) (string, error) { switch os { case providerconfigtypes.OperatingSystemUbuntu: return "linode/ubuntu18.04", nil - - /** - // StackScript for CloudInit is not centos7 ready - case providerconfigtypes.OperatingSystemCentOS: - return "linode/centos7", nil - **/ } return "", providerconfigtypes.ErrOSNotSupported } diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go index 632a6464f..c31dbc874 100644 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ b/pkg/cloudprovider/provider/scaleway/provider.go @@ -81,8 +81,6 @@ func getImageNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { // ubuntu_focal doesn't work (see https://bugs.launchpad.net/ubuntu/+source/linux-kvm/+bug/1880522) // modprobe ip_vs will fail return "ubuntu_bionic", nil - case providerconfigtypes.OperatingSystemCentOS: - return "centos_7.6", nil } return "", providerconfigtypes.ErrOSNotSupported } diff --git a/pkg/cloudprovider/provider/vultr/provider.go b/pkg/cloudprovider/provider/vultr/provider.go index f2b3c9698..d51bac3d2 100644 --- a/pkg/cloudprovider/provider/vultr/provider.go +++ b/pkg/cloudprovider/provider/vultr/provider.go @@ -82,9 +82,6 @@ func getIDForOS(os providerconfigtypes.OperatingSystem) (int, error) { switch os { case providerconfigtypes.OperatingSystemUbuntu: return 1743, nil - // name: CentOS 7 x64 - case providerconfigtypes.OperatingSystemCentOS: - return 167, nil // name: Rocky Linux 9 x64 case providerconfigtypes.OperatingSystemRockyLinux: return 1869, nil diff --git a/pkg/providerconfig/types.go b/pkg/providerconfig/types.go index 13d21b0a3..c88f6df06 100644 --- a/pkg/providerconfig/types.go +++ b/pkg/providerconfig/types.go @@ -26,7 +26,6 @@ import ( providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" "k8c.io/machine-controller/pkg/userdata/amzn2" - "k8c.io/machine-controller/pkg/userdata/centos" "k8c.io/machine-controller/pkg/userdata/flatcar" "k8c.io/machine-controller/pkg/userdata/rhel" "k8c.io/machine-controller/pkg/userdata/rockylinux" @@ -186,8 +185,6 @@ func DefaultOperatingSystemSpec( switch osys { case providerconfigtypes.OperatingSystemAmazonLinux2: return amzn2.DefaultConfig(operatingSystemSpec), nil - case providerconfigtypes.OperatingSystemCentOS: - return centos.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemFlatcar: return flatcar.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemRHEL: diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 5ca2a3169..5fe41f0d5 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -36,7 +36,6 @@ type OperatingSystem string const ( OperatingSystemUbuntu OperatingSystem = "ubuntu" - OperatingSystemCentOS OperatingSystem = "centos" OperatingSystemAmazonLinux2 OperatingSystem = "amzn2" OperatingSystemRHEL OperatingSystem = "rhel" OperatingSystemFlatcar OperatingSystem = "flatcar" @@ -85,7 +84,6 @@ var ( // AllOperatingSystems is a slice containing all supported operating system identifiers. AllOperatingSystems = []OperatingSystem{ OperatingSystemUbuntu, - OperatingSystemCentOS, OperatingSystemAmazonLinux2, OperatingSystemRHEL, OperatingSystemFlatcar, diff --git a/pkg/userdata/centos/centos.go b/pkg/userdata/centos/centos.go deleted file mode 100644 index 0350c43b5..000000000 --- a/pkg/userdata/centos/centos.go +++ /dev/null @@ -1,59 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package centos - -import ( - "encoding/json" - - "k8s.io/apimachinery/pkg/runtime" -) - -// Config contains specific configuration for CentOS. -type Config struct { - DistUpgradeOnBoot bool `json:"distUpgradeOnBoot"` -} - -func DefaultConfig(operatingSystemSpec runtime.RawExtension) runtime.RawExtension { - if operatingSystemSpec.Raw == nil { - operatingSystemSpec.Raw, _ = json.Marshal(Config{}) - } - - return operatingSystemSpec -} - -// LoadConfig retrieves the CentOS configuration from raw data. -func LoadConfig(r runtime.RawExtension) (*Config, error) { - r = DefaultConfig(r) - cfg := Config{} - - if err := json.Unmarshal(r.Raw, &cfg); err != nil { - return nil, err - } - return &cfg, nil -} - -// Spec return the configuration as raw data. -func (cfg *Config) Spec() (*runtime.RawExtension, error) { - ext := &runtime.RawExtension{} - b, err := json.Marshal(cfg) - if err != nil { - return nil, err - } - - ext.Raw = b - return ext, nil -} From 62b9f9efea2009e11ce4fe39d0e557e3b4094dd8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Nov 2024 09:59:03 +0100 Subject: [PATCH 444/489] Bump github.com/golang-jwt/jwt/v4 from 4.5.0 to 4.5.1 (#1875) Bumps [github.com/golang-jwt/jwt/v4](https://github.com/golang-jwt/jwt) from 4.5.0 to 4.5.1. - [Release notes](https://github.com/golang-jwt/jwt/releases) - [Changelog](https://github.com/golang-jwt/jwt/blob/main/VERSION_HISTORY.md) - [Commits](https://github.com/golang-jwt/jwt/compare/v4.5.0...v4.5.1) --- updated-dependencies: - dependency-name: github.com/golang-jwt/jwt/v4 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index fe6441475..af6039be7 100644 --- a/go.mod +++ b/go.mod @@ -111,7 +111,7 @@ require ( github.com/go-resty/resty/v2 v2.14.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang-jwt/jwt/v4 v4.5.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect diff --git a/go.sum b/go.sum index ee7d9e6c9..8f363eb9e 100644 --- a/go.sum +++ b/go.sum @@ -207,8 +207,9 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= From 86c117a5f29fb17145f636f4507c6ab39129ca15 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Mon, 18 Nov 2024 10:43:03 +0100 Subject: [PATCH 445/489] Support Importing Volume Over HTTP Using Basic Auth (#1878) * support extra headers in the CDI volume importing Signed-off-by: moadqassem * adding tests Signed-off-by: moadqassem * exclude getConfig method from the cyclomatic check Signed-off-by: moadqassem --------- Signed-off-by: moadqassem --- .golangci.yml | 1 + .../provider/kubevirt/provider.go | 65 +++++++++++++-- .../provider/kubevirt/provider_test.go | 12 ++- .../kubevirt/testdata/extra-headers-set.yaml | 82 +++++++++++++++++++ .../provider/kubevirt/types/types.go | 6 ++ 5 files changed, 157 insertions(+), 9 deletions(-) create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/extra-headers-set.yaml diff --git a/.golangci.yml b/.golangci.yml index 039495351..0dd9bcf42 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -56,6 +56,7 @@ issues: - func Convert_Machine_ProviderConfig_To_ProviderSpec should be ConvertMachineProviderConfigToProviderSpec - 'cyclomatic complexity [0-9]+ of func `\(\*provider\)\.Create` is high' - 'cyclomatic complexity [0-9]+ of func `\(\*provider\)\.Validate` is high' + - 'cyclomatic complexity [0-9]+ of func `\(\*provider\)\.getConfig` is high' - "SA1019: s.server.IPv6 is deprecated" exclude-dirs: - pkg/machines diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 71202ad8f..12db060aa 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -114,6 +114,8 @@ type Config struct { Region string Zone string EnableNetworkMultiQueue bool + ExtraHeaders []string + ExtraHeadersSecretRef string ProviderNetworkName string SubnetName string @@ -279,8 +281,18 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf(`failed to get value of "memory" field: %w`, err) } config.Namespace = getNamespace() - - config.OSImageSource, err = p.parseOSImageSource(rawConfig.VirtualMachine.Template.PrimaryDisk, config.Namespace) + if len(rawConfig.VirtualMachine.Template.PrimaryDisk.ExtraHeaders) > 0 { + config.ExtraHeaders = rawConfig.VirtualMachine.Template.PrimaryDisk.ExtraHeaders + } + extraHeadersSecretRef, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.ExtraHeadersSecretRef) + if err != nil { + return nil, nil, fmt.Errorf(`failed to get value of "extraHeadersSecretRef" field: %w`, err) + } + config.ExtraHeadersSecretRef = extraHeadersSecretRef + if len(config.ExtraHeaders) > 0 && extraHeadersSecretRef != "" { + return nil, nil, errors.New(`field "extraHeaders" and "extraHeadersSecretRef" are mutually exclusive`) + } + config.OSImageSource, err = p.parseOSImageSource(rawConfig.VirtualMachine.Template.PrimaryDisk, &config) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "osImageSource" field: %w`, err) } @@ -302,7 +314,6 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "storageClassName" field: %w`, err) } - // Instancetype and Preference config.Instancetype = rawConfig.VirtualMachine.Instancetype config.Preference = rawConfig.VirtualMachine.Preference @@ -427,7 +438,7 @@ func (p *provider) parseTopologySpreadConstraint(topologyConstraints []kubevirtt return parsedTopologyConstraints, nil } -func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, namespace string) (*cdiv1beta1.DataVolumeSource, error) { +func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, config *Config) (*cdiv1beta1.DataVolumeSource, error) { osImage, err := p.configVarResolver.GetConfigVarStringValue(primaryDisk.OsImage) if err != nil { return nil, fmt.Errorf(`failed to get value of "primaryDisk.osImage" field: %w`, err) @@ -442,29 +453,67 @@ func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, nam } switch imageSource(osImageSource) { case httpSource: - return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage}}, nil + extraHeaders, err := getHTTPExtraHeaders(config) + if err != nil { + return nil, fmt.Errorf(`failed to get value of "primaryDisk.extraHeaders" field: %w`, err) + } + + return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage, ExtraHeaders: extraHeaders}}, nil case registrySource: return registryDataVolume(osImage, pullMethod), nil case pvcSource: if namespaceAndName := strings.Split(osImage, "/"); len(namespaceAndName) >= 2 { return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: namespaceAndName[1], Namespace: namespaceAndName[0]}}, nil } - return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: namespace}}, nil + return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: config.Namespace}}, nil default: // handle old API for backward compatibility. if srcURL, err := url.ParseRequestURI(osImage); err == nil { if srcURL.Scheme == cdiv1beta1.RegistrySchemeDocker || srcURL.Scheme == cdiv1beta1.RegistrySchemeOci { return registryDataVolume(osImage, pullMethod), nil } - return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage}}, nil + + extraHeaders, err := getHTTPExtraHeaders(config) + if err != nil { + return nil, fmt.Errorf(`failed to get value of "primaryDisk.extraHeaders" field: %w`, err) + } + + return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage, ExtraHeaders: extraHeaders}}, nil } if namespaceAndName := strings.Split(osImage, "/"); len(namespaceAndName) >= 2 { return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: namespaceAndName[1], Namespace: namespaceAndName[0]}}, nil } - return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: namespace}}, nil + return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: config.Namespace}}, nil } } +func getHTTPExtraHeaders(config *Config) ([]string, error) { + var extraHeaders []string + if config.ExtraHeadersSecretRef != "" { + sigClient, err := client.New(config.RestConfig, client.Options{}) + if err != nil { + return nil, fmt.Errorf("failed to get kubevirt client: %w", err) + } + + extraHeadersSecretRef := &corev1.Secret{} + if err := sigClient.Get(context.TODO(), types.NamespacedName{Namespace: config.Namespace, Name: config.ExtraHeadersSecretRef}, + extraHeadersSecretRef); err != nil { + return nil, fmt.Errorf("failed to get extra headers secret: %w", err) + } + + for key, val := range extraHeadersSecretRef.Data { + trimmedVal := strings.TrimSuffix(string(val), "\n") + extraHeaders = append(extraHeaders, fmt.Sprintf("%v: %v", key, trimmedVal)) + } + } + + if len(config.ExtraHeaders) > 0 { + extraHeaders = config.ExtraHeaders + } + + return extraHeaders, nil +} + // getNamespace returns the namespace where the VM is created. // VM is created in a dedicated namespace // which is the namespace where the machine-controller pod is running. diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index c58412147..49b5b1a5d 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -21,7 +21,6 @@ import ( "context" "embed" "html/template" - "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/types" "path" "reflect" "testing" @@ -29,6 +28,7 @@ import ( kubevirtv1 "kubevirt.io/api/core/v1" cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" + "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/types" cloudprovidertesting "k8c.io/machine-controller/pkg/cloudprovider/testing" "k8c.io/machine-controller/pkg/providerconfig" @@ -70,6 +70,7 @@ type kubevirtProviderSpecConf struct { OsImageSourceURL string PullMethod cdiv1beta1.RegistryPullMethod ProviderNetwork *types.ProviderNetwork + ExtraHeadersSet bool } func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { @@ -139,6 +140,9 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { "storageClassName": "longhorn3"}], {{- end }} "primaryDisk": { + {{- if .ExtraHeadersSet }} + "extraHeaders": ["authorization: Basic bXE6cGFzc3dvcmQ="], + {{- end }} "storageAccessType": "ReadWriteMany", {{- if .StorageTarget }} "storageTarget": "{{ .StorageTarget }}", @@ -192,6 +196,12 @@ func TestNewVirtualMachine(t *testing.T) { name: "nominal-case", specConf: kubevirtProviderSpecConf{}, }, + { + name: "extra-headers-set", + specConf: kubevirtProviderSpecConf{ + ExtraHeadersSet: true, + }, + }, { name: "instancetype-preference-standard", specConf: kubevirtProviderSpecConf{ diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/extra-headers-set.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/extra-headers-set.yaml new file mode 100644 index 000000000..becb92f5f --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/extra-headers-set.yaml @@ -0,0 +1,82 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: extra-headers-set + md: md-name + name: extra-headers-set + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: extra-headers-set + spec: + storage: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + extraHeaders: + - 'authorization: Basic bXE6cGFzc3dvcmQ=' + runStrategy: Once + template: + metadata: + creationTimestamp: null + annotations: + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + "ovn.kubernetes.io/allow_live_migration": "true" + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: extra-headers-set + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - name: default + bridge: {} + networkInterfaceMultiqueue: true + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: extra-headers-set + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index 4cac85ec0..b958150dc 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -78,6 +78,12 @@ type Template struct { // PrimaryDisk. type PrimaryDisk struct { Disk + // ExtraHeaders is a list of strings containing extra headers to include with HTTP transfer requests + // +optional + ExtraHeaders []string `json:"extraHeaders,omitempty"` + // ExtraHeadersSecretRef is a secret that contains a list of strings containing extra headers to include with HTTP transfer requests + // +optional + ExtraHeadersSecretRef providerconfigtypes.ConfigVarString `json:"extraHeadersSecretRef,omitempty"` // StorageTarget describes which VirtualMachine storage target will be used in the DataVolumeTemplate. StorageTarget providerconfigtypes.ConfigVarString `json:"storageTarget,omitempty"` // OsImage describes the OS that will be installed on the VirtualMachine. From 63ed9f028d32e9ceffbb3cb690200e04e65075b7 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Wed, 20 Nov 2024 08:22:05 +0100 Subject: [PATCH 446/489] refactor ubuntu arm image in aws (#1880) Signed-off-by: moadqassem --- pkg/cloudprovider/provider/aws/provider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 4dba90c62..34b51612f 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -138,7 +138,7 @@ var ( }, awstypes.CPUArchitectureARM64: { // Be as precise as possible - otherwise we might get a nightly dev build - description: "Canonical, Ubuntu, 24.04 LTS, arm64 noble image build on 2024-05-29 ????-??-??", + description: "Canonical, Ubuntu, 24.04, arm64 noble image", // The AWS marketplace ID from Canonical owner: "099720109477", }, From be6a6aaf2fefc2880805263d861f67cc53782ac1 Mon Sep 17 00:00:00 2001 From: Kubermatic Bot <41968677+kubermatic-bot@users.noreply.github.com> Date: Fri, 29 Nov 2024 17:45:45 +0100 Subject: [PATCH 447/489] Synchronize OWNERS_ALIASES file with Github teams (#1881) --- OWNERS_ALIASES | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index d604ea1c1..14142c01a 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -6,6 +6,7 @@ aliases: - ahmedwaleedmalik - cnvergence - embik + - julioc-p - kron4eg - moadqassem - moelsayed From 1e563a29d4e7d2a0138071b1a5b0d30fd58cf4e7 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Fri, 6 Dec 2024 16:03:44 +0100 Subject: [PATCH 448/489] Support KubeVirt CDI Datavolume secret ref (#1885) Signed-off-by: moadqassem --- pkg/cloudprovider/provider/kubevirt/provider.go | 11 ++++++++--- pkg/cloudprovider/provider/kubevirt/types/types.go | 2 ++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 12db060aa..8f1e35d98 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -116,6 +116,7 @@ type Config struct { EnableNetworkMultiQueue bool ExtraHeaders []string ExtraHeadersSecretRef string + DataVolumeSecretRef string ProviderNetworkName string SubnetName string @@ -284,6 +285,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if len(rawConfig.VirtualMachine.Template.PrimaryDisk.ExtraHeaders) > 0 { config.ExtraHeaders = rawConfig.VirtualMachine.Template.PrimaryDisk.ExtraHeaders } + dataVolumeSecretRef, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.DataVolumeSecretRef) + if err != nil { + return nil, nil, fmt.Errorf(`failed to get value of "dataVolumeSecretRef" field: %w`, err) + } + config.DataVolumeSecretRef = dataVolumeSecretRef extraHeadersSecretRef, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.ExtraHeadersSecretRef) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "extraHeadersSecretRef" field: %w`, err) @@ -457,8 +463,7 @@ func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, con if err != nil { return nil, fmt.Errorf(`failed to get value of "primaryDisk.extraHeaders" field: %w`, err) } - - return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage, ExtraHeaders: extraHeaders}}, nil + return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage, ExtraHeaders: extraHeaders, SecretRef: config.DataVolumeSecretRef}}, nil case registrySource: return registryDataVolume(osImage, pullMethod), nil case pvcSource: @@ -478,7 +483,7 @@ func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, con return nil, fmt.Errorf(`failed to get value of "primaryDisk.extraHeaders" field: %w`, err) } - return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage, ExtraHeaders: extraHeaders}}, nil + return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage, ExtraHeaders: extraHeaders, SecretRef: config.DataVolumeSecretRef}}, nil } if namespaceAndName := strings.Split(osImage, "/"); len(namespaceAndName) >= 2 { return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: namespaceAndName[1], Namespace: namespaceAndName[0]}}, nil diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index b958150dc..e86c8c4e8 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -78,6 +78,8 @@ type Template struct { // PrimaryDisk. type PrimaryDisk struct { Disk + // DataVolumeSecretRef is the name of the secret that will be sent to the CDI data importer pod to read basic auth parameters. + DataVolumeSecretRef providerconfigtypes.ConfigVarString `json:"dataVolumeSecretRef,omitempty"` // ExtraHeaders is a list of strings containing extra headers to include with HTTP transfer requests // +optional ExtraHeaders []string `json:"extraHeaders,omitempty"` From d269a83954102b895ab4823edb75d1fdf10bfd63 Mon Sep 17 00:00:00 2001 From: Mohamed Rafraf <81432497+mohamed-rafraf@users.noreply.github.com> Date: Thu, 2 Jan 2025 14:36:37 +0100 Subject: [PATCH 449/489] fix decode with tink-kubeconfig envar (#1888) --- .../provider/baremetal/plugins/tinkerbell/driver.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go index 3461497b2..575063852 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go @@ -185,7 +185,14 @@ func GetConfig(driverConfig tinktypes.TinkerbellPluginSpec, valueFromStringOrEnv if err != nil { return nil, fmt.Errorf(`failed to get value of "kubeconfig" field: %w`, err) } + val, err := base64.StdEncoding.DecodeString(config.Kubeconfig) + // We intentionally ignore errors here with an assumption that an unencoded YAML or JSON must have been passed on + // in this case. + if err == nil { + config.Kubeconfig = string(val) + } } + config.ClusterName, err = valueFromStringOrEnvVar(driverConfig.ClusterName, "CLUSTER_NAME") if err != nil { return nil, fmt.Errorf(`failed to get value of "clusterName" field: %w`, err) From 049efb4f3fb39ee7dcfd9a9790be4a03cc680eb9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Jan 2025 15:09:36 +0100 Subject: [PATCH 450/489] Bump golang.org/x/crypto from 0.27.0 to 0.31.0 (#1887) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.27.0 to 0.31.0. - [Commits](https://github.com/golang/crypto/compare/v0.27.0...v0.31.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index af6039be7..286da3126 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/vultr/govultr/v3 v3.9.1 go.anx.io/go-anxcloud v0.7.3 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.27.0 + golang.org/x/crypto v0.31.0 golang.org/x/oauth2 v0.23.0 gomodules.xyz/jsonpatch/v2 v2.4.0 google.golang.org/api v0.197.0 @@ -164,10 +164,10 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect golang.org/x/net v0.29.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/term v0.24.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.25.0 // indirect google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect diff --git a/go.sum b/go.sum index 8f363eb9e..63c5a8375 100644 --- a/go.sum +++ b/go.sum @@ -493,8 +493,8 @@ golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -564,8 +564,8 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -599,8 +599,8 @@ golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -611,8 +611,8 @@ golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= -golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= -golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -625,8 +625,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From fadb8b7491bb97142c19db9b4619e042db5ea5bc Mon Sep 17 00:00:00 2001 From: Mohamed Rafraf <81432497+mohamed-rafraf@users.noreply.github.com> Date: Fri, 3 Jan 2025 09:16:38 +0100 Subject: [PATCH 451/489] Add partition number annotation (#1890) * add partition number annotation * fix lint --- .../plugins/tinkerbell/client/template.go | 11 ++++++----- .../plugins/tinkerbell/client/workflow.go | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go index c5baf8ad1..aa82435d9 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go @@ -60,6 +60,7 @@ const ( hardwareDisk1 = "{{ index .Hardware.Disks 0 }}" hardwareName = "{{.hardware_name}}" ProvisionWorkerNodeTemplate = "provision-worker-node" + PartitionNumber = "{{.partition_number}}" ) // TemplateClient handles interactions with the Tinkerbell Templates in the Tinkerbell cluster. @@ -195,11 +196,11 @@ func createGrowPartitionAction(destDisk string) Action { Image: "quay.io/tinkerbell/actions/cexec:c5bde803d9f6c90f1a9d5e06930d856d1481854c", Timeout: 90, Environment: map[string]string{ - "BLOCK_DEVICE": "{{ index .Hardware.Disks 0 }}3", + "BLOCK_DEVICE": fmt.Sprintf("{{ index .Hardware.Disks 0 }}%s", PartitionNumber), "FS_TYPE": fsType, "CHROOT": "y", "DEFAULT_INTERPRETER": defaultInterpreter, - "CMD_LINE": fmt.Sprintf("growpart %s 3 && resize2fs %s3", destDisk, destDisk), + "CMD_LINE": fmt.Sprintf("growpart %s %s && resize2fs %s%s", destDisk, PartitionNumber, destDisk, PartitionNumber), }, } } @@ -225,7 +226,7 @@ network: Image: "quay.io/tinkerbell-actions/writefile:v1.0.0", Timeout: 90, Environment: map[string]string{ - "DEST_DISK": "{{ index .Hardware.Disks 0 }}3", + "DEST_DISK": fmt.Sprintf("{{ index .Hardware.Disks 0 }}%s", PartitionNumber), "FS_TYPE": fsType, "DEST_PATH": "/etc/netplan/config.yaml", "CONTENTS": netplaneConfig, @@ -250,7 +251,7 @@ echo 'local-hostname: {{.hardware_name}}' >> /var/lib/cloud/seed/nocloud/meta-da Image: "quay.io/tinkerbell-actions/cexec:v1.0.0", Timeout: 90, Environment: map[string]string{ - "BLOCK_DEVICE": "{{ index .Hardware.Disks 0 }}3", + "BLOCK_DEVICE": fmt.Sprintf("{{ index .Hardware.Disks 0 }}%s", PartitionNumber), "FS_TYPE": fsType, "CHROOT": "y", "DEFAULT_INTERPRETER": defaultInterpreter, @@ -265,7 +266,7 @@ func decodeCloudInitFile(hardwareName string) Action { Image: "quay.io/tinkerbell/actions/cexec:latest", Timeout: 90, Environment: map[string]string{ - "BLOCK_DEVICE": "{{ index .Hardware.Disks 0 }}3", + "BLOCK_DEVICE": fmt.Sprintf("{{ index .Hardware.Disks 0 }}%s", PartitionNumber), "FS_TYPE": fsType, "CHROOT": "y", "DEFAULT_INTERPRETER": "/bin/sh -c", diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go index da3d4bd3c..ec76933c0 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go @@ -31,6 +31,12 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +// DefaultPartitionNumber defines the default value for the "partition_number" field. +const DefaultPartitionNumber = "3" + +// PartitionNumberAnnotation is used to specify the main partition number of the disk device. +const PartitionNumberAnnotation = "hardware.kubermatic.io/partition-number" + // WorkflowClient handles interactions with the Tinkerbell Workflows. type WorkflowClient struct { tinkclient client.Client @@ -73,6 +79,7 @@ func (w *WorkflowClient) CreateWorkflow(ctx context.Context, userData, templateR "cidr": convertNetmaskToCIDR(ifaceConfig.IP), "ns": dnsNameservers, "default_route": ifaceConfig.IP.Gateway, + "partition_number": w.getPartitionNumber(hardware), }, }, } @@ -124,3 +131,11 @@ func (w *WorkflowClient) CleanupWorkflows(ctx context.Context, hardwareName, nam return nil } + +func (w *WorkflowClient) getPartitionNumber(hardware tink.Hardware) string { + partitionNumber, exists := hardware.Annotations[PartitionNumberAnnotation] + if !exists { + partitionNumber = DefaultPartitionNumber // Use the default value + } + return partitionNumber +} From 083c6d54933149f71c45725edeabf8784c0ba977 Mon Sep 17 00:00:00 2001 From: Kubermatic Bot <41968677+kubermatic-bot@users.noreply.github.com> Date: Sat, 11 Jan 2025 11:59:51 +0100 Subject: [PATCH 452/489] Synchronize OWNERS_ALIASES file with Github teams (#1894) --- OWNERS_ALIASES | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 14142c01a..93b0fc0ac 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -10,6 +10,7 @@ aliases: - kron4eg - moadqassem - moelsayed + - soer3n - xmudrii - xrstf - yaa110 From d6ee7f43eac2aec6e64d32136b0bfb7ee8354f22 Mon Sep 17 00:00:00 2001 From: Mohamed Rafraf <81432497+mohamed-rafraf@users.noreply.github.com> Date: Mon, 13 Jan 2025 11:57:54 +0100 Subject: [PATCH 453/489] baremetal: Templating OSImage links through Tinkerbell workflow objects (#1891) * templating the OSImageURL * update --- .../provider/baremetal/plugins/tinkerbell/client/template.go | 5 +++-- .../provider/baremetal/plugins/tinkerbell/client/workflow.go | 3 ++- .../provider/baremetal/plugins/tinkerbell/driver.go | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go index aa82435d9..7ea237ea6 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go @@ -61,6 +61,7 @@ const ( hardwareName = "{{.hardware_name}}" ProvisionWorkerNodeTemplate = "provision-worker-node" PartitionNumber = "{{.partition_number}}" + OSImageURL = "{{.os_image}}" ) // TemplateClient handles interactions with the Tinkerbell Templates in the Tinkerbell cluster. @@ -91,14 +92,14 @@ func (t *TemplateClient) Delete(ctx context.Context, namespacedName types.Namesp } // CreateTemplate creates a Tinkerbell Template in the Kubernetes cluster. -func (t *TemplateClient) CreateTemplate(ctx context.Context, namespace, osImageURL string) error { +func (t *TemplateClient) CreateTemplate(ctx context.Context, namespace string) error { template := &tinkv1alpha1.Template{} if err := t.tinkclient.Get(ctx, types.NamespacedName{ Name: ProvisionWorkerNodeTemplate, Namespace: namespace, }, template); err != nil { if kerrors.IsNotFound(err) { - data, err := getTemplate(osImageURL) + data, err := getTemplate(OSImageURL) if err != nil { return err } diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go index ec76933c0..53f8b9d9a 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go @@ -50,7 +50,7 @@ func NewWorkflowClient(k8sClient client.Client) *WorkflowClient { } // CreateWorkflow creates a new Tinkerbell Workflow resource in the cluster. -func (w *WorkflowClient) CreateWorkflow(ctx context.Context, userData, templateRef string, hardware tink.Hardware) error { +func (w *WorkflowClient) CreateWorkflow(ctx context.Context, userData, templateRef, osImageURL string, hardware tink.Hardware) error { // Construct the Workflow object ifaceConfig := hardware.Spec.Interfaces[0].DHCP dnsNameservers := "1.1.1.1" @@ -80,6 +80,7 @@ func (w *WorkflowClient) CreateWorkflow(ctx context.Context, userData, templateR "ns": dnsNameservers, "default_route": ifaceConfig.IP.Gateway, "partition_number": w.getPartitionNumber(hardware), + "os_image": osImageURL, }, }, } diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go index 575063852..4b1e162a4 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go @@ -123,14 +123,14 @@ func (d *driver) ProvisionServer(ctx context.Context, _ *zap.SugaredLogger, meta } // Create template if it doesn't exist - err = d.TemplateClient.CreateTemplate(ctx, d.HardwareRef.Namespace, d.OSImageURL) + err = d.TemplateClient.CreateTemplate(ctx, d.HardwareRef.Namespace) if err != nil { return nil, err } // Create Workflow to match the template and server server := tinktypes.Hardware{Hardware: hardware} - if err = d.WorkflowClient.CreateWorkflow(ctx, userdata, client.ProvisionWorkerNodeTemplate, server); err != nil { + if err = d.WorkflowClient.CreateWorkflow(ctx, userdata, client.ProvisionWorkerNodeTemplate, d.OSImageURL, server); err != nil { return nil, err } From cb0630860ba78d1d20b38d4327c8f39751ab4c63 Mon Sep 17 00:00:00 2001 From: Sahil Raja Date: Wed, 22 Jan 2025 13:38:57 +0530 Subject: [PATCH 454/489] feat(metrics): add machine deployment metrics (#1879) * feat(metrics): add machine deployment metrics Signed-off-by: rajaSahil * feat(metrics): fix lint Signed-off-by: rajaSahil --------- Signed-off-by: rajaSahil --- cmd/machine-controller/main.go | 3 + .../provider/anexia/types/types.go | 7 -- pkg/controller/machinedeployment/metrics.go | 112 ++++++++++++++++++ 3 files changed, 115 insertions(+), 7 deletions(-) create mode 100644 pkg/controller/machinedeployment/metrics.go diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 6e1d25a90..6bf69cd38 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -393,6 +393,9 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { machineCollector := machinecontroller.NewMachineCollector(ctx, bs.mgr.GetClient()) metrics.Registry.MustRegister(machineCollector) + machineDeploymentCollector := machinedeploymentcontroller.NewCollector(ctx, bs.mgr.GetClient()) + metrics.Registry.MustRegister(machineDeploymentCollector) + if err := machinecontroller.Add( ctx, bs.opt.log, diff --git a/pkg/cloudprovider/provider/anexia/types/types.go b/pkg/cloudprovider/provider/anexia/types/types.go index 4a6788bc8..9c01eabf3 100644 --- a/pkg/cloudprovider/provider/anexia/types/types.go +++ b/pkg/cloudprovider/provider/anexia/types/types.go @@ -19,8 +19,6 @@ package types import ( "time" - "k8c.io/machine-controller/pkg/apis/cluster/common" - cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/jsonutil" providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" @@ -42,11 +40,6 @@ const ( MachinePoweredOn = "poweredOn" ) -var StatusUpdateFailed = cloudprovidererrors.TerminalError{ - Reason: common.UpdateMachineError, - Message: "Failed to update the machine status", -} - // RawDisk specifies a single disk, with some values maybe being fetched from secrets. type RawDisk struct { Size int `json:"size"` diff --git a/pkg/controller/machinedeployment/metrics.go b/pkg/controller/machinedeployment/metrics.go new file mode 100644 index 000000000..767ea4d89 --- /dev/null +++ b/pkg/controller/machinedeployment/metrics.go @@ -0,0 +1,112 @@ +/* +Copyright 2025 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package machinedeployment + +import ( + "context" + + "github.com/prometheus/client_golang/prometheus" + "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +const metricsPrefix = "machine_deployment_" + +type Collector struct { + ctx context.Context + client ctrlruntimeclient.Client + + replicas *prometheus.Desc + availableReplicas *prometheus.Desc + readyReplicas *prometheus.Desc + updatedReplicas *prometheus.Desc +} + +// NewCollector creates new machine deployment collector for metrics collection. +func NewCollector(ctx context.Context, client ctrlruntimeclient.Client) *Collector { + return &Collector{ + ctx: ctx, + client: client, + replicas: prometheus.NewDesc( + metricsPrefix+"replicas", + "The number of replicas defined for a machine deployment", + []string{"name", "namespace"}, nil, + ), + availableReplicas: prometheus.NewDesc( + metricsPrefix+"available_replicas", + "The number of available replicas for a machine deployment", + []string{"name", "namespace"}, nil, + ), + readyReplicas: prometheus.NewDesc( + metricsPrefix+"ready_replicas", + "The number of ready replicas for a machine deployment", + []string{"name", "namespace"}, nil, + ), + updatedReplicas: prometheus.NewDesc( + metricsPrefix+"updated_replicas", + "The number of replicas updated for a machine deployment", + []string{"name", "namespace"}, nil, + ), + } +} + +// Describe implements the prometheus.Describe interface. +func (c *Collector) Describe(desc chan<- *prometheus.Desc) { + desc <- c.replicas + desc <- c.readyReplicas + desc <- c.availableReplicas + desc <- c.readyReplicas +} + +// Collect implements the prometheus.Collector interface. +func (c *Collector) Collect(metrics chan<- prometheus.Metric) { + machineDeployments := &v1alpha1.MachineDeploymentList{} + if err := c.client.List(c.ctx, machineDeployments); err != nil { + return + } + + for _, machineDeployment := range machineDeployments.Items { + metrics <- prometheus.MustNewConstMetric( + c.replicas, + prometheus.GaugeValue, + float64(machineDeployment.Status.Replicas), + machineDeployment.Name, + machineDeployment.Namespace, + ) + metrics <- prometheus.MustNewConstMetric( + c.readyReplicas, + prometheus.GaugeValue, + float64(machineDeployment.Status.ReadyReplicas), + machineDeployment.Name, + machineDeployment.Namespace, + ) + metrics <- prometheus.MustNewConstMetric( + c.availableReplicas, + prometheus.GaugeValue, + float64(machineDeployment.Status.AvailableReplicas), + machineDeployment.Name, + machineDeployment.Namespace, + ) + metrics <- prometheus.MustNewConstMetric( + c.updatedReplicas, + prometheus.GaugeValue, + float64(machineDeployment.Status.UpdatedReplicas), + machineDeployment.Name, + machineDeployment.Namespace, + ) + } +} From e0c29cda724313f0d06227d3e97bdd1c7c480c19 Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 28 Jan 2025 09:31:01 +0100 Subject: [PATCH 455/489] Support kubevirt evection strategy in kv provider (#1897) * support evictStrategy in VMs Signed-off-by: moadqassem Signed-off-by: moadqassem * add tests Signed-off-by: moadqassem Signed-off-by: moadqassem --------- Signed-off-by: moadqassem --- .../provider/kubevirt/provider.go | 15 ++++ .../provider/kubevirt/provider_test.go | 8 ++ .../eviction-strategy-live-migrate.yaml | 80 +++++++++++++++++++ .../provider/kubevirt/types/types.go | 1 + 4 files changed, 104 insertions(+) create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/eviction-strategy-live-migrate.yaml diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 8f1e35d98..dbcb0dc8c 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -117,6 +117,7 @@ type Config struct { ExtraHeaders []string ExtraHeadersSecretRef string DataVolumeSecretRef string + EvictionStrategy kubevirtv1.EvictionStrategy ProviderNetworkName string SubnetName string @@ -366,6 +367,10 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } } + if rawConfig.VirtualMachine.EvictionStrategy != "" { + config.EvictionStrategy = kubevirtv1.EvictionStrategy(rawConfig.VirtualMachine.EvictionStrategy) + } + return &config, pconfig, nil } @@ -635,6 +640,13 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus return fmt.Errorf("failed to request VirtualMachineInstances: %w", err) } + if c.EvictionStrategy != "" { + if c.EvictionStrategy != kubevirtv1.EvictionStrategyExternal && + c.EvictionStrategy != kubevirtv1.EvictionStrategyLiveMigrate { + return fmt.Errorf("unsupported vm eviction strategy: %s", c.EvictionStrategy) + } + } + return nil } @@ -729,6 +741,9 @@ func (p *provider) newVirtualMachine(c *Config, pc *providerconfigtypes.Config, terminationGracePeriodSeconds := int64(30) evictionStrategy := kubevirtv1.EvictionStrategyExternal + if c.EvictionStrategy != "" { + evictionStrategy = c.EvictionStrategy + } resourceRequirements := kubevirtv1.ResourceRequirements{} labels["kubevirt.io/vm"] = machine.Name diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index 49b5b1a5d..4651889d5 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -71,6 +71,7 @@ type kubevirtProviderSpecConf struct { PullMethod cdiv1beta1.RegistryPullMethod ProviderNetwork *types.ProviderNetwork ExtraHeadersSet bool + EvictStrategy string } func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { @@ -104,6 +105,9 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { }, {{- end }} "virtualMachine": { + {{- if .EvictStrategy }} + "evictionStrategy": "LiveMigrate", + {{- end }} {{- if .ProviderNetwork }} "providerNetwork": { "name": "kubeovn", @@ -275,6 +279,10 @@ func TestNewVirtualMachine(t *testing.T) { name: "pvc-image-source", specConf: kubevirtProviderSpecConf{OsImageSource: pvcSource, OsImageDV: "ns/dvname"}, }, + { + name: "eviction-strategy-live-migrate", + specConf: kubevirtProviderSpecConf{EvictStrategy: "LiveMigrate"}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/eviction-strategy-live-migrate.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/eviction-strategy-live-migrate.yaml new file mode 100644 index 000000000..709d0f4ac --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/eviction-strategy-live-migrate.yaml @@ -0,0 +1,80 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: eviction-strategy-live-migrate + md: md-name + name: eviction-strategy-live-migrate + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: eviction-strategy-live-migrate + spec: + storage: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + runStrategy: Once + template: + metadata: + creationTimestamp: null + annotations: + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + "ovn.kubernetes.io/allow_live_migration": "true" + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: eviction-strategy-live-migrate + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - name: default + bridge: {} + networkInterfaceMultiqueue: true + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: eviction-strategy-live-migrate + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: LiveMigrate diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index e86c8c4e8..79e7100fc 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -59,6 +59,7 @@ type VirtualMachine struct { Location *Location `json:"location,omitempty"` ProviderNetwork *ProviderNetwork `json:"providerNetwork,omitempty"` EnableNetworkMultiQueue providerconfigtypes.ConfigVarBool `json:"enableNetworkMultiQueue,omitempty"` + EvictionStrategy string `json:"evictionStrategy,omitempty"` } // Flavor. From e9ea007b76e2afef2cf29b531029acaa4709afef Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Mon, 10 Feb 2025 22:36:15 +0100 Subject: [PATCH 456/489] add new external provider (#1898) Signed-off-by: moadqassem --- pkg/cloudprovider/provider.go | 4 + .../provider/external/provider.go | 95 +++++++++++++++++++ pkg/controller/machine/controller.go | 10 +- pkg/providerconfig/types/types.go | 1 + 4 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 pkg/cloudprovider/provider/external/provider.go diff --git a/pkg/cloudprovider/provider.go b/pkg/cloudprovider/provider.go index 23ea1671b..032f8c775 100644 --- a/pkg/cloudprovider/provider.go +++ b/pkg/cloudprovider/provider.go @@ -28,6 +28,7 @@ import ( "k8c.io/machine-controller/pkg/cloudprovider/provider/digitalocean" "k8c.io/machine-controller/pkg/cloudprovider/provider/edge" "k8c.io/machine-controller/pkg/cloudprovider/provider/equinixmetal" + "k8c.io/machine-controller/pkg/cloudprovider/provider/external" "k8c.io/machine-controller/pkg/cloudprovider/provider/fake" "k8c.io/machine-controller/pkg/cloudprovider/provider/gce" "k8c.io/machine-controller/pkg/cloudprovider/provider/hetzner" @@ -111,6 +112,9 @@ var ( providerconfigtypes.CloudProviderVMwareCloudDirector: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return vcd.New(cvr) }, + providerconfigtypes.CloudProviderExternal: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return external.New(cvr) + }, } // communityProviders holds a map of cloud providers that have been implemented by community members and diff --git a/pkg/cloudprovider/provider/external/provider.go b/pkg/cloudprovider/provider/external/provider.go new file mode 100644 index 000000000..73d595814 --- /dev/null +++ b/pkg/cloudprovider/provider/external/provider.go @@ -0,0 +1,95 @@ +/* +Copyright 2025 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package external + +import ( + "context" + "go.uber.org/zap" + + clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/instance" + cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + "k8c.io/machine-controller/pkg/providerconfig" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" +) + +type provider struct{} + +type CloudProviderSpec struct { +} + +type CloudProviderInstance struct{} + +func (f CloudProviderInstance) Name() string { + return "" +} + +func (f CloudProviderInstance) ID() string { + return "" +} + +func (f CloudProviderInstance) ProviderID() string { + return "" +} + +func (f CloudProviderInstance) Addresses() map[string]corev1.NodeAddressType { + return nil +} + +func (f CloudProviderInstance) Status() instance.Status { + return instance.StatusUnknown +} + +// New returns an external cloud provider. +func New(_ *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return &provider{} +} + +func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { + return spec, nil +} + +// Validate returns success or failure based according to its ExternalCloudProviderSpec. +func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, _ clusterv1alpha1.MachineSpec) error { + return nil +} + +func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { + return CloudProviderInstance{}, nil +} + +// Create creates a cloud instance according to the given machine. +func (p *provider) Create(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData, _ string) (instance.Instance, error) { + return CloudProviderInstance{}, nil +} + +func (p *provider) Cleanup(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { + return true, nil +} + +func (p *provider) MigrateUID(_ context.Context, _ *zap.SugaredLogger, _ *clusterv1alpha1.Machine, _ types.UID) error { + return nil +} + +func (p *provider) MachineMetricsLabels(_ *clusterv1alpha1.Machine) (map[string]string, error) { + return map[string]string{}, nil +} + +func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { + return nil +} diff --git a/pkg/controller/machine/controller.go b/pkg/controller/machine/controller.go index 87ed6849a..ea9addf71 100644 --- a/pkg/controller/machine/controller.go +++ b/pkg/controller/machine/controller.go @@ -751,8 +751,14 @@ func (r *Reconciler) deleteCloudProviderInstance(ctx context.Context, log *zap.S return nil, r.updateMachine(machine, func(m *clusterv1alpha1.Machine) { finalizers := sets.NewString(m.Finalizers...) - finalizers.Delete(FinalizerDeleteInstance) - m.Finalizers = finalizers.List() + // If a machine deployment belongs to an external cloud provider, the 'machine-delete-finalizer' must be manually + // removed by an administrator or an external service. This is because the machine controller lacks access to cloud + // instances and cannot ensure their deletion. If the external service fails to delete the instance, it may result + // in orphaned resources or nodes without a machine reference. + if machineConfig.CloudProvider != providerconfigtypes.CloudProviderExternal { + finalizers.Delete(FinalizerDeleteInstance) + m.Finalizers = finalizers.List() + } }) } diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index 5fe41f0d5..b733f0476 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -113,6 +113,7 @@ var ( CloudProviderBaremetal, CloudProviderVultr, CloudProviderOpenNebula, + CloudProviderExternal, } ) From 6243011f708b34fb9793fcf0805d7e9344ff08f8 Mon Sep 17 00:00:00 2001 From: julioc-p <32308064+julioc-p@users.noreply.github.com> Date: Fri, 28 Feb 2025 16:47:12 +0100 Subject: [PATCH 457/489] Change Ubuntu template for Vsphere e2e tests (#1899) * changing ubuntu template for vsphere tests Signed-off-by: julioc-p <14-10820@usb.ve> * chaging template to ubuntu 24.04 Signed-off-by: julioc-p <14-10820@usb.ve> --------- Signed-off-by: julioc-p <14-10820@usb.ve> --- test/e2e/provisioning/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index f2f8d52a9..00f0b1b03 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -63,7 +63,7 @@ var ( string(providerconfigtypes.OperatingSystemFlatcar): "kkp-flatcar-3139.2.0", string(providerconfigtypes.OperatingSystemRHEL): "kkp-rhel-8.6", string(providerconfigtypes.OperatingSystemRockyLinux): "kkp-rockylinux-8", - string(providerconfigtypes.OperatingSystemUbuntu): "kkp-ubuntu-22.04", + string(providerconfigtypes.OperatingSystemUbuntu): "kkp-ubuntu-24.04", } kubevirtImages = map[string]string{ From 91d2aec2c8cabe3ba05940a1a60d0015bba697b4 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Fri, 28 Feb 2025 18:03:12 +0100 Subject: [PATCH 458/489] Move APIs and cloud provider types into standalone Go module in ./sdk (#1900) * it compiles * fix/harmonize/lint imports across the project * move configvarresolver into SDK * add license to SDK to make wwhrd work * update boilerplate check * move skip-eviction annotation into the sdk * move anexia errors into SDK * move baremetal plugin types to sdk * move bootstrap types into SDK * move testhelper into SDK, improve linting * use the same API registering method as for machines * better filenames --- .golangci.yml | 39 +- Makefile | 7 +- cmd/machine-controller/main.go | 6 +- go.mod | 5 +- hack/verify-boilerplate.sh | 3 +- pkg/admission/machinedeployments.go | 2 +- pkg/admission/machinedeployments_test.go | 2 +- .../machinedeployments_validation.go | 20 +- pkg/admission/machines.go | 14 +- pkg/admission/util.go | 2 +- pkg/cloudprovider/cache/cloudprovidercache.go | 2 +- .../cache/cloudprovidercache_test.go | 2 +- pkg/cloudprovider/errors/errors.go | 2 +- pkg/cloudprovider/instance/instance.go | 4 +- pkg/cloudprovider/provider.go | 53 ++- .../provider/alibaba/provider.go | 29 +- .../provider/alibaba/types/types.go | 42 -- .../provider/anexia/helper_test.go | 24 +- pkg/cloudprovider/provider/anexia/instance.go | 18 +- .../provider/anexia/instance_test.go | 46 +-- .../provider/anexia/network_provisioning.go | 4 +- pkg/cloudprovider/provider/anexia/provider.go | 49 +-- .../provider/anexia/provider_test.go | 43 +- .../provider/anexia/reconcile_context.go | 8 +- .../provider/anexia/resolve_config.go | 15 +- .../provider/anexia/types/errors.go | 49 --- .../provider/aws}/gzip.go | 4 +- pkg/cloudprovider/provider/aws/provider.go | 76 ++-- pkg/cloudprovider/provider/aws/types/types.go | 69 ---- .../provider/azure/create_delete_resources.go | 4 +- pkg/cloudprovider/provider/azure/provider.go | 73 ++-- .../provider/azure/types/types.go | 77 ---- .../provider/baremetal/plugins/driver.go | 12 +- .../plugins/tinkerbell/client/hardware.go | 11 +- .../plugins/tinkerbell/client/template.go | 10 +- .../plugins/tinkerbell/client/workflow.go | 19 +- .../baremetal/plugins/tinkerbell/driver.go | 16 +- .../plugins/tinkerbell/types/hardware.go | 10 + .../provider/baremetal/provider.go | 26 +- .../provider/digitalocean/provider.go | 47 ++- .../provider/digitalocean/types/types.go | 39 -- pkg/cloudprovider/provider/edge/provider.go | 4 +- .../provider/equinixmetal/provider.go | 25 +- .../provider/external/provider.go | 6 +- pkg/cloudprovider/provider/fake/provider.go | 7 +- pkg/cloudprovider/provider/gce/config.go | 31 +- pkg/cloudprovider/provider/gce/instance.go | 18 +- pkg/cloudprovider/provider/gce/provider.go | 18 +- .../provider/gce/provider_test.go | 18 +- pkg/cloudprovider/provider/gce/types/types.go | 86 ---- .../provider/hetzner/provider.go | 35 +- .../provider/hetzner/types/types.go | 42 -- .../provider/kubevirt/provider.go | 197 +++++---- .../provider/kubevirt/provider_test.go | 40 +- .../provider/kubevirt/types/types.go | 167 -------- pkg/cloudprovider/provider/linode/provider.go | 29 +- pkg/cloudprovider/provider/nutanix/client.go | 4 +- .../provider/nutanix/provider.go | 13 +- .../provider/opennebula/provider.go | 21 +- .../provider/opennebula/types/types.go | 46 --- .../provider/openstack/provider.go | 15 +- .../provider/openstack/provider_test.go | 6 +- .../provider/openstack/types/types.go | 64 --- .../provider/scaleway/provider.go | 19 +- .../provider/vmwareclouddirector/helper.go | 27 +- .../provider/vmwareclouddirector/provider.go | 15 +- .../provider/vsphere/provider.go | 19 +- .../provider/vsphere/provider_test.go | 2 +- pkg/cloudprovider/provider/vsphere/rule.go | 2 +- .../provider/vsphere/types/types.go | 69 ---- pkg/cloudprovider/provider/vsphere/vmgroup.go | 4 +- pkg/cloudprovider/provider/vultr/provider.go | 37 +- .../provider/vultr/types/types.go | 42 -- pkg/cloudprovider/testing/testing.go | 10 +- pkg/cloudprovider/types/types.go | 2 +- pkg/cloudprovider/util/cloud_init_settings.go | 4 +- pkg/cloudprovider/util/net.go | 34 -- pkg/cloudprovider/util/util.go | 6 +- pkg/cloudprovider/util/util_test.go | 15 +- pkg/cloudprovider/validationwrapper.go | 18 +- pkg/controller/machine/controller.go | 63 ++- pkg/controller/machine/controller_test.go | 13 +- pkg/controller/machine/metrics.go | 15 +- .../machinedeployment/controller.go | 50 +-- pkg/controller/machinedeployment/metrics.go | 6 +- pkg/controller/machinedeployment/rolling.go | 38 +- pkg/controller/machinedeployment/sync.go | 16 +- pkg/controller/machineset/controller.go | 24 +- pkg/controller/machineset/delete_policy.go | 26 +- pkg/controller/machineset/machine.go | 14 +- pkg/controller/machineset/status.go | 16 +- pkg/controller/nodecsrapprover/controller.go | 20 +- .../nodecsrapprover/controller_test.go | 16 +- pkg/controller/util/machine.go | 6 +- pkg/controller/util/machine_deployment.go | 94 ++--- .../v1alpha1 => }/migrations/migrations.go | 23 +- pkg/node/eviction/eviction.go | 16 +- pkg/node/poddeletion/pod_deletion.go | 8 +- pkg/rhsm/util.go | 10 +- sdk/.golangci.yml | 39 ++ sdk/LICENSE | 201 +++++++++ sdk/Makefile | 18 + {pkg => sdk}/apis/cluster/common/consts.go | 0 {pkg => sdk}/apis/cluster/common/plugins.go | 4 +- .../apis/cluster/v1alpha1/common_types.go | 0 .../v1alpha1/conversions/conversions.go | 8 +- .../v1alpha1/conversions/conversions_test.go | 4 +- .../providerconfig_to_providerspec.go | 2 +- .../providerconfig_to_providerspec_test.go | 2 +- .../hetzner.yaml | 0 .../hetzner.yaml | 0 .../aws.yaml | 0 .../hetzner.yaml | 0 .../testdata/machinesv1alpha1machine/aws.yaml | 0 .../machinesv1alpha1machine/azure.yaml | 0 .../machinesv1alpha1machine/digitalocean.yaml | 0 .../machinesv1alpha1machine/hetzner.yaml | 0 .../machinesv1alpha1machine/linode.yaml | 0 .../machinesv1alpha1machine/openstack.yaml | 0 .../vsphere-static-ip.yaml | 0 .../machinesv1alpha1machine/vsphere.yaml | 0 .../migrated_clusterv1alpha1machine/aws.yaml | 0 .../azure.yaml | 0 .../digitalocean.yaml | 0 .../hetzner.yaml | 0 .../linode.yaml | 0 .../openstack.yaml | 0 .../vsphere-static-ip.yaml | 0 .../vsphere.yaml | 0 .../hetzner.yaml | 0 .../hetzner.yaml | 0 .../aws.yaml | 0 .../hetzner.yaml | 0 .../apis/cluster/v1alpha1/defaults.go | 2 +- {pkg => sdk}/apis/cluster/v1alpha1/doc.go | 2 +- .../apis/cluster/v1alpha1/machine_types.go | 6 +- .../cluster/v1alpha1/machineclass_types.go | 4 - .../v1alpha1/machinedeployment_types.go | 6 +- .../apis/cluster/v1alpha1/machineset_types.go | 6 +- .../apis/cluster/v1alpha1/register.go | 40 +- .../cluster/v1alpha1/zz_generated.deepcopy.go | 16 +- {pkg => sdk/apis}/machines/register.go | 2 +- .../apis}/machines/v1alpha1/defaults.go | 0 {pkg => sdk/apis}/machines/v1alpha1/doc.go | 0 .../apis}/machines/v1alpha1/register.go | 0 {pkg => sdk/apis}/machines/v1alpha1/types.go | 0 .../v1alpha1/zz_generated.deepcopy.go | 8 +- {pkg => sdk}/bootstrap/doc.go | 0 {pkg => sdk}/bootstrap/types.go | 0 sdk/cloudprovider/alibaba/types.go | 42 ++ .../cloudprovider/anexia}/types.go | 47 ++- sdk/cloudprovider/aws/types.go | 69 ++++ sdk/cloudprovider/azure/types.go | 77 ++++ .../baremetal/plugins/plugins.go | 21 + .../baremetal/plugins/tinkerbell}/types.go | 15 +- .../cloudprovider/baremetal}/types.go | 12 +- sdk/cloudprovider/digitalocean/types.go | 39 ++ .../cloudprovider/equinixmetal}/types.go | 22 +- sdk/cloudprovider/gce/types.go | 86 ++++ sdk/cloudprovider/hetzner/types.go | 42 ++ sdk/cloudprovider/kubevirt/types.go | 167 ++++++++ .../cloudprovider/linode}/types.go | 20 +- .../cloudprovider/nutanix}/types.go | 30 +- sdk/cloudprovider/opennebula/types.go | 46 +++ sdk/cloudprovider/openstack/types.go | 64 +++ .../cloudprovider/scaleway}/types.go | 22 +- .../vmwareclouddirector}/types.go | 34 +- sdk/cloudprovider/vsphere/types.go | 69 ++++ sdk/cloudprovider/vultr/types.go | 42 ++ sdk/go.mod | 64 +++ sdk/go.sum | 382 ++++++++++++++++++ {pkg => sdk/internal}/test/helper.go | 0 {pkg => sdk}/jsonutil/strict.go | 0 sdk/net/net.go | 55 +++ .../types/types.go => sdk/node/eviction.go | 2 +- .../providerconfig/resolver.go | 55 +-- .../types => sdk/providerconfig}/types.go | 20 +- .../providerconfig}/types_test.go | 54 +-- .../amzn.go => sdk/userdata/amzn2/config.go | 0 sdk/userdata/default.go | 47 +++ .../userdata/default_test.go | 14 +- .../userdata/flatcar/config.go | 0 .../rhel.go => sdk/userdata/rhel/config.go | 0 .../userdata/rockylinux/config.go | 0 .../userdata/ubuntu/config.go | 0 test/e2e/provisioning/all_e2e_test.go | 9 +- test/e2e/provisioning/deploymentscenario.go | 6 +- test/e2e/provisioning/helper.go | 2 +- test/e2e/provisioning/migrateuidscenario.go | 11 +- test/e2e/provisioning/verify.go | 16 +- 190 files changed, 2733 insertions(+), 2014 deletions(-) delete mode 100644 pkg/cloudprovider/provider/alibaba/types/types.go delete mode 100644 pkg/cloudprovider/provider/anexia/types/errors.go rename pkg/{userdata/convert => cloudprovider/provider/aws}/gzip.go (93%) delete mode 100644 pkg/cloudprovider/provider/aws/types/types.go delete mode 100644 pkg/cloudprovider/provider/azure/types/types.go delete mode 100644 pkg/cloudprovider/provider/digitalocean/types/types.go delete mode 100644 pkg/cloudprovider/provider/gce/types/types.go delete mode 100644 pkg/cloudprovider/provider/hetzner/types/types.go delete mode 100644 pkg/cloudprovider/provider/kubevirt/types/types.go delete mode 100644 pkg/cloudprovider/provider/opennebula/types/types.go delete mode 100644 pkg/cloudprovider/provider/openstack/types/types.go delete mode 100644 pkg/cloudprovider/provider/vsphere/types/types.go delete mode 100644 pkg/cloudprovider/provider/vultr/types/types.go rename pkg/{apis/cluster/v1alpha1 => }/migrations/migrations.go (96%) create mode 100644 sdk/.golangci.yml create mode 100644 sdk/LICENSE create mode 100644 sdk/Makefile rename {pkg => sdk}/apis/cluster/common/consts.go (100%) rename {pkg => sdk}/apis/cluster/common/plugins.go (93%) rename {pkg => sdk}/apis/cluster/v1alpha1/common_types.go (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/conversions.go (89%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/conversions_test.go (94%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec.go (99%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go (98%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/aws.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/hetzner.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/aws.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/azure.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/digitalocean.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/hetzner.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/linode.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere-static-ip.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/aws.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/azure.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/digitalocean.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/hetzner.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/linode.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/openstack.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere-static-ip.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/aws.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/hetzner.yaml (100%) rename {pkg => sdk}/apis/cluster/v1alpha1/defaults.go (97%) rename {pkg => sdk}/apis/cluster/v1alpha1/doc.go (92%) rename {pkg => sdk}/apis/cluster/v1alpha1/machine_types.go (98%) rename {pkg => sdk}/apis/cluster/v1alpha1/machineclass_types.go (97%) rename {pkg => sdk}/apis/cluster/v1alpha1/machinedeployment_types.go (98%) rename {pkg => sdk}/apis/cluster/v1alpha1/machineset_types.go (98%) rename {pkg => sdk}/apis/cluster/v1alpha1/register.go (54%) rename {pkg => sdk}/apis/cluster/v1alpha1/zz_generated.deepcopy.go (98%) rename {pkg => sdk/apis}/machines/register.go (93%) rename {pkg => sdk/apis}/machines/v1alpha1/defaults.go (100%) rename {pkg => sdk/apis}/machines/v1alpha1/doc.go (100%) rename {pkg => sdk/apis}/machines/v1alpha1/register.go (100%) rename {pkg => sdk/apis}/machines/v1alpha1/types.go (100%) rename {pkg => sdk/apis}/machines/v1alpha1/zz_generated.deepcopy.go (97%) rename {pkg => sdk}/bootstrap/doc.go (100%) rename {pkg => sdk}/bootstrap/types.go (100%) create mode 100644 sdk/cloudprovider/alibaba/types.go rename {pkg/cloudprovider/provider/anexia/types => sdk/cloudprovider/anexia}/types.go (62%) create mode 100644 sdk/cloudprovider/aws/types.go create mode 100644 sdk/cloudprovider/azure/types.go create mode 100644 sdk/cloudprovider/baremetal/plugins/plugins.go rename {pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types => sdk/cloudprovider/baremetal/plugins/tinkerbell}/types.go (86%) rename {pkg/cloudprovider/provider/baremetal/types => sdk/cloudprovider/baremetal}/types.go (69%) create mode 100644 sdk/cloudprovider/digitalocean/types.go rename {pkg/cloudprovider/provider/equinixmetal/types => sdk/cloudprovider/equinixmetal}/types.go (51%) create mode 100644 sdk/cloudprovider/gce/types.go create mode 100644 sdk/cloudprovider/hetzner/types.go create mode 100644 sdk/cloudprovider/kubevirt/types.go rename {pkg/cloudprovider/provider/linode/types => sdk/cloudprovider/linode}/types.go (52%) rename {pkg/cloudprovider/provider/nutanix/types => sdk/cloudprovider/nutanix}/types.go (61%) create mode 100644 sdk/cloudprovider/opennebula/types.go create mode 100644 sdk/cloudprovider/openstack/types.go rename {pkg/cloudprovider/provider/scaleway/types => sdk/cloudprovider/scaleway}/types.go (51%) rename {pkg/cloudprovider/provider/vmwareclouddirector/types => sdk/cloudprovider/vmwareclouddirector}/types.go (57%) create mode 100644 sdk/cloudprovider/vsphere/types.go create mode 100644 sdk/cloudprovider/vultr/types.go create mode 100644 sdk/go.mod create mode 100644 sdk/go.sum rename {pkg => sdk/internal}/test/helper.go (100%) rename {pkg => sdk}/jsonutil/strict.go (100%) create mode 100644 sdk/net/net.go rename pkg/node/eviction/types/types.go => sdk/node/eviction.go (97%) rename pkg/providerconfig/types.go => sdk/providerconfig/resolver.go (79%) rename {pkg/providerconfig/types => sdk/providerconfig}/types.go (96%) rename {pkg/providerconfig/types => sdk/providerconfig}/types_test.go (72%) rename pkg/userdata/amzn2/amzn.go => sdk/userdata/amzn2/config.go (100%) create mode 100644 sdk/userdata/default.go rename pkg/providerconfig/types_test.go => sdk/userdata/default_test.go (77%) rename pkg/userdata/flatcar/flatcar.go => sdk/userdata/flatcar/config.go (100%) rename pkg/userdata/rhel/rhel.go => sdk/userdata/rhel/config.go (100%) rename pkg/userdata/rockylinux/rockylinux.go => sdk/userdata/rockylinux/config.go (100%) rename pkg/userdata/ubuntu/ubuntu.go => sdk/userdata/ubuntu/config.go (100%) diff --git a/.golangci.yml b/.golangci.yml index 0dd9bcf42..8c9c3df8c 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -44,11 +44,48 @@ linters-settings: - { pkg: io/ioutil, desc: https://go.dev/doc/go1.16#ioutil } - { pkg: github.com/ghodss/yaml, desc: use sigs.k8s.io/yaml instead } + revive: + rules: + # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#add-constant + - name: duplicated-imports + severity: warning + govet: enable: - nilness # find tautologies / impossible conditions + importas: + no-unaliased: true + alias: + # Machine Controller + - pkg: k8c.io/machine-controller/sdk/apis/(\w+)/(v[\w\d]+) + alias: $1$2 + # Kubernetes + - pkg: k8s.io/api/(\w+)/(v[\w\d]+) + alias: $1$2 + - pkg: k8s.io/apimachinery/pkg/apis/meta/v1 + alias: metav1 + - pkg: k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 + alias: apiextensionsv1 + - pkg: k8s.io/apimachinery/pkg/api/errors + alias: apierrors + - pkg: k8s.io/apimachinery/pkg/util/errors + alias: kerrors + # Controller Runtime + - pkg: sigs.k8s.io/controller-runtime/pkg/client + alias: ctrlruntimeclient + # Other Kube APIs + - pkg: go.anx.io/go-anxcloud/pkg/apis/(\w+)/(v[\w\d]+) + alias: anx$1$2 + - pkg: github.com/tinkerbell/tink/api/(v[\w\d]+) + alias: tink$1 + - pkg: kubevirt.io/api/(\w+)/(v[\w\d]+) + alias: kubevirt$1$2 + - pkg: kubevirt.io/containerized-data-importer-api/pkg/apis/(\w+)/(v[\w\d]+) + alias: cdi$1$2 + issues: + max-same-issues: 0 exclude: - func Convert_MachinesV1alpha1Machine_To_ClusterV1alpha1Machine should be ConvertMachinesV1alpha1MachineToClusterV1alpha1Machine - func Convert_MachineDeployment_ProviderConfig_To_ProviderSpec should be ConvertMachineDeploymentProviderConfigToProviderSpec @@ -59,4 +96,4 @@ issues: - 'cyclomatic complexity [0-9]+ of func `\(\*provider\)\.getConfig` is high' - "SA1019: s.server.IPv6 is deprecated" exclude-dirs: - - pkg/machines + - apis/machines diff --git a/Makefile b/Makefile index e338ed2ec..396c9f8e4 100644 --- a/Makefile +++ b/Makefile @@ -52,12 +52,12 @@ build-machine-controller: machine-controller .PHONY: clean clean: - rm -f machine-controller \ - webhook + rm -f machine-controller webhook .PHONY: lint lint: golangci-lint run -v + make -C sdk lint yamllint: yamllint -c .yamllint.conf . @@ -91,15 +91,18 @@ test-unit-docker: .PHONY: test-unit test-unit: go test -v ./... + cd sdk && go test -v ./... .PHONY: build-tests build-tests: go test -run nope ./... + cd sdk && go test -run nope ./... go test -tags e2e -run nope ./... .PHONY: check-dependencies check-dependencies: go mod verify + cd sdk && go mod verify .PHONY: download-gocache download-gocache: diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 6bf69cd38..8197231ae 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -30,8 +30,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1/migrations" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" "k8c.io/machine-controller/pkg/cloudprovider/util" clusterinfo "k8c.io/machine-controller/pkg/clusterinfo" @@ -41,8 +39,10 @@ import ( "k8c.io/machine-controller/pkg/controller/nodecsrapprover" "k8c.io/machine-controller/pkg/health" machinecontrollerlog "k8c.io/machine-controller/pkg/log" - machinesv1alpha1 "k8c.io/machine-controller/pkg/machines/v1alpha1" + "k8c.io/machine-controller/pkg/migrations" "k8c.io/machine-controller/pkg/node" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + machinesv1alpha1 "k8c.io/machine-controller/sdk/apis/machines/v1alpha1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/types" diff --git a/go.mod b/go.mod index 286da3126..308b0b177 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.22.3 toolchain go1.23.1 +replace k8c.io/machine-controller/sdk => ./sdk + require ( cloud.google.com/go/logging v1.11.0 cloud.google.com/go/monitoring v1.21.1 @@ -49,6 +51,7 @@ require ( gomodules.xyz/jsonpatch/v2 v2.4.0 google.golang.org/api v0.197.0 gopkg.in/yaml.v3 v3.0.1 + k8c.io/machine-controller/sdk v0.0.0-00010101000000-000000000000 k8s.io/api v0.31.1 k8s.io/apiextensions-apiserver v0.31.1 k8s.io/apimachinery v0.31.1 @@ -60,7 +63,6 @@ require ( kubevirt.io/api v1.3.1 kubevirt.io/containerized-data-importer-api v1.60.3 sigs.k8s.io/controller-runtime v0.19.0 - sigs.k8s.io/yaml v1.4.0 ) require ( @@ -185,4 +187,5 @@ require ( kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/hack/verify-boilerplate.sh b/hack/verify-boilerplate.sh index f7d2ed3b0..70e2169e0 100755 --- a/hack/verify-boilerplate.sh +++ b/hack/verify-boilerplate.sh @@ -20,5 +20,4 @@ cd $(dirname $0)/.. boilerplate \ -boilerplates hack/boilerplate \ - -exclude pkg/machines/v1alpha1 \ - -exclude pkg/signals + -exclude sdk/apis/machines/v1alpha1 diff --git a/pkg/admission/machinedeployments.go b/pkg/admission/machinedeployments.go index c380a2b6e..ec3d9f5cb 100644 --- a/pkg/admission/machinedeployments.go +++ b/pkg/admission/machinedeployments.go @@ -21,7 +21,7 @@ import ( "encoding/json" "fmt" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" admissionv1 "k8s.io/api/admission/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" diff --git a/pkg/admission/machinedeployments_test.go b/pkg/admission/machinedeployments_test.go index d33146359..763473f10 100644 --- a/pkg/admission/machinedeployments_test.go +++ b/pkg/admission/machinedeployments_test.go @@ -19,7 +19,7 @@ package admission import ( "testing" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) diff --git a/pkg/admission/machinedeployments_validation.go b/pkg/admission/machinedeployments_validation.go index e2ae84463..7079d9ba6 100644 --- a/pkg/admission/machinedeployments_validation.go +++ b/pkg/admission/machinedeployments_validation.go @@ -20,9 +20,9 @@ import ( "encoding/json" "fmt" - "k8c.io/machine-controller/pkg/apis/cluster/common" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + providerconfigtypes "k8c.io/machine-controller/sdk/providerconfig" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" @@ -32,13 +32,13 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" ) -func validateMachineDeployment(md v1alpha1.MachineDeployment) field.ErrorList { +func validateMachineDeployment(md clusterv1alpha1.MachineDeployment) field.ErrorList { allErrs := field.ErrorList{} allErrs = append(allErrs, validateMachineDeploymentSpec(&md.Spec, field.NewPath("spec"))...) return allErrs } -func validateMachineDeploymentSpec(spec *v1alpha1.MachineDeploymentSpec, fldPath *field.Path) field.ErrorList { +func validateMachineDeploymentSpec(spec *clusterv1alpha1.MachineDeploymentSpec, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} allErrs = append(allErrs, metav1validation.ValidateLabelSelector(&spec.Selector, metav1validation.LabelSelectorValidationOptions{}, fldPath.Child("selector"))...) if len(spec.Selector.MatchLabels)+len(spec.Selector.MatchExpressions) == 0 { @@ -60,7 +60,7 @@ func validateMachineDeploymentSpec(spec *v1alpha1.MachineDeploymentSpec, fldPath return allErrs } -func validateMachineDeploymentStrategy(strategy *v1alpha1.MachineDeploymentStrategy, fldPath *field.Path) field.ErrorList { +func validateMachineDeploymentStrategy(strategy *clusterv1alpha1.MachineDeploymentStrategy, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} switch strategy.Type { case common.RollingUpdateMachineDeploymentStrategyType: @@ -73,7 +73,7 @@ func validateMachineDeploymentStrategy(strategy *v1alpha1.MachineDeploymentStrat return allErrs } -func validateMachineRollingUpdateDeployment(rollingUpdate *v1alpha1.MachineRollingUpdateDeployment, fldPath *field.Path) field.ErrorList { +func validateMachineRollingUpdateDeployment(rollingUpdate *clusterv1alpha1.MachineRollingUpdateDeployment, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} var maxUnavailable int var maxSurge int @@ -110,11 +110,11 @@ func getIntOrPercent(s *intstr.IntOrString, roundUp bool) (int, error) { return intstr.GetValueFromIntOrPercent(s, 100, roundUp) } -func machineDeploymentDefaultingFunction(md *v1alpha1.MachineDeployment) { - v1alpha1.PopulateDefaultsMachineDeployment(md) +func machineDeploymentDefaultingFunction(md *clusterv1alpha1.MachineDeployment) { + clusterv1alpha1.PopulateDefaultsMachineDeployment(md) } -func mutationsForMachineDeployment(md *v1alpha1.MachineDeployment) error { +func mutationsForMachineDeployment(md *clusterv1alpha1.MachineDeployment) error { providerConfig, err := providerconfigtypes.GetConfig(md.Spec.Template.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to read MachineDeployment.Spec.Template.Spec.ProviderSpec: %w", err) diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index b564176fc..2975877db 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -24,11 +24,11 @@ import ( "github.com/Masterminds/semver/v3" "golang.org/x/crypto/ssh" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8c.io/machine-controller/pkg/cloudprovider" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + "k8c.io/machine-controller/sdk/providerconfig" + "k8c.io/machine-controller/sdk/userdata" admissionv1 "k8s.io/api/admission/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -101,7 +101,7 @@ func (ad *admissionData) mutateMachines(ctx context.Context, ar admissionv1.Admi common.SetKubeletFlags(&machine, map[common.KubeletFlags]string{ common.ExternalCloudProviderKubeletFlag: fmt.Sprintf("%t", ad.nodeSettings.ExternalCloudProvider), }) - providerConfig, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) + providerConfig, err := providerconfig.GetConfig(machine.Spec.ProviderSpec) if err != nil { return nil, err } @@ -116,7 +116,7 @@ func (ad *admissionData) mutateMachines(ctx context.Context, ar admissionv1.Admi } func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec *clusterv1alpha1.MachineSpec) error { - providerConfig, err := providerconfigtypes.GetConfig(spec.ProviderSpec) + providerConfig, err := providerconfig.GetConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to read machine.spec.providerSpec: %w", err) } @@ -164,7 +164,7 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec return fmt.Errorf("Invalid public keys specified: %w", err) } - defaultedOperatingSystemSpec, err := providerconfig.DefaultOperatingSystemSpec( + defaultedOperatingSystemSpec, err := userdata.DefaultOperatingSystemSpec( providerConfig.OperatingSystem, providerConfig.OperatingSystemSpec, ) diff --git a/pkg/admission/util.go b/pkg/admission/util.go index 83124ed4b..d435b356f 100644 --- a/pkg/admission/util.go +++ b/pkg/admission/util.go @@ -20,7 +20,7 @@ import ( "encoding/json" "fmt" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + providerconfigtypes "k8c.io/machine-controller/sdk/providerconfig" ) const cloudProviderPacket = "packet" diff --git a/pkg/cloudprovider/cache/cloudprovidercache.go b/pkg/cloudprovider/cache/cloudprovidercache.go index 8254ffd1a..f9d6c46fc 100644 --- a/pkg/cloudprovider/cache/cloudprovidercache.go +++ b/pkg/cloudprovider/cache/cloudprovidercache.go @@ -24,7 +24,7 @@ import ( gocache "github.com/patrickmn/go-cache" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" ) type CloudproviderCache struct { diff --git a/pkg/cloudprovider/cache/cloudprovidercache_test.go b/pkg/cloudprovider/cache/cloudprovidercache_test.go index 9d8b0d7eb..b7013109c 100644 --- a/pkg/cloudprovider/cache/cloudprovidercache_test.go +++ b/pkg/cloudprovider/cache/cloudprovidercache_test.go @@ -20,7 +20,7 @@ import ( "errors" "testing" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" diff --git a/pkg/cloudprovider/errors/errors.go b/pkg/cloudprovider/errors/errors.go index 81b9c55ea..7182c5033 100644 --- a/pkg/cloudprovider/errors/errors.go +++ b/pkg/cloudprovider/errors/errors.go @@ -20,7 +20,7 @@ import ( "errors" "fmt" - "k8c.io/machine-controller/pkg/apis/cluster/common" + "k8c.io/machine-controller/sdk/apis/cluster/common" ) var ( diff --git a/pkg/cloudprovider/instance/instance.go b/pkg/cloudprovider/instance/instance.go index f97c327a9..6bee0865d 100644 --- a/pkg/cloudprovider/instance/instance.go +++ b/pkg/cloudprovider/instance/instance.go @@ -16,7 +16,7 @@ limitations under the License. package instance -import v1 "k8s.io/api/core/v1" +import corev1 "k8s.io/api/core/v1" // Instance represents a instance on the cloud provider. type Instance interface { @@ -27,7 +27,7 @@ type Instance interface { // ProviderID returns the expected providerID for the instance ProviderID() string // Addresses returns a list of addresses associated with the instance. - Addresses() map[string]v1.NodeAddressType + Addresses() map[string]corev1.NodeAddressType // Status returns the instance status. Status() Status } diff --git a/pkg/cloudprovider/provider.go b/pkg/cloudprovider/provider.go index 032f8c775..afa5eede4 100644 --- a/pkg/cloudprovider/provider.go +++ b/pkg/cloudprovider/provider.go @@ -42,8 +42,7 @@ import ( "k8c.io/machine-controller/pkg/cloudprovider/provider/vsphere" "k8c.io/machine-controller/pkg/cloudprovider/provider/vultr" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/providerconfig" ) var ( @@ -52,88 +51,88 @@ var ( // ErrProviderNotFound tells that the requested cloud provider was not found. ErrProviderNotFound = errors.New("cloudprovider not found") - providers = map[providerconfigtypes.CloudProvider]func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider{ - providerconfigtypes.CloudProviderDigitalocean: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providers = map[providerconfig.CloudProvider]func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider{ + providerconfig.CloudProviderDigitalocean: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return digitalocean.New(cvr) }, - providerconfigtypes.CloudProviderAWS: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderAWS: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return aws.New(cvr) }, - providerconfigtypes.CloudProviderOpenstack: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderOpenstack: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return openstack.New(cvr) }, - providerconfigtypes.CloudProviderGoogle: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderGoogle: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return gce.New(cvr) }, - providerconfigtypes.CloudProviderHetzner: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderHetzner: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return hetzner.New(cvr) }, - providerconfigtypes.CloudProviderVsphere: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderVsphere: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return vsphere.New(cvr) }, - providerconfigtypes.CloudProviderAzure: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderAzure: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return azure.New(cvr) }, - providerconfigtypes.CloudProviderEquinixMetal: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderEquinixMetal: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return equinixmetal.New(cvr) }, // NB: This is explicitly left to allow old Packet machines to be deleted. // We can handle those machines in the same way as Equinix Metal machines // because there are no API changes. // TODO: Remove this after deprecation period. - providerconfigtypes.CloudProviderPacket: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderPacket: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return equinixmetal.New(cvr) }, - providerconfigtypes.CloudProviderFake: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderFake: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return fake.New(cvr) }, - providerconfigtypes.CloudProviderEdge: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderEdge: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return edge.New(cvr) }, - providerconfigtypes.CloudProviderKubeVirt: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderKubeVirt: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return kubevirt.New(cvr) }, - providerconfigtypes.CloudProviderAlibaba: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderAlibaba: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return alibaba.New(cvr) }, - providerconfigtypes.CloudProviderScaleway: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderScaleway: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return scaleway.New(cvr) }, - providerconfigtypes.CloudProviderAnexia: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderAnexia: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return anexia.New(cvr) }, - providerconfigtypes.CloudProviderBaremetal: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderBaremetal: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { // TODO(MQ): add a baremetal driver. return baremetal.New(cvr) }, - providerconfigtypes.CloudProviderNutanix: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderNutanix: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return nutanix.New(cvr) }, - providerconfigtypes.CloudProviderVMwareCloudDirector: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderVMwareCloudDirector: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return vcd.New(cvr) }, - providerconfigtypes.CloudProviderExternal: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderExternal: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return external.New(cvr) }, } // communityProviders holds a map of cloud providers that have been implemented by community members and // contributed to machine-controller. They are not end-to-end tested by the machine-controller development team. - communityProviders = map[providerconfigtypes.CloudProvider]func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider{ - providerconfigtypes.CloudProviderLinode: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + communityProviders = map[providerconfig.CloudProvider]func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider{ + providerconfig.CloudProviderLinode: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return linode.New(cvr) }, - providerconfigtypes.CloudProviderVultr: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderVultr: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return vultr.New(cvr) }, - providerconfigtypes.CloudProviderOpenNebula: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderOpenNebula: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return opennebula.New(cvr) }, } ) // ForProvider returns a CloudProvider actuator for the requested provider. -func ForProvider(p providerconfigtypes.CloudProvider, cvr *providerconfig.ConfigVarResolver) (cloudprovidertypes.Provider, error) { +func ForProvider(p providerconfig.CloudProvider, cvr *providerconfig.ConfigVarResolver) (cloudprovidertypes.Provider, error) { if p, found := providers[p]; found { return NewValidationCacheWrappingCloudProvider(p(cvr)), nil } diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index e46ce5d1f..7c5029185 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -27,18 +27,17 @@ import ( "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - alibabatypes "k8c.io/machine-controller/pkg/cloudprovider/provider/alibaba/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" "k8c.io/machine-controller/pkg/cloudprovider/util" kuberneteshelper "k8c.io/machine-controller/pkg/kubernetes" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + alibabatypes "k8c.io/machine-controller/sdk/cloudprovider/alibaba" + "k8c.io/machine-controller/sdk/providerconfig" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" ) @@ -91,10 +90,10 @@ func (a *alibabaInstance) ProviderID() string { return "" } -func (a *alibabaInstance) Addresses() map[string]v1.NodeAddressType { - primaryIPAddresses := map[string]v1.NodeAddressType{} +func (a *alibabaInstance) Addresses() map[string]corev1.NodeAddressType { + primaryIPAddresses := map[string]corev1.NodeAddressType{} for _, networkInterface := range a.instance.NetworkInterfaces.NetworkInterface { - primaryIPAddresses[networkInterface.PrimaryIpAddress] = v1.NodeInternalIP + primaryIPAddresses[networkInterface.PrimaryIpAddress] = corev1.NodeInternalIP } return primaryIPAddresses @@ -341,8 +340,8 @@ func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfig.Config, error) { + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, fmt.Errorf("failed to decode providers config: %w", err) } @@ -429,7 +428,7 @@ func getInstance(client *ecs.Client, instanceName string, uid string) (*ecs.Inst return &response.Instances.Instance[0], nil } -func (p *provider) getImageIDForOS(machineSpec clusterv1alpha1.MachineSpec, os providerconfigtypes.OperatingSystem) (string, error) { +func (p *provider) getImageIDForOS(machineSpec clusterv1alpha1.MachineSpec, os providerconfig.OperatingSystem) (string, error) { c, _, err := p.getConfig(machineSpec.ProviderSpec) if err != nil { return "", fmt.Errorf("failed to get alibaba client: %w", err) @@ -450,11 +449,11 @@ func (p *provider) getImageIDForOS(machineSpec clusterv1alpha1.MachineSpec, os p return "", fmt.Errorf("failed to describe alibaba images: %w", err) } - var availableImage = map[providerconfigtypes.OperatingSystem]string{} + var availableImage = map[providerconfig.OperatingSystem]string{} for _, image := range response.Images.Image { switch image.OSNameEn { case ubuntuImageName: - availableImage[providerconfigtypes.OperatingSystemUbuntu] = image.ImageId + availableImage[providerconfig.OperatingSystemUbuntu] = image.ImageId } } @@ -462,5 +461,5 @@ func (p *provider) getImageIDForOS(machineSpec clusterv1alpha1.MachineSpec, os p return imageID, nil } - return "", providerconfigtypes.ErrOSNotSupported + return "", providerconfig.ErrOSNotSupported } diff --git a/pkg/cloudprovider/provider/alibaba/types/types.go b/pkg/cloudprovider/provider/alibaba/types/types.go deleted file mode 100644 index 10d1022ef..000000000 --- a/pkg/cloudprovider/provider/alibaba/types/types.go +++ /dev/null @@ -1,42 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" -) - -type RawConfig struct { - AccessKeyID providerconfigtypes.ConfigVarString `json:"accessKeyID,omitempty"` - AccessKeySecret providerconfigtypes.ConfigVarString `json:"accessKeySecret,omitempty"` - RegionID providerconfigtypes.ConfigVarString `json:"regionID,omitempty"` - InstanceName providerconfigtypes.ConfigVarString `json:"instanceName,omitempty"` - InstanceType providerconfigtypes.ConfigVarString `json:"instanceType,omitempty"` - VSwitchID providerconfigtypes.ConfigVarString `json:"vSwitchID,omitempty"` - InternetMaxBandwidthOut providerconfigtypes.ConfigVarString `json:"internetMaxBandwidthOut,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - ZoneID providerconfigtypes.ConfigVarString `json:"zoneID,omitempty"` - DiskType providerconfigtypes.ConfigVarString `json:"diskType,omitempty"` - DiskSize providerconfigtypes.ConfigVarString `json:"diskSize,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/anexia/helper_test.go b/pkg/cloudprovider/provider/anexia/helper_test.go index 2936ccb53..1bcfec34f 100644 --- a/pkg/cloudprovider/provider/anexia/helper_test.go +++ b/pkg/cloudprovider/provider/anexia/helper_test.go @@ -22,12 +22,10 @@ import ( "github.com/gophercloud/gophercloud/testhelper" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" - anxtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/anexia/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/providerconfig/types" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + anxtypes "k8c.io/machine-controller/sdk/cloudprovider/anexia" + providerconfigtypes "k8c.io/machine-controller/sdk/providerconfig" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -46,7 +44,7 @@ type ConfigTestCase struct { } type ValidateCallTestCase struct { - Spec v1alpha1.MachineSpec + Spec clusterv1alpha1.MachineSpec ExpectedError error } @@ -56,14 +54,14 @@ func getSpecsForValidationTest(t *testing.T, configCases []ConfigTestCase) []Val for _, configCase := range configCases { jsonConfig, err := json.Marshal(configCase.Config) testhelper.AssertNoErr(t, err) - jsonProviderConfig, err := json.Marshal(types.Config{ + jsonProviderConfig, err := json.Marshal(providerconfigtypes.Config{ CloudProviderSpec: runtime.RawExtension{Raw: jsonConfig}, OperatingSystemSpec: runtime.RawExtension{Raw: []byte("{}")}, }) testhelper.AssertNoErr(t, err) testCases = append(testCases, ValidateCallTestCase{ - Spec: v1alpha1.MachineSpec{ - ProviderSpec: v1alpha1.ProviderSpec{ + Spec: clusterv1alpha1.MachineSpec{ + ProviderSpec: clusterv1alpha1.ProviderSpec{ Value: &runtime.RawExtension{Raw: jsonProviderConfig}, }, }, @@ -73,8 +71,8 @@ func getSpecsForValidationTest(t *testing.T, configCases []ConfigTestCase) []Val return testCases } -func newConfigVarString(str string) types.ConfigVarString { - return types.ConfigVarString{ +func newConfigVarString(str string) providerconfigtypes.ConfigVarString { + return providerconfigtypes.ConfigVarString{ Value: str, } } @@ -91,7 +89,7 @@ func hookableConfig(hook func(*anxtypes.RawConfig)) anxtypes.RawConfig { }, Networks: []anxtypes.RawNetwork{ - {VlanID: newConfigVarString("test-vlan"), PrefixIDs: []types.ConfigVarString{newConfigVarString("test-prefix")}}, + {VlanID: newConfigVarString("test-vlan"), PrefixIDs: []providerconfigtypes.ConfigVarString{newConfigVarString("test-prefix")}}, }, Token: newConfigVarString("test-token"), @@ -109,7 +107,7 @@ func hookableConfig(hook func(*anxtypes.RawConfig)) anxtypes.RawConfig { // this generates a full reconcileContext with some default values and allows hooking into it to e.g. remove/overwrite a value. func hookableReconcileContext(locationID string, templateID string, hook func(*reconcileContext)) reconcileContext { context := reconcileContext{ - Machine: &v1alpha1.Machine{ + Machine: &clusterv1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{Name: "TestMachine"}, }, Status: &anxtypes.ProviderStatus{}, diff --git a/pkg/cloudprovider/provider/anexia/instance.go b/pkg/cloudprovider/provider/anexia/instance.go index 8af72f93c..9bb212802 100644 --- a/pkg/cloudprovider/provider/anexia/instance.go +++ b/pkg/cloudprovider/provider/anexia/instance.go @@ -22,9 +22,9 @@ import ( "go.anx.io/go-anxcloud/pkg/vsphere/info" "k8c.io/machine-controller/pkg/cloudprovider/instance" - anxtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/anexia/types" + anxtypes "k8c.io/machine-controller/sdk/cloudprovider/anexia" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" ) type anexiaInstance struct { @@ -57,22 +57,22 @@ func (ai *anexiaInstance) ProviderID() string { return ai.ID() } -func (ai *anexiaInstance) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{} +func (ai *anexiaInstance) Addresses() map[string]corev1.NodeAddressType { + addresses := map[string]corev1.NodeAddressType{} if ai.reservedAddresses != nil { for _, reservedIP := range ai.reservedAddresses { - addresses[reservedIP] = v1.NodeExternalIP + addresses[reservedIP] = corev1.NodeExternalIP } } if ai.info != nil { for _, network := range ai.info.Network { for _, ip := range network.IPv4 { - addresses[ip] = v1.NodeExternalIP + addresses[ip] = corev1.NodeExternalIP } for _, ip := range network.IPv6 { - addresses[ip] = v1.NodeExternalIP + addresses[ip] = corev1.NodeExternalIP } } } @@ -80,9 +80,9 @@ func (ai *anexiaInstance) Addresses() map[string]v1.NodeAddressType { for ip := range addresses { parsed := net.ParseIP(ip) if parsed.IsPrivate() { - addresses[ip] = v1.NodeInternalIP + addresses[ip] = corev1.NodeInternalIP } else { - addresses[ip] = v1.NodeExternalIP + addresses[ip] = corev1.NodeExternalIP } } diff --git a/pkg/cloudprovider/provider/anexia/instance_test.go b/pkg/cloudprovider/provider/anexia/instance_test.go index 8340752a9..dbda1c43e 100644 --- a/pkg/cloudprovider/provider/anexia/instance_test.go +++ b/pkg/cloudprovider/provider/anexia/instance_test.go @@ -22,11 +22,11 @@ import ( "github.com/gophercloud/gophercloud/testhelper" "go.anx.io/go-anxcloud/pkg/vsphere/info" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" ) func TestAnexiaInstance(t *testing.T) { - addressCheck := func(t *testing.T, testcase string, instance *anexiaInstance, expected map[string]v1.NodeAddressType) { + addressCheck := func(t *testing.T, testcase string, instance *anexiaInstance, expected map[string]corev1.NodeAddressType) { t.Run(testcase, func(t *testing.T) { addresses := instance.Addresses() @@ -36,7 +36,7 @@ func TestAnexiaInstance(t *testing.T) { t.Run("empty instance", func(t *testing.T) { instance := anexiaInstance{} - addressCheck(t, "no addresses", &instance, map[string]v1.NodeAddressType{}) + addressCheck(t, "no addresses", &instance, map[string]corev1.NodeAddressType{}) }) t.Run("instance with only reservedAddresses set", func(t *testing.T) { @@ -44,11 +44,11 @@ func TestAnexiaInstance(t *testing.T) { reservedAddresses: []string{"10.0.0.2", "fda0:23::2", "8.8.8.8", "2001:db8::2"}, } - addressCheck(t, "expected addresses", &instance, map[string]v1.NodeAddressType{ - "10.0.0.2": v1.NodeInternalIP, - "fda0:23::2": v1.NodeInternalIP, - "8.8.8.8": v1.NodeExternalIP, - "2001:db8::2": v1.NodeExternalIP, + addressCheck(t, "expected addresses", &instance, map[string]corev1.NodeAddressType{ + "10.0.0.2": corev1.NodeInternalIP, + "fda0:23::2": corev1.NodeInternalIP, + "8.8.8.8": corev1.NodeExternalIP, + "2001:db8::2": corev1.NodeExternalIP, }) }) @@ -68,11 +68,11 @@ func TestAnexiaInstance(t *testing.T) { }, } - addressCheck(t, "expected addresses", &instance, map[string]v1.NodeAddressType{ - "10.0.0.2": v1.NodeInternalIP, - "fda0:23::2": v1.NodeInternalIP, - "8.8.8.8": v1.NodeExternalIP, - "2001:db8::2": v1.NodeExternalIP, + addressCheck(t, "expected addresses", &instance, map[string]corev1.NodeAddressType{ + "10.0.0.2": corev1.NodeInternalIP, + "fda0:23::2": corev1.NodeInternalIP, + "8.8.8.8": corev1.NodeExternalIP, + "2001:db8::2": corev1.NodeExternalIP, }) }) @@ -93,11 +93,11 @@ func TestAnexiaInstance(t *testing.T) { }, } - addressCheck(t, "expected addresses", &instance, map[string]v1.NodeAddressType{ - "10.0.0.2": v1.NodeInternalIP, - "fda0:23::2": v1.NodeInternalIP, - "8.8.8.8": v1.NodeExternalIP, - "2001:db8::2": v1.NodeExternalIP, + addressCheck(t, "expected addresses", &instance, map[string]corev1.NodeAddressType{ + "10.0.0.2": corev1.NodeInternalIP, + "fda0:23::2": corev1.NodeInternalIP, + "8.8.8.8": corev1.NodeExternalIP, + "2001:db8::2": corev1.NodeExternalIP, }) }) @@ -117,11 +117,11 @@ func TestAnexiaInstance(t *testing.T) { }, } - addressCheck(t, "expected addresses", &instance, map[string]v1.NodeAddressType{ - "10.0.0.2": v1.NodeInternalIP, - "fda0:23::2": v1.NodeInternalIP, - "8.8.8.8": v1.NodeExternalIP, - "2001:db8::2": v1.NodeExternalIP, + addressCheck(t, "expected addresses", &instance, map[string]corev1.NodeAddressType{ + "10.0.0.2": corev1.NodeInternalIP, + "fda0:23::2": corev1.NodeInternalIP, + "8.8.8.8": corev1.NodeExternalIP, + "2001:db8::2": corev1.NodeExternalIP, }) }) } diff --git a/pkg/cloudprovider/provider/anexia/network_provisioning.go b/pkg/cloudprovider/provider/anexia/network_provisioning.go index 963fb6f28..a133509a3 100644 --- a/pkg/cloudprovider/provider/anexia/network_provisioning.go +++ b/pkg/cloudprovider/provider/anexia/network_provisioning.go @@ -26,8 +26,8 @@ import ( anxvm "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/common" - anxtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/anexia/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + anxtypes "k8c.io/machine-controller/sdk/cloudprovider/anexia" ) func networkInterfacesForProvisioning(ctx context.Context, log *zap.SugaredLogger, client anxclient.Client) ([]anxvm.Network, error) { diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 514c6d6a6..795add1e6 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -27,42 +27,33 @@ import ( "time" "go.anx.io/go-anxcloud/pkg/api" - "go.anx.io/go-anxcloud/pkg/client" anxclient "go.anx.io/go-anxcloud/pkg/client" "go.anx.io/go-anxcloud/pkg/vsphere" "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/progress" anxvm "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8c.io/machine-controller/pkg/cloudprovider/common/ssh" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - anxtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/anexia/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" cloudproviderutil "k8c.io/machine-controller/pkg/cloudprovider/util" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + anxtypes "k8c.io/machine-controller/sdk/cloudprovider/anexia" + "k8c.io/machine-controller/sdk/providerconfig" "k8s.io/apimachinery/pkg/api/meta" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" k8stypes "k8s.io/apimachinery/pkg/types" + kerrors "k8s.io/apimachinery/pkg/util/errors" ) const ( ProvisionedType = "Provisioned" ) -var ( - // ErrConfigDiskSizeAndDisks is returned when the config has both DiskSize and Disks set, which is unsupported. - ErrConfigDiskSizeAndDisks = errors.New("both the deprecated DiskSize and new Disks attribute are set") - - // ErrConfigVlanIDAndNetworks is returned when the config has both VlanID and Networks set, which is unsupported. - ErrConfigVlanIDAndNetworks = errors.New("both the deprecated VlanID and new Networks attribute are set") -) - type provider struct { configVarResolver *providerconfig.ConfigVarResolver } @@ -96,7 +87,7 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * // make sure status is reflected in Machine Object defer func() { // if error occurs during updating the machine object don't override the original error - retErr = anxtypes.NewMultiError(retErr, updateMachineStatus(machine, status, data.Update)) + retErr = kerrors.NewAggregate([]error{retErr, updateMachineStatus(machine, status, data.Update)}) }() // provision machine @@ -174,9 +165,9 @@ func provisionVM(ctx context.Context, log *zap.SugaredLogger, client anxclient.C vm.SSH = sshKey.PublicKey provisionResponse, err := vmAPI.Provisioning().VM().Provision(ctx, vm, false) - meta.SetStatusCondition(&status.Conditions, v1.Condition{ + meta.SetStatusCondition(&status.Conditions, metav1.Condition{ Type: ProvisionedType, - Status: v1.ConditionFalse, + Status: metav1.ConditionFalse, Reason: "Provisioning", Message: "provisioning request was sent", }) @@ -196,9 +187,9 @@ func provisionVM(ctx context.Context, log *zap.SugaredLogger, client anxclient.C log.Info("Using provisionID from machine to await completion") - meta.SetStatusCondition(&status.Conditions, v1.Condition{ + meta.SetStatusCondition(&status.Conditions, metav1.Condition{ Type: ProvisionedType, - Status: v1.ConditionTrue, + Status: metav1.ConditionTrue, Reason: "Provisioned", Message: "Machine has been successfully created", }) @@ -212,20 +203,20 @@ func isAlreadyProvisioning(ctx context.Context) bool { lastChange := condition.LastTransitionTime.Time const reasonInProvisioning = "InProvisioning" if condition.Reason == reasonInProvisioning && time.Since(lastChange) > 5*time.Minute { - meta.SetStatusCondition(&status.Conditions, v1.Condition{ + meta.SetStatusCondition(&status.Conditions, metav1.Condition{ Type: ProvisionedType, Reason: "ReInitialising", Message: "Could not find ongoing VM provisioning", - Status: v1.ConditionFalse, + Status: metav1.ConditionFalse, }) } - return condition.Status == v1.ConditionFalse && condition.Reason == reasonInProvisioning + return condition.Status == metav1.ConditionFalse && condition.Reason == reasonInProvisioning } func ensureConditions(status *anxtypes.ProviderStatus) { - conditions := [...]v1.Condition{ - {Type: ProvisionedType, Message: "", Status: v1.ConditionUnknown, Reason: "Initialising"}, + conditions := [...]metav1.Condition{ + {Type: ProvisionedType, Message: "", Status: metav1.ConditionUnknown, Reason: "Initialising"}, } for _, condition := range conditions { if meta.FindStatusCondition(status.Conditions, condition.Type) == nil { @@ -234,8 +225,8 @@ func ensureConditions(status *anxtypes.ProviderStatus) { } } -func (p *provider) getConfig(ctx context.Context, log *zap.SugaredLogger, provSpec clusterv1alpha1.ProviderSpec) (*resolvedConfig, *providerconfigtypes.Config, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) +func (p *provider) getConfig(ctx context.Context, log *zap.SugaredLogger, provSpec clusterv1alpha1.ProviderSpec) (*resolvedConfig, *providerconfig.Config, error) { + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, err } @@ -395,7 +386,7 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine // make sure status is reflected in Machine Object defer func() { // if error occurs during updating the machine object don't override the original error - retErr = anxtypes.NewMultiError(retErr, updateMachineStatus(machine, status, data.Update)) + retErr = kerrors.NewAggregate([]error{retErr, updateMachineStatus(machine, status, data.Update)}) }() ensureConditions(&status) @@ -549,7 +540,7 @@ func anexiaErrorToTerminalError(err error, msg string) error { } } - var responseError *client.ResponseError + var responseError *anxclient.ResponseError if errors.As(err, &responseError) && (responseError.ErrorData.Code == http.StatusForbidden || responseError.ErrorData.Code == http.StatusUnauthorized) { return cloudprovidererrors.TerminalError{ Reason: common.InvalidConfigurationMachineError, diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index 61b6e4ff6..71ba0601a 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -31,9 +31,8 @@ import ( "github.com/gophercloud/gophercloud/testhelper" "go.anx.io/go-anxcloud/pkg/api" "go.anx.io/go-anxcloud/pkg/api/mock" - corev1 "go.anx.io/go-anxcloud/pkg/apis/core/v1" - vspherev1 "go.anx.io/go-anxcloud/pkg/apis/vsphere/v1" - "go.anx.io/go-anxcloud/pkg/client" + anxcorev1 "go.anx.io/go-anxcloud/pkg/apis/core/v1" + anxvspherev1 "go.anx.io/go-anxcloud/pkg/apis/vsphere/v1" anxclient "go.anx.io/go-anxcloud/pkg/client" "go.anx.io/go-anxcloud/pkg/core" "go.anx.io/go-anxcloud/pkg/ipam/address" @@ -41,11 +40,11 @@ import ( "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" - anxtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/anexia/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + anxtypes "k8c.io/machine-controller/sdk/cloudprovider/anexia" + providerconfigtypes "k8c.io/machine-controller/sdk/providerconfig" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -63,11 +62,11 @@ func TestAnexiaProvider(t *testing.T) { log := zap.NewNop().Sugar() a := mock.NewMockAPI() - a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID-OLD-BUILD", Name: testTemplateName, Build: "b01"}) - a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID", Name: testTemplateName, Build: "b02"}) - a.FakeExisting(&vspherev1.Template{Identifier: "WRONG-TEMPLATE-NAME", Name: "Wrong Template Name", Build: "b02"}) - a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID-NO-NETWORK-CONFIG", Name: "no-network-config", Build: "b03"}) - a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID-ADDITIONAL-DISKS", Name: "additional-disks", Build: "b03"}) + a.FakeExisting(&anxvspherev1.Template{Identifier: "TEMPLATE-ID-OLD-BUILD", Name: testTemplateName, Build: "b01"}) + a.FakeExisting(&anxvspherev1.Template{Identifier: "TEMPLATE-ID", Name: testTemplateName, Build: "b02"}) + a.FakeExisting(&anxvspherev1.Template{Identifier: "WRONG-TEMPLATE-NAME", Name: "Wrong Template Name", Build: "b02"}) + a.FakeExisting(&anxvspherev1.Template{Identifier: "TEMPLATE-ID-NO-NETWORK-CONFIG", Name: "no-network-config", Build: "b03"}) + a.FakeExisting(&anxvspherev1.Template{Identifier: "TEMPLATE-ID-ADDITIONAL-DISKS", Name: "additional-disks", Build: "b03"}) t.Cleanup(func() { testhelper.TeardownHTTP() @@ -336,7 +335,7 @@ func TestValidate(t *testing.T) { }, ConfigTestCase{ Config: hookableConfig(func(c *anxtypes.RawConfig) { c.DiskSize = 10 }), - Error: ErrConfigDiskSizeAndDisks, + Error: anxtypes.ErrConfigDiskSizeAndDisks, }, ConfigTestCase{ Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Disks[0].Size = 0 }), @@ -357,7 +356,7 @@ func TestValidate(t *testing.T) { }, ConfigTestCase{ Config: hookableConfig(func(c *anxtypes.RawConfig) { c.VlanID.Value = "legacy VLAN-ID" }), - Error: ErrConfigVlanIDAndNetworks, + Error: anxtypes.ErrConfigVlanIDAndNetworks, }, ConfigTestCase{ Config: hookableConfig(func(c *anxtypes.RawConfig) { c.DiskSize = 10; c.Disks = []anxtypes.RawDisk{} }), @@ -399,7 +398,7 @@ func TestEnsureConditions(t *testing.T) { func TestGetProviderStatus(t *testing.T) { t.Parallel() - machine := &v1alpha1.Machine{} + machine := &clusterv1alpha1.Machine{} providerStatus := anxtypes.ProviderStatus{ InstanceID: "InstanceID", } @@ -414,7 +413,7 @@ func TestGetProviderStatus(t *testing.T) { func TestUpdateStatus(t *testing.T) { t.Parallel() - machine := &v1alpha1.Machine{} + machine := &clusterv1alpha1.Machine{} providerStatus := anxtypes.ProviderStatus{ InstanceID: "InstanceID", } @@ -423,7 +422,7 @@ func TestUpdateStatus(t *testing.T) { machine.Status.ProviderStatus = &runtime.RawExtension{Raw: providerStatusJSON} called := false - err = updateMachineStatus(machine, providerStatus, func(paramMachine *v1alpha1.Machine, _ ...cloudprovidertypes.MachineModifier) error { + err = updateMachineStatus(machine, providerStatus, func(paramMachine *clusterv1alpha1.Machine, _ ...cloudprovidertypes.MachineModifier) error { called = true testhelper.AssertEquals(t, machine, paramMachine) status := getProviderStatus(zap.NewNop().Sugar(), machine) @@ -449,19 +448,19 @@ func Test_anexiaErrorToTerminalError(t *testing.T) { }) legacyClientRun := func(url string) error { - client, err := client.New(client.BaseURL(url), client.IgnoreMissingToken(), client.ParseEngineErrors(true)) + client, err := anxclient.New(anxclient.BaseURL(url), anxclient.IgnoreMissingToken(), anxclient.ParseEngineErrors(true)) testhelper.AssertNoErr(t, err) _, err = core.NewAPI(client).Location().List(context.Background(), 1, 1, "", "") return err } apiClientRun := func(url string) error { - client, err := api.NewAPI(api.WithClientOptions( - client.BaseURL(url), - client.IgnoreMissingToken(), + api, err := api.NewAPI(api.WithClientOptions( + anxclient.BaseURL(url), + anxclient.IgnoreMissingToken(), )) testhelper.AssertNoErr(t, err) - return client.Get(context.Background(), &corev1.Location{Identifier: "foo"}) + return api.Get(context.Background(), &anxcorev1.Location{Identifier: "foo"}) } testCases := []struct { @@ -512,7 +511,7 @@ func Test_anexiaErrorToTerminalError(t *testing.T) { }) t.Run("legacy api client unspecific ResponseError shouldn't convert to TerminalError", func(t *testing.T) { - var err error = &client.ResponseError{} + var err error = &anxclient.ResponseError{} err = anexiaErrorToTerminalError(err, "foo") if ok, _, _ := cloudprovidererrors.IsTerminalError(err); ok { t.Errorf("unexpected error %#v, expected no TerminalError", err) diff --git a/pkg/cloudprovider/provider/anexia/reconcile_context.go b/pkg/cloudprovider/provider/anexia/reconcile_context.go index 276851a7f..e30e44365 100644 --- a/pkg/cloudprovider/provider/anexia/reconcile_context.go +++ b/pkg/cloudprovider/provider/anexia/reconcile_context.go @@ -19,10 +19,10 @@ package anexia import ( "context" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" - anxtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/anexia/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + anxtypes "k8c.io/machine-controller/sdk/cloudprovider/anexia" + providerconfigtypes "k8c.io/machine-controller/sdk/providerconfig" ) type contextKey byte @@ -30,7 +30,7 @@ type contextKey byte const machineReconcileContextKey contextKey = 0 type reconcileContext struct { - Machine *v1alpha1.Machine + Machine *clusterv1alpha1.Machine Status *anxtypes.ProviderStatus UserData string Config resolvedConfig diff --git a/pkg/cloudprovider/provider/anexia/resolve_config.go b/pkg/cloudprovider/provider/anexia/resolve_config.go index 7dddd112a..620639715 100644 --- a/pkg/cloudprovider/provider/anexia/resolve_config.go +++ b/pkg/cloudprovider/provider/anexia/resolve_config.go @@ -20,13 +20,12 @@ import ( "context" "fmt" - "go.uber.org/zap" - "go.anx.io/go-anxcloud/pkg/api" - corev1 "go.anx.io/go-anxcloud/pkg/apis/core/v1" - vspherev1 "go.anx.io/go-anxcloud/pkg/apis/vsphere/v1" + anxcorev1 "go.anx.io/go-anxcloud/pkg/apis/core/v1" + anxvspherev1 "go.anx.io/go-anxcloud/pkg/apis/vsphere/v1" + "go.uber.org/zap" - anxtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/anexia/types" + anxtypes "k8c.io/machine-controller/sdk/cloudprovider/anexia" ) // resolvedDisk contains the resolved values from types.RawDisk. @@ -71,7 +70,7 @@ func (p *provider) resolveTemplateID(ctx context.Context, a api.API, config anxt return "", fmt.Errorf("failed to get 'templateBuild': %w", err) } - template, err := vspherev1.FindNamedTemplate(ctx, a, templateName, templateBuild, corev1.Location{Identifier: locationID}) + template, err := anxvspherev1.FindNamedTemplate(ctx, a, templateName, templateBuild, anxcorev1.Location{Identifier: locationID}) if err != nil { return "", fmt.Errorf("failed to retrieve named template: %w", err) } @@ -83,7 +82,7 @@ func (p *provider) resolveNetworkConfig(log *zap.SugaredLogger, config anxtypes. legacyVlanIDConfig, _ := config.VlanID.MarshalJSON() if string(legacyVlanIDConfig) != `""` { if len(config.Networks) != 0 { - return nil, ErrConfigVlanIDAndNetworks + return nil, anxtypes.ErrConfigVlanIDAndNetworks } log.Info("Configuration uses the deprecated VlanID attribute, please migrate to the Networks array instead.") @@ -130,7 +129,7 @@ func (p *provider) resolveNetworkConfig(log *zap.SugaredLogger, config anxtypes. func (p *provider) resolveDiskConfig(log *zap.SugaredLogger, config anxtypes.RawConfig) (*[]resolvedDisk, error) { if config.DiskSize != 0 { if len(config.Disks) != 0 { - return nil, ErrConfigDiskSizeAndDisks + return nil, anxtypes.ErrConfigDiskSizeAndDisks } log.Info("Configuration uses the deprecated DiskSize attribute, please migrate to the Disks array instead.") diff --git a/pkg/cloudprovider/provider/anexia/types/errors.go b/pkg/cloudprovider/provider/anexia/types/errors.go deleted file mode 100644 index 65f7ab6d2..000000000 --- a/pkg/cloudprovider/provider/anexia/types/errors.go +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "fmt" - "strings" -) - -// MultiError represent multiple errors at the same time. -type MultiErrors []error - -func (r MultiErrors) Error() string { - errString := make([]string, len(r)) - for i, err := range r { - errString[i] = fmt.Sprintf("Error %d: %s", i, err) - } - return fmt.Sprintf("Multiple errors occurred:\n%s", strings.Join(errString, "\n")) -} - -func NewMultiError(errs ...error) error { - var combinedErr []error - for _, err := range errs { - if err == nil { - continue - } - combinedErr = append(combinedErr, err) - } - - if len(combinedErr) > 0 { - return MultiErrors(combinedErr) - } - - return nil -} diff --git a/pkg/userdata/convert/gzip.go b/pkg/cloudprovider/provider/aws/gzip.go similarity index 93% rename from pkg/userdata/convert/gzip.go rename to pkg/cloudprovider/provider/aws/gzip.go index a15af3f60..689fd89d4 100644 --- a/pkg/userdata/convert/gzip.go +++ b/pkg/cloudprovider/provider/aws/gzip.go @@ -14,14 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package convert +package aws import ( "bytes" "compress/gzip" ) -func GzipString(s string) (string, error) { +func gzipString(s string) (string, error) { var b bytes.Buffer gz := gzip.NewWriter(&b) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index 34b51612f..fa9b494a7 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -38,18 +38,16 @@ import ( "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - awstypes "k8c.io/machine-controller/pkg/cloudprovider/provider/aws/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/cloudprovider/util" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" - "k8c.io/machine-controller/pkg/userdata/convert" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + awstypes "k8c.io/machine-controller/sdk/cloudprovider/aws" + "k8c.io/machine-controller/sdk/net" + "k8c.io/machine-controller/sdk/providerconfig" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -104,8 +102,8 @@ var ( ec2types.VolumeTypeSt1: nil, } - amiFilters = map[providerconfigtypes.OperatingSystem]map[awstypes.CPUArchitecture]amiFilter{ - providerconfigtypes.OperatingSystemRockyLinux: { + amiFilters = map[providerconfig.OperatingSystem]map[awstypes.CPUArchitecture]amiFilter{ + providerconfig.OperatingSystemRockyLinux: { awstypes.CPUArchitectureX86_64: { description: "*Rocky-8-EC2-*.x86_64", // The AWS marketplace ID from Rocky Linux Community Platform Engineering (CPE) @@ -117,7 +115,7 @@ var ( owner: "792107900819", }, }, - providerconfigtypes.OperatingSystemAmazonLinux2: { + providerconfig.OperatingSystemAmazonLinux2: { awstypes.CPUArchitectureX86_64: { description: "Amazon Linux 2 AMI * x86_64 HVM gp2", // The AWS marketplace ID from Amazon @@ -129,7 +127,7 @@ var ( owner: "137112412989", }, }, - providerconfigtypes.OperatingSystemUbuntu: { + providerconfig.OperatingSystemUbuntu: { awstypes.CPUArchitectureX86_64: { // Be as precise as possible - otherwise we might get a nightly dev build description: "Canonical, Ubuntu, 24.04 LTS, amd64 noble image build on ????-??-??", @@ -143,7 +141,7 @@ var ( owner: "099720109477", }, }, - providerconfigtypes.OperatingSystemRHEL: { + providerconfig.OperatingSystemRHEL: { awstypes.CPUArchitectureX86_64: { // Be as precise as possible - otherwise we might get a nightly dev build description: "Provided by Red Hat, Inc.", @@ -157,7 +155,7 @@ var ( owner: "309956199498", }, }, - providerconfigtypes.OperatingSystemFlatcar: { + providerconfig.OperatingSystemFlatcar: { awstypes.CPUArchitectureX86_64: { // Be as precise as possible - otherwise we might get a nightly dev build description: "Flatcar Container Linux stable *", @@ -208,7 +206,7 @@ type amiFilter struct { productCode string } -func getDefaultAMIID(ctx context.Context, log *zap.SugaredLogger, client *ec2.Client, os providerconfigtypes.OperatingSystem, region string, cpuArchitecture awstypes.CPUArchitecture) (string, error) { +func getDefaultAMIID(ctx context.Context, log *zap.SugaredLogger, client *ec2.Client, os providerconfig.OperatingSystem, region string, cpuArchitecture awstypes.CPUArchitecture) (string, error) { cacheLock.Lock() defer cacheLock.Unlock() @@ -267,7 +265,7 @@ func getDefaultAMIID(ctx context.Context, log *zap.SugaredLogger, client *ec2.Cl return "", fmt.Errorf("could not find Image for '%s' with arch '%s'", os, cpuArchitecture) } - if os == providerconfigtypes.OperatingSystemRHEL { + if os == providerconfig.OperatingSystemRHEL { imagesOut.Images, err = filterSupportedRHELImages(imagesOut.Images) if err != nil { return "", err @@ -315,22 +313,22 @@ func getCPUArchitecture(ctx context.Context, client *ec2.Client, instanceType ec return "", errors.New("returned instance type data did not include supported architectures") } -func getDefaultRootDevicePath(os providerconfigtypes.OperatingSystem) (string, error) { +func getDefaultRootDevicePath(os providerconfig.OperatingSystem) (string, error) { const ( rootDevicePathSDA = "/dev/sda1" rootDevicePathXVDA = "/dev/xvda" ) switch os { - case providerconfigtypes.OperatingSystemUbuntu: + case providerconfig.OperatingSystemUbuntu: return rootDevicePathSDA, nil - case providerconfigtypes.OperatingSystemRockyLinux: + case providerconfig.OperatingSystemRockyLinux: return rootDevicePathSDA, nil - case providerconfigtypes.OperatingSystemRHEL: + case providerconfig.OperatingSystemRHEL: return rootDevicePathSDA, nil - case providerconfigtypes.OperatingSystemFlatcar: + case providerconfig.OperatingSystemFlatcar: return rootDevicePathXVDA, nil - case providerconfigtypes.OperatingSystemAmazonLinux2: + case providerconfig.OperatingSystemAmazonLinux2: return rootDevicePathXVDA, nil } @@ -338,8 +336,8 @@ func getDefaultRootDevicePath(os providerconfigtypes.OperatingSystem) (string, e } //gocyclo:ignore -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *awstypes.RawConfig, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfig.Config, *awstypes.RawConfig, error) { + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, nil, err } @@ -568,14 +566,14 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus } switch f := pc.Network.GetIPFamily(); f { - case util.IPFamilyUnspecified, util.IPFamilyIPv4: + case net.IPFamilyUnspecified, net.IPFamilyIPv4: // noop - case util.IPFamilyIPv6, util.IPFamilyIPv4IPv6, util.IPFamilyIPv6IPv4: + case net.IPFamilyIPv6, net.IPFamilyIPv4IPv6, net.IPFamilyIPv6IPv4: if len(vpc.Ipv6CidrBlockAssociationSet) == 0 { return fmt.Errorf("vpc %s does not have IPv6 CIDR block", ptr.Deref(vpc.VpcId, "")) } default: - return fmt.Errorf(util.ErrUnknownNetworkFamily, f) + return fmt.Errorf(net.ErrUnknownNetworkFamily, f) } dnsHostnames, err := areVpcDNSHostnamesEnabled(ctx, ec2Client, config.VpcID) @@ -694,9 +692,9 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * } } - if pc.OperatingSystem != providerconfigtypes.OperatingSystemFlatcar { + if pc.OperatingSystem != providerconfig.OperatingSystemFlatcar { // Gzip the userdata in case we don't use Flatcar - userdata, err = convert.GzipString(userdata) + userdata, err = gzipString(userdata) if err != nil { return nil, fmt.Errorf("failed to gzip the userdata") } @@ -819,7 +817,7 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine return false, err } - // (*Config, *providerconfigtypes.Config, *awstypes.RawConfig, error) + // (*Config, *providerconfig.Config, *awstypes.RawConfig, error) config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { @@ -979,12 +977,12 @@ func (d *awsInstance) ProviderID() string { return "aws:///" + *d.instance.Placement.AvailabilityZone + "/" + *d.instance.InstanceId } -func (d *awsInstance) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{ - ptr.Deref(d.instance.PublicIpAddress, ""): v1.NodeExternalIP, - ptr.Deref(d.instance.PublicDnsName, ""): v1.NodeExternalDNS, - ptr.Deref(d.instance.PrivateIpAddress, ""): v1.NodeInternalIP, - ptr.Deref(d.instance.PrivateDnsName, ""): v1.NodeInternalDNS, +func (d *awsInstance) Addresses() map[string]corev1.NodeAddressType { + addresses := map[string]corev1.NodeAddressType{ + ptr.Deref(d.instance.PublicIpAddress, ""): corev1.NodeExternalIP, + ptr.Deref(d.instance.PublicDnsName, ""): corev1.NodeExternalDNS, + ptr.Deref(d.instance.PrivateIpAddress, ""): corev1.NodeInternalIP, + ptr.Deref(d.instance.PrivateDnsName, ""): corev1.NodeInternalDNS, } for _, netInterface := range d.instance.NetworkInterfaces { @@ -993,8 +991,8 @@ func (d *awsInstance) Addresses() map[string]v1.NodeAddressType { // link-local addresses not very useful in machine status // filter them out - if !util.IsLinkLocal(ipAddr) { - addresses[ipAddr] = v1.NodeExternalIP + if !net.IsLinkLocal(ipAddr) { + addresses[ipAddr] = corev1.NodeExternalIP } } } @@ -1070,7 +1068,7 @@ func awsErrorToTerminalError(err error, msg string) error { } func setProviderSpec(rawConfig awstypes.RawConfig, provSpec clusterv1alpha1.ProviderSpec) (*runtime.RawExtension, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, err } diff --git a/pkg/cloudprovider/provider/aws/types/types.go b/pkg/cloudprovider/provider/aws/types/types.go deleted file mode 100644 index dae0a56fa..000000000 --- a/pkg/cloudprovider/provider/aws/types/types.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" -) - -type RawConfig struct { - AccessKeyID providerconfigtypes.ConfigVarString `json:"accessKeyId,omitempty"` - SecretAccessKey providerconfigtypes.ConfigVarString `json:"secretAccessKey,omitempty"` - - AssumeRoleARN providerconfigtypes.ConfigVarString `json:"assumeRoleARN,omitempty"` - AssumeRoleExternalID providerconfigtypes.ConfigVarString `json:"assumeRoleExternalID,omitempty"` - - Region providerconfigtypes.ConfigVarString `json:"region"` - AvailabilityZone providerconfigtypes.ConfigVarString `json:"availabilityZone,omitempty"` - VpcID providerconfigtypes.ConfigVarString `json:"vpcId"` - SubnetID providerconfigtypes.ConfigVarString `json:"subnetId"` - SecurityGroupIDs []providerconfigtypes.ConfigVarString `json:"securityGroupIDs,omitempty"` - InstanceProfile providerconfigtypes.ConfigVarString `json:"instanceProfile,omitempty"` - InstanceType providerconfigtypes.ConfigVarString `json:"instanceType,omitempty"` - AMI providerconfigtypes.ConfigVarString `json:"ami,omitempty"` - DiskSize int32 `json:"diskSize"` - DiskType providerconfigtypes.ConfigVarString `json:"diskType,omitempty"` - DiskIops *int32 `json:"diskIops,omitempty"` - EBSVolumeEncrypted providerconfigtypes.ConfigVarBool `json:"ebsVolumeEncrypted"` - Tags map[string]string `json:"tags,omitempty"` - AssignPublicIP *bool `json:"assignPublicIP,omitempty"` - - IsSpotInstance *bool `json:"isSpotInstance,omitempty"` - SpotInstanceConfig *SpotInstanceConfig `json:"spotInstanceConfig,omitempty"` -} - -type SpotInstanceConfig struct { - MaxPrice providerconfigtypes.ConfigVarString `json:"maxPrice,omitempty"` - PersistentRequest providerconfigtypes.ConfigVarBool `json:"persistentRequest,omitempty"` - InterruptionBehavior providerconfigtypes.ConfigVarString `json:"interruptionBehavior,omitempty"` -} - -// CPUArchitecture defines processor architectures returned by the AWS API. -type CPUArchitecture string - -const ( - CPUArchitectureARM64 CPUArchitecture = "arm64" - CPUArchitectureX86_64 CPUArchitecture = "x86_64" - CPUArchitectureI386 CPUArchitecture = "i386" -) - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/azure/create_delete_resources.go b/pkg/cloudprovider/provider/azure/create_delete_resources.go index 9b57d9d50..34742a0ae 100644 --- a/pkg/cloudprovider/provider/azure/create_delete_resources.go +++ b/pkg/cloudprovider/provider/azure/create_delete_resources.go @@ -26,7 +26,7 @@ import ( "github.com/Azure/go-autorest/autorest/to" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/cloudprovider/util" + "k8c.io/machine-controller/sdk/net" "k8s.io/apimachinery/pkg/types" ) @@ -319,7 +319,7 @@ func getVirtualNetwork(ctx context.Context, c *config) (network.VirtualNetwork, return virtualNetworksClient.Get(ctx, c.VNetResourceGroup, c.VNetName, "") } -func createOrUpdateNetworkInterface(ctx context.Context, log *zap.SugaredLogger, ifName string, machineUID types.UID, config *config, publicIP, publicIPv6 *network.PublicIPAddress, ipFamily util.IPFamily, enableAcceleratedNetworking *bool) (*network.Interface, error) { +func createOrUpdateNetworkInterface(ctx context.Context, log *zap.SugaredLogger, ifName string, machineUID types.UID, config *config, publicIP, publicIPv6 *network.PublicIPAddress, ipFamily net.IPFamily, enableAcceleratedNetworking *bool) (*network.Interface, error) { ifClient, err := getInterfacesClient(config) if err != nil { return nil, fmt.Errorf("failed to create interfaces client: %w", err) diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 96532d88e..f14cae897 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -33,19 +33,18 @@ import ( gocache "github.com/patrickmn/go-cache" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8c.io/machine-controller/pkg/cloudprovider/common/ssh" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - azuretypes "k8c.io/machine-controller/pkg/cloudprovider/provider/azure/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/cloudprovider/util" kuberneteshelper "k8c.io/machine-controller/pkg/kubernetes" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + azuretypes "k8c.io/machine-controller/sdk/cloudprovider/azure" + "k8c.io/machine-controller/sdk/net" + "k8c.io/machine-controller/sdk/providerconfig" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/ptr" ) @@ -112,11 +111,11 @@ type config struct { type azureVM struct { vm *compute.VirtualMachine - ipAddresses map[string]v1.NodeAddressType + ipAddresses map[string]corev1.NodeAddressType status instance.Status } -func (vm *azureVM) Addresses() map[string]v1.NodeAddressType { +func (vm *azureVM) Addresses() map[string]corev1.NodeAddressType { return vm.ipAddresses } @@ -140,26 +139,26 @@ func (vm *azureVM) Status() instance.Status { return vm.status } -var imageReferences = map[providerconfigtypes.OperatingSystem]compute.ImageReference{ - providerconfigtypes.OperatingSystemUbuntu: { +var imageReferences = map[providerconfig.OperatingSystem]compute.ImageReference{ + providerconfig.OperatingSystemUbuntu: { Publisher: to.StringPtr("Canonical"), Offer: to.StringPtr("ubuntu-24_04-lts"), Sku: to.StringPtr("server-gen1"), Version: to.StringPtr("latest"), }, - providerconfigtypes.OperatingSystemRHEL: { + providerconfig.OperatingSystemRHEL: { Publisher: to.StringPtr("RedHat"), Offer: to.StringPtr("rhel-byos"), Sku: to.StringPtr("rhel-lvm85"), Version: to.StringPtr("8.5.20220316"), }, - providerconfigtypes.OperatingSystemFlatcar: { + providerconfig.OperatingSystemFlatcar: { Publisher: to.StringPtr("kinvolk"), Offer: to.StringPtr("flatcar-container-linux"), Sku: to.StringPtr("stable"), Version: to.StringPtr("3374.2.0"), }, - providerconfigtypes.OperatingSystemRockyLinux: { + providerconfig.OperatingSystemRockyLinux: { Publisher: to.StringPtr("procomputers"), Offer: to.StringPtr("rocky-linux-8-5"), Sku: to.StringPtr("rocky-linux-8-5"), @@ -167,18 +166,18 @@ var imageReferences = map[providerconfigtypes.OperatingSystem]compute.ImageRefer }, } -var osPlans = map[providerconfigtypes.OperatingSystem]*compute.Plan{ - providerconfigtypes.OperatingSystemFlatcar: { +var osPlans = map[providerconfig.OperatingSystem]*compute.Plan{ + providerconfig.OperatingSystemFlatcar: { Name: ptr.To("stable"), Publisher: ptr.To("kinvolk"), Product: ptr.To("flatcar-container-linux"), }, - providerconfigtypes.OperatingSystemRHEL: { + providerconfig.OperatingSystemRHEL: { Name: ptr.To("rhel-lvm85"), Publisher: ptr.To("redhat"), Product: ptr.To("rhel-byos"), }, - providerconfigtypes.OperatingSystemRockyLinux: { + providerconfig.OperatingSystemRockyLinux: { Name: ptr.To("rocky-linux-8-5"), Publisher: ptr.To("procomputers"), Product: ptr.To("rocky-linux-8-5"), @@ -205,7 +204,7 @@ var ( cache = gocache.New(10*time.Minute, 10*time.Minute) ) -func getOSImageReference(c *config, os providerconfigtypes.OperatingSystem) (*compute.ImageReference, error) { +func getOSImageReference(c *config, os providerconfig.OperatingSystem) (*compute.ImageReference, error) { if c.ImageID != "" { return &compute.ImageReference{ ID: to.StringPtr(c.ImageID), @@ -234,8 +233,8 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes return &provider{configVarResolver: configVarResolver} } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *providerconfigtypes.Config, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *providerconfig.Config, error) { + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, err } @@ -378,9 +377,9 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p return &c, pconfig, nil } -func getVMIPAddresses(ctx context.Context, log *zap.SugaredLogger, c *config, vm *compute.VirtualMachine, ipFamily util.IPFamily) (map[string]v1.NodeAddressType, error) { +func getVMIPAddresses(ctx context.Context, log *zap.SugaredLogger, c *config, vm *compute.VirtualMachine, ipFamily net.IPFamily) (map[string]corev1.NodeAddressType, error) { var ( - ipAddresses = map[string]v1.NodeAddressType{} + ipAddresses = map[string]corev1.NodeAddressType{} err error ) @@ -412,7 +411,7 @@ func getVMIPAddresses(ctx context.Context, log *zap.SugaredLogger, c *config, vm return ipAddresses, nil } -func getNICIPAddresses(ctx context.Context, log *zap.SugaredLogger, c *config, ipFamily util.IPFamily, ifaceName string) (map[string]v1.NodeAddressType, error) { +func getNICIPAddresses(ctx context.Context, log *zap.SugaredLogger, c *config, ipFamily net.IPFamily, ifaceName string) (map[string]corev1.NodeAddressType, error) { ifClient, err := getInterfacesClient(c) if err != nil { return nil, fmt.Errorf("failed to create interfaces client: %w", err) @@ -423,7 +422,7 @@ func getNICIPAddresses(ctx context.Context, log *zap.SugaredLogger, c *config, i return nil, fmt.Errorf("failed to get interface %q: %w", ifaceName, err) } - ipAddresses := map[string]v1.NodeAddressType{} + ipAddresses := map[string]corev1.NodeAddressType{} if netIf.IPConfigurations == nil { return ipAddresses, nil @@ -448,7 +447,7 @@ func getNICIPAddresses(ctx context.Context, log *zap.SugaredLogger, c *config, i return nil, fmt.Errorf("failed to retrieve IP string for IP %q: %w", name, err) } for _, ip := range publicIPs { - ipAddresses[ip] = v1.NodeExternalIP + ipAddresses[ip] = corev1.NodeExternalIP } if ipFamily.HasIPv6() { @@ -457,7 +456,7 @@ func getNICIPAddresses(ctx context.Context, log *zap.SugaredLogger, c *config, i return nil, fmt.Errorf("failed to retrieve IP string for IP %q: %w", name, err) } for _, ip := range publicIP6s { - ipAddresses[ip] = v1.NodeExternalIP + ipAddresses[ip] = corev1.NodeExternalIP } } } @@ -467,7 +466,7 @@ func getNICIPAddresses(ctx context.Context, log *zap.SugaredLogger, c *config, i return nil, fmt.Errorf("failed to retrieve internal IP string for IP %q: %w", name, err) } for _, ip := range internalIPs { - ipAddresses[ip] = v1.NodeInternalIP + ipAddresses[ip] = corev1.NodeInternalIP } } return ipAddresses, nil @@ -522,7 +521,7 @@ func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.Machin return spec, nil } -func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) (*compute.StorageProfile, error) { +func getStorageProfile(config *config, providerCfg *providerconfig.Config) (*compute.StorageProfile, error) { osRef, err := getOSImageReference(config, providerCfg.OperatingSystem) if err != nil { return nil, fmt.Errorf("failed to get OSImageReference: %w", err) @@ -1002,14 +1001,14 @@ func (p *provider) Validate(ctx context.Context, log *zap.SugaredLogger, spec cl } switch f := providerConfig.Network.GetIPFamily(); f { - case util.IPFamilyUnspecified, util.IPFamilyIPv4: + case net.IPFamilyUnspecified, net.IPFamilyIPv4: //noop - case util.IPFamilyIPv6: - return fmt.Errorf(util.ErrIPv6OnlyUnsupported) - case util.IPFamilyIPv4IPv6, util.IPFamilyIPv6IPv4: + case net.IPFamilyIPv6: + return fmt.Errorf(net.ErrIPv6OnlyUnsupported) + case net.IPFamilyIPv4IPv6, net.IPFamilyIPv6IPv4: // validate default: - return fmt.Errorf(util.ErrUnknownNetworkFamily, f) + return fmt.Errorf(net.ErrUnknownNetworkFamily, f) } if c.PublicIPSKU != nil { @@ -1109,7 +1108,7 @@ func (p *provider) MigrateUID(ctx context.Context, log *zap.SugaredLogger, machi } if kuberneteshelper.HasFinalizer(machine, finalizerNIC) { - _, err = createOrUpdateNetworkInterface(ctx, log, ifaceName(machine), newUID, config, publicIP, publicIPv6, util.IPFamilyUnspecified, config.EnableAcceleratedNetworking) + _, err = createOrUpdateNetworkInterface(ctx, log, ifaceName(machine), newUID, config, publicIP, publicIPv6, net.IPFamilyUnspecified, config.EnableAcceleratedNetworking) if err != nil { return fmt.Errorf("failed to update UID on main network interface: %w", err) } @@ -1173,9 +1172,9 @@ func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } -func getOSUsername(os providerconfigtypes.OperatingSystem) string { +func getOSUsername(os providerconfig.OperatingSystem) string { switch os { - case providerconfigtypes.OperatingSystemFlatcar: + case providerconfig.OperatingSystemFlatcar: return "core" default: return string(os) diff --git a/pkg/cloudprovider/provider/azure/types/types.go b/pkg/cloudprovider/provider/azure/types/types.go deleted file mode 100644 index 12746fb32..000000000 --- a/pkg/cloudprovider/provider/azure/types/types.go +++ /dev/null @@ -1,77 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" -) - -// RawConfig is a direct representation of an Azure machine object's configuration. -type RawConfig struct { - SubscriptionID providerconfigtypes.ConfigVarString `json:"subscriptionID,omitempty"` - TenantID providerconfigtypes.ConfigVarString `json:"tenantID,omitempty"` - ClientID providerconfigtypes.ConfigVarString `json:"clientID,omitempty"` - ClientSecret providerconfigtypes.ConfigVarString `json:"clientSecret,omitempty"` - - Location providerconfigtypes.ConfigVarString `json:"location"` - ResourceGroup providerconfigtypes.ConfigVarString `json:"resourceGroup"` - VNetResourceGroup providerconfigtypes.ConfigVarString `json:"vnetResourceGroup"` - VMSize providerconfigtypes.ConfigVarString `json:"vmSize"` - VNetName providerconfigtypes.ConfigVarString `json:"vnetName"` - SubnetName providerconfigtypes.ConfigVarString `json:"subnetName"` - LoadBalancerSku providerconfigtypes.ConfigVarString `json:"loadBalancerSku"` - RouteTableName providerconfigtypes.ConfigVarString `json:"routeTableName"` - AvailabilitySet providerconfigtypes.ConfigVarString `json:"availabilitySet"` - AssignAvailabilitySet *bool `json:"assignAvailabilitySet"` - SecurityGroupName providerconfigtypes.ConfigVarString `json:"securityGroupName"` - Zones []string `json:"zones"` - ImagePlan *ImagePlan `json:"imagePlan,omitempty"` - ImageReference *ImageReference `json:"imageReference,omitempty"` - EnableAcceleratedNetworking *bool `json:"enableAcceleratedNetworking"` - EnableBootDiagnostics *bool `json:"enableBootDiagnostics,omitempty"` - - ImageID providerconfigtypes.ConfigVarString `json:"imageID"` - OSDiskSize int32 `json:"osDiskSize"` - OSDiskSKU *string `json:"osDiskSKU,omitempty"` - DataDiskSize int32 `json:"dataDiskSize"` - DataDiskSKU *string `json:"dataDiskSKU,omitempty"` - AssignPublicIP providerconfigtypes.ConfigVarBool `json:"assignPublicIP"` - PublicIPSKU *string `json:"publicIPSKU,omitempty"` - Tags map[string]string `json:"tags,omitempty"` -} - -// ImagePlan contains azure OS Plan fields for the marketplace images. -type ImagePlan struct { - Name string `json:"name,omitempty"` - Publisher string `json:"publisher,omitempty"` - Product string `json:"product,omitempty"` -} - -// ImageReference specifies information about the image to use. -type ImageReference struct { - Publisher string `json:"publisher,omitempty"` - Offer string `json:"offer,omitempty"` - Sku string `json:"sku,omitempty"` - Version string `json:"version,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/baremetal/plugins/driver.go b/pkg/cloudprovider/provider/baremetal/plugins/driver.go index 79f9d5d36..e42090cb5 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/driver.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/driver.go @@ -18,23 +18,13 @@ package plugins import ( "context" + "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) -type Driver string - -const Tinkerbell Driver = "tinkerbell" - -type CloudConfigSettings struct { - Token string - Namespace string - SecretName string - ClusterHost string -} - // PluginDriver manages the communications between the machine controller cloud provider and the bare metal env. type PluginDriver interface { GetServer(context.Context) (Server, error) diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/hardware.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/hardware.go index e9ce4695f..387a7f157 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/hardware.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/hardware.go @@ -21,20 +21,21 @@ import ( "fmt" tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/errors" tbtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) // HardwareClient manages Tinkerbell hardware resources across two clusters. type HardwareClient struct { - TinkerbellClient client.Client + TinkerbellClient ctrlruntimeclient.Client } // NewHardwareClient creates a new instance of HardwareClient. -func NewHardwareClient(tinkerbellClient client.Client) *HardwareClient { +func NewHardwareClient(tinkerbellClient ctrlruntimeclient.Client) *HardwareClient { return &HardwareClient{ TinkerbellClient: tinkerbellClient, } @@ -44,8 +45,8 @@ func NewHardwareClient(tinkerbellClient client.Client) *HardwareClient { // deployment object. func (h *HardwareClient) GetHardware(ctx context.Context, hardwareRef types.NamespacedName) (*tinkv1alpha1.Hardware, error) { hardware := &tinkv1alpha1.Hardware{} - if err := h.TinkerbellClient.Get(ctx, client.ObjectKey{Namespace: hardwareRef.Namespace, Name: hardwareRef.Name}, hardware); err != nil { - return nil, fmt.Errorf("failed to get hardware '%s' in namespace '%s': %w", hardwareRef.Name, hardwareRef.Namespace, err) + if err := h.TinkerbellClient.Get(ctx, hardwareRef, hardware); err != nil { + return nil, fmt.Errorf("failed to get hardware '%v': %w", hardwareRef, err) } return hardware, nil diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go index 7ea237ea6..4bc1cffaf 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go @@ -23,10 +23,10 @@ import ( tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" "gopkg.in/yaml.v3" - kerrors "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) type Task struct { @@ -66,11 +66,11 @@ const ( // TemplateClient handles interactions with the Tinkerbell Templates in the Tinkerbell cluster. type TemplateClient struct { - tinkclient client.Client + tinkclient ctrlruntimeclient.Client } // NewTemplateClient creates a new client for managing Tinkerbell Templates. -func NewTemplateClient(k8sClient client.Client) *TemplateClient { +func NewTemplateClient(k8sClient ctrlruntimeclient.Client) *TemplateClient { return &TemplateClient{ tinkclient: k8sClient, } @@ -98,7 +98,7 @@ func (t *TemplateClient) CreateTemplate(ctx context.Context, namespace string) e Name: ProvisionWorkerNodeTemplate, Namespace: namespace, }, template); err != nil { - if kerrors.IsNotFound(err) { + if apierrors.IsNotFound(err) { data, err := getTemplate(OSImageURL) if err != nil { return err diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go index 53f8b9d9a..7135356b9 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/workflow.go @@ -22,13 +22,14 @@ import ( "fmt" "time" + tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" + tink "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" - tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" - kerrors "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) // DefaultPartitionNumber defines the default value for the "partition_number" field. @@ -39,11 +40,11 @@ const PartitionNumberAnnotation = "hardware.kubermatic.io/partition-number" // WorkflowClient handles interactions with the Tinkerbell Workflows. type WorkflowClient struct { - tinkclient client.Client + tinkclient ctrlruntimeclient.Client } // NewWorkflowClient creates a new client for managing Tinkerbell workflows. -func NewWorkflowClient(k8sClient client.Client) *WorkflowClient { +func NewWorkflowClient(k8sClient ctrlruntimeclient.Client) *WorkflowClient { return &WorkflowClient{ tinkclient: k8sClient, } @@ -96,7 +97,7 @@ func (w *WorkflowClient) CreateWorkflow(ctx context.Context, userData, templateR // GetWorkflow retrieves a Tinkerbell Workflow resource from the cluster. func (w *WorkflowClient) GetWorkflow(ctx context.Context, name string, namespace string) (*tinkv1alpha1.Workflow, error) { workflow := &tinkv1alpha1.Workflow{} - if err := w.tinkclient.Get(ctx, client.ObjectKey{Name: name, Namespace: namespace}, workflow); err != nil { + if err := w.tinkclient.Get(ctx, ctrlruntimeclient.ObjectKey{Name: name, Namespace: namespace}, workflow); err != nil { return nil, fmt.Errorf("failed to get workflow: %w", err) } return workflow, nil @@ -106,13 +107,13 @@ func (w *WorkflowClient) GetWorkflow(ctx context.Context, name string, namespace // state, to avoid the situation of re-running a workflow for a de-provisioned machine. func (w *WorkflowClient) CleanupWorkflows(ctx context.Context, hardwareName, namespace string) error { workflows := &tinkv1alpha1.WorkflowList{} - if err := w.tinkclient.List(ctx, workflows, &client.ListOptions{ + if err := w.tinkclient.List(ctx, workflows, &ctrlruntimeclient.ListOptions{ Namespace: namespace, LabelSelector: labels.SelectorFromSet(map[string]string{ tink.HardwareRefLabel: hardwareName, }), }); err != nil { - if kerrors.IsNotFound(err) { + if apierrors.IsNotFound(err) { return nil } @@ -123,7 +124,7 @@ func (w *WorkflowClient) CleanupWorkflows(ctx context.Context, hardwareName, nam if workflow.Status.State == tinkv1alpha1.WorkflowStatePending || workflow.Status.State == tinkv1alpha1.WorkflowStateTimeout { if err := w.tinkclient.Delete(ctx, &workflow); err != nil { - if !kerrors.IsNotFound(err) { + if !apierrors.IsNotFound(err) { return fmt.Errorf("failed to delete workflow: %w", err) } } diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go index 4b1e162a4..941ba0148 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/driver.go @@ -22,18 +22,18 @@ import ( "fmt" "github.com/aws/smithy-go/ptr" + tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" "go.uber.org/zap" - cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" - - tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" + cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins" "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client" - tinktypes "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" + tinkerbelltypes "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" + tinktypes "k8c.io/machine-controller/sdk/cloudprovider/baremetal/plugins/tinkerbell" + providerconfigtypes "k8c.io/machine-controller/sdk/providerconfig" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" "k8s.io/kubectl/pkg/scheme" @@ -94,7 +94,7 @@ func (d *driver) GetServer(ctx context.Context) (plugins.Server, error) { return nil, cloudprovidererrors.ErrInstanceNotFound } - server := tinktypes.Hardware{Hardware: targetHardware} + server := tinkerbelltypes.Hardware{Hardware: targetHardware} return &server, nil } @@ -129,7 +129,7 @@ func (d *driver) ProvisionServer(ctx context.Context, _ *zap.SugaredLogger, meta } // Create Workflow to match the template and server - server := tinktypes.Hardware{Hardware: hardware} + server := tinkerbelltypes.Hardware{Hardware: hardware} if err = d.WorkflowClient.CreateWorkflow(ctx, userdata, client.ProvisionWorkerNodeTemplate, d.OSImageURL, server); err != nil { return nil, err } diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/hardware.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/hardware.go index b52434f35..6263dd675 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/hardware.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/hardware.go @@ -18,9 +18,19 @@ package types import ( tinkv1alpha1 "github.com/tinkerbell/tink/api/v1alpha1" + "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins" ) +const ( + Unknown string = "Unknown" + Staged string = "Staged" + Provisioned string = "Provisioned" + Decommissioned string = "Decommissioned" + + HardwareRefLabel = "app.kubernetes.io/hardware-reference" +) + type Hardware struct { *tinkv1alpha1.Hardware `json:"hardware"` } diff --git a/pkg/cloudprovider/provider/baremetal/provider.go b/pkg/cloudprovider/provider/baremetal/provider.go index f4e18ed1f..22c4ab50b 100644 --- a/pkg/cloudprovider/provider/baremetal/provider.go +++ b/pkg/cloudprovider/provider/baremetal/provider.go @@ -24,21 +24,21 @@ import ( "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins" tink "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell" - tinktypes "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types" - baremetaltypes "k8c.io/machine-controller/pkg/cloudprovider/provider/baremetal/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" "k8c.io/machine-controller/pkg/cloudprovider/util" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + baremetaltypes "k8c.io/machine-controller/sdk/cloudprovider/baremetal" + plugintypes "k8c.io/machine-controller/sdk/cloudprovider/baremetal/plugins" + tinktypes "k8c.io/machine-controller/sdk/cloudprovider/baremetal/plugins/tinkerbell" + "k8c.io/machine-controller/sdk/providerconfig" corev1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ) @@ -83,16 +83,16 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes type Config struct { driver plugins.PluginDriver - driverName plugins.Driver + driverName plugintypes.Driver driverSpec runtime.RawExtension } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfig.Config, error) { if provSpec.Value == nil { return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig, err := providerconfigtypes.GetConfig(provSpec) + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, err } @@ -112,12 +112,12 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if err != nil { return nil, nil, fmt.Errorf("failed to get baremetal provider's driver name: %w", err) } - c.driverName = plugins.Driver(driverName) + c.driverName = plugintypes.Driver(driverName) c.driverSpec = rawConfig.DriverSpec switch c.driverName { - case plugins.Tinkerbell: + case plugintypes.Tinkerbell: driverConfig := &tinktypes.TinkerbellPluginSpec{} if err := json.Unmarshal(c.driverSpec.Raw, &driverConfig); err != nil { @@ -224,7 +224,7 @@ func (p provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *cl secret := &corev1.Secret{} if err := data.Client.Get(ctx, types.NamespacedName{Namespace: util.CloudInitNamespace, Name: machine.Name}, secret); err != nil { - if !kerrors.IsNotFound(err) { + if !apierrors.IsNotFound(err) { return false, fmt.Errorf("failed to fetching secret for userdata: %w", err) } diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index e8d03129e..794a4a2a7 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -28,18 +28,17 @@ import ( "go.uber.org/zap" "golang.org/x/oauth2" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8c.io/machine-controller/pkg/cloudprovider/common/ssh" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - digitaloceantypes "k8c.io/machine-controller/pkg/cloudprovider/provider/digitalocean/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/cloudprovider/util" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + digitaloceantypes "k8c.io/machine-controller/sdk/cloudprovider/digitalocean" + "k8c.io/machine-controller/sdk/net" + "k8c.io/machine-controller/sdk/providerconfig" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" @@ -82,14 +81,14 @@ func (t *TokenSource) Token() (*oauth2.Token, error) { return token, nil } -func getSlugForOS(os providerconfigtypes.OperatingSystem) (string, error) { +func getSlugForOS(os providerconfig.OperatingSystem) (string, error) { switch os { - case providerconfigtypes.OperatingSystemUbuntu: + case providerconfig.OperatingSystemUbuntu: return "ubuntu-24-04-x64", nil - case providerconfigtypes.OperatingSystemRockyLinux: + case providerconfig.OperatingSystemRockyLinux: return "rockylinux-8-x64", nil } - return "", providerconfigtypes.ErrOSNotSupported + return "", providerconfig.ErrOSNotSupported } func getClient(ctx context.Context, token string) *godo.Client { @@ -101,8 +100,8 @@ func getClient(ctx context.Context, token string) *godo.Client { return godo.NewClient(oauthClient) } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfig.Config, error) { + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, err } @@ -184,14 +183,14 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus } switch f := pc.Network.GetIPFamily(); f { - case util.IPFamilyUnspecified, util.IPFamilyIPv4: + case net.IPFamilyUnspecified, net.IPFamilyIPv4: // noop - case util.IPFamilyIPv6: - return fmt.Errorf(util.ErrIPv6OnlyUnsupported) - case util.IPFamilyIPv4IPv6, util.IPFamilyIPv6IPv4: + case net.IPFamilyIPv6: + return fmt.Errorf(net.ErrIPv6OnlyUnsupported) + case net.IPFamilyIPv4IPv6, net.IPFamilyIPv6IPv4: // noop default: - return fmt.Errorf(util.ErrUnknownNetworkFamily, f) + return fmt.Errorf(net.ErrUnknownNetworkFamily, f) } client := getClient(ctx, c.Token) @@ -502,20 +501,20 @@ func (d *doInstance) ProviderID() string { return fmt.Sprintf("digitalocean://%d", d.droplet.ID) } -func (d *doInstance) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{} +func (d *doInstance) Addresses() map[string]corev1.NodeAddressType { + addresses := map[string]corev1.NodeAddressType{} for _, n := range d.droplet.Networks.V4 { if n.Type == "public" { - addresses[n.IPAddress] = v1.NodeExternalIP + addresses[n.IPAddress] = corev1.NodeExternalIP } else { - addresses[n.IPAddress] = v1.NodeInternalIP + addresses[n.IPAddress] = corev1.NodeInternalIP } } for _, n := range d.droplet.Networks.V6 { if n.Type == "public" { - addresses[n.IPAddress] = v1.NodeExternalIP + addresses[n.IPAddress] = corev1.NodeExternalIP } else { - addresses[n.IPAddress] = v1.NodeInternalIP + addresses[n.IPAddress] = corev1.NodeInternalIP } } return addresses diff --git a/pkg/cloudprovider/provider/digitalocean/types/types.go b/pkg/cloudprovider/provider/digitalocean/types/types.go deleted file mode 100644 index 89175c734..000000000 --- a/pkg/cloudprovider/provider/digitalocean/types/types.go +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" -) - -type RawConfig struct { - Token providerconfigtypes.ConfigVarString `json:"token,omitempty"` - Region providerconfigtypes.ConfigVarString `json:"region"` - Size providerconfigtypes.ConfigVarString `json:"size"` - Backups providerconfigtypes.ConfigVarBool `json:"backups"` - IPv6 providerconfigtypes.ConfigVarBool `json:"ipv6"` - PrivateNetworking providerconfigtypes.ConfigVarBool `json:"private_networking"` - Monitoring providerconfigtypes.ConfigVarBool `json:"monitoring"` - Tags []providerconfigtypes.ConfigVarString `json:"tags,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/edge/provider.go b/pkg/cloudprovider/provider/edge/provider.go index ed3b2cd98..e9f1e3008 100644 --- a/pkg/cloudprovider/provider/edge/provider.go +++ b/pkg/cloudprovider/provider/edge/provider.go @@ -21,10 +21,10 @@ import ( "go.uber.org/zap" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8c.io/machine-controller/pkg/cloudprovider/instance" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/providerconfig" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + "k8c.io/machine-controller/sdk/providerconfig" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index faaba8db7..b759fb5fe 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -28,14 +28,13 @@ import ( "github.com/equinix/equinix-sdk-go/services/metalv1" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - equinixmetaltypes "k8c.io/machine-controller/pkg/cloudprovider/provider/equinixmetal/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + equinixmetaltypes "k8c.io/machine-controller/sdk/cloudprovider/equinixmetal" + "k8c.io/machine-controller/sdk/providerconfig" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -81,8 +80,8 @@ type provider struct { configVarResolver *providerconfig.ConfigVarResolver } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *equinixmetaltypes.RawConfig, *providerconfigtypes.Config, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *equinixmetaltypes.RawConfig, *providerconfig.Config, error) { + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, nil, err } @@ -481,7 +480,7 @@ func setProviderSpec(rawConfig equinixmetaltypes.RawConfig, s clusterv1alpha1.Pr return nil, fmt.Errorf("machine.spec.providerconfig.value is nil") } - pconfig, err := providerconfigtypes.GetConfig(s) + pconfig, err := providerconfig.GetConfig(s) if err != nil { return nil, err } @@ -521,16 +520,16 @@ func getDeviceByTag(ctx context.Context, client *metalv1.APIClient, projectID, t } // given a defined Kubermatic constant for an operating system, return the canonical slug for Equinix Metal. -func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { +func getNameForOS(os providerconfig.OperatingSystem) (string, error) { switch os { - case providerconfigtypes.OperatingSystemUbuntu: + case providerconfig.OperatingSystemUbuntu: return "ubuntu_24_04", nil - case providerconfigtypes.OperatingSystemFlatcar: + case providerconfig.OperatingSystemFlatcar: return "flatcar_stable", nil - case providerconfigtypes.OperatingSystemRockyLinux: + case providerconfig.OperatingSystemRockyLinux: return "rocky_8", nil } - return "", providerconfigtypes.ErrOSNotSupported + return "", providerconfig.ErrOSNotSupported } func getClient(apiKey string) *metalv1.APIClient { diff --git a/pkg/cloudprovider/provider/external/provider.go b/pkg/cloudprovider/provider/external/provider.go index 73d595814..a881360fc 100644 --- a/pkg/cloudprovider/provider/external/provider.go +++ b/pkg/cloudprovider/provider/external/provider.go @@ -18,12 +18,14 @@ package external import ( "context" + "go.uber.org/zap" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8c.io/machine-controller/pkg/cloudprovider/instance" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/providerconfig" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + "k8c.io/machine-controller/sdk/providerconfig" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" ) diff --git a/pkg/cloudprovider/provider/fake/provider.go b/pkg/cloudprovider/provider/fake/provider.go index 219050aff..1ddbf68ae 100644 --- a/pkg/cloudprovider/provider/fake/provider.go +++ b/pkg/cloudprovider/provider/fake/provider.go @@ -23,11 +23,10 @@ import ( "go.uber.org/zap" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8c.io/machine-controller/pkg/cloudprovider/instance" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + "k8c.io/machine-controller/sdk/providerconfig" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -72,7 +71,7 @@ func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.Machin // Validate returns success or failure based according to its FakeCloudProviderSpec. func (p *provider) Validate(_ context.Context, log *zap.SugaredLogger, machinespec clusterv1alpha1.MachineSpec) error { - pconfig, err := providerconfigtypes.GetConfig(machinespec.ProviderSpec) + pconfig, err := providerconfig.GetConfig(machinespec.ProviderSpec) if err != nil { return err } diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index c3faaeb56..c8561d245 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -32,10 +32,9 @@ import ( googleoauth "golang.org/x/oauth2/google" "google.golang.org/api/compute/v1" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" - gcetypes "k8c.io/machine-controller/pkg/cloudprovider/provider/gce/types" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + gcetypes "k8c.io/machine-controller/sdk/cloudprovider/gce" + "k8c.io/machine-controller/sdk/providerconfig" ) // Environment variables for the configuration of the Google Cloud project access. @@ -44,15 +43,15 @@ const ( ) // imageProjects maps the OS to the Google Cloud image projects. -var imageProjects = map[providerconfigtypes.OperatingSystem]string{ - providerconfigtypes.OperatingSystemUbuntu: "ubuntu-os-cloud", - providerconfigtypes.OperatingSystemFlatcar: "kinvolk-public", +var imageProjects = map[providerconfig.OperatingSystem]string{ + providerconfig.OperatingSystemUbuntu: "ubuntu-os-cloud", + providerconfig.OperatingSystemFlatcar: "kinvolk-public", } // imageFamilies maps the OS to the Google Cloud image projects. -var imageFamilies = map[providerconfigtypes.OperatingSystem]string{ - providerconfigtypes.OperatingSystemUbuntu: "ubuntu-2404-lts-amd64", - providerconfigtypes.OperatingSystemFlatcar: "flatcar-stable", +var imageFamilies = map[providerconfig.OperatingSystem]string{ + providerconfig.OperatingSystemUbuntu: "ubuntu-2404-lts-amd64", + providerconfig.OperatingSystemFlatcar: "flatcar-stable", } // diskTypes are the disk types of the Google Cloud. Map is used for @@ -70,9 +69,9 @@ const ( // newCloudProviderSpec creates a cloud provider specification out of the // given ProviderSpec. -func newCloudProviderSpec(provSpec v1alpha1.ProviderSpec) (*gcetypes.CloudProviderSpec, *providerconfigtypes.Config, error) { +func newCloudProviderSpec(provSpec clusterv1alpha1.ProviderSpec) (*gcetypes.CloudProviderSpec, *providerconfig.Config, error) { // Retrieve provider configuration from machine specification. - pconfig, err := providerconfigtypes.GetConfig(provSpec) + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, fmt.Errorf("cannot unmarshal machine.spec.providerconfig.value: %w", err) } @@ -105,7 +104,7 @@ type config struct { provisioningModel *string labels map[string]string tags []string - providerConfig *providerconfigtypes.Config + providerConfig *providerconfig.Config assignPublicIPAddress bool multizone bool regional bool @@ -123,7 +122,7 @@ type clientConfig struct { } // newConfig creates a Provider configuration out of the passed resolver and spec. -func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.ProviderSpec) (*config, error) { +func newConfig(resolver *providerconfig.ConfigVarResolver, spec clusterv1alpha1.ProviderSpec) (*config, error) { // Create cloud provider spec. cpSpec, providerConfig, err := newCloudProviderSpec(spec) if err != nil { @@ -311,11 +310,11 @@ func (cfg *config) sourceImageDescriptor() (string, error) { } project, ok := imageProjects[cfg.providerConfig.OperatingSystem] if !ok { - return "", providerconfigtypes.ErrOSNotSupported + return "", providerconfig.ErrOSNotSupported } family, ok := imageFamilies[cfg.providerConfig.OperatingSystem] if !ok { - return "", providerconfigtypes.ErrOSNotSupported + return "", providerconfig.ErrOSNotSupported } return fmt.Sprintf("projects/%s/global/images/family/%s", project, family), nil } diff --git a/pkg/cloudprovider/provider/gce/instance.go b/pkg/cloudprovider/provider/gce/instance.go index 139556775..5259ce8b2 100644 --- a/pkg/cloudprovider/provider/gce/instance.go +++ b/pkg/cloudprovider/provider/gce/instance.go @@ -28,7 +28,7 @@ import ( "k8c.io/machine-controller/pkg/cloudprovider/instance" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" ) // Possible instance statuses. @@ -68,15 +68,15 @@ func (gi *googleInstance) ProviderID() string { } // Addresses implements instance.Instance. -func (gi *googleInstance) Addresses() map[string]v1.NodeAddressType { - addrs := map[string]v1.NodeAddressType{} +func (gi *googleInstance) Addresses() map[string]corev1.NodeAddressType { + addrs := map[string]corev1.NodeAddressType{} for _, ifc := range gi.ci.NetworkInterfaces { - addrs[ifc.NetworkIP] = v1.NodeInternalIP + addrs[ifc.NetworkIP] = corev1.NodeInternalIP for _, ac := range ifc.AccessConfigs { - addrs[ac.NatIP] = v1.NodeExternalIP + addrs[ac.NatIP] = corev1.NodeExternalIP } for _, ac := range ifc.Ipv6AccessConfigs { - addrs[ac.ExternalIpv6] = v1.NodeExternalIP + addrs[ac.ExternalIpv6] = corev1.NodeExternalIP } } @@ -86,17 +86,17 @@ func (gi *googleInstance) Addresses() map[string]v1.NodeAddressType { // Zonal DNS is present for newer projects and has the following FQDN format: // [INSTANCE_NAME].[ZONE].c.[PROJECT_ID].internal zonalDNS := fmt.Sprintf("%s.%s.c.%s.internal", gi.ci.Name, gi.zone, gi.projectID) - addrs[zonalDNS] = v1.NodeInternalDNS + addrs[zonalDNS] = corev1.NodeInternalDNS // Global DNS is present for older projects and has the following FQDN format: // [INSTANCE_NAME].c.[PROJECT_ID].internal globalDNS := fmt.Sprintf("%s.c.%s.internal", gi.ci.Name, gi.projectID) - addrs[globalDNS] = v1.NodeInternalDNS + addrs[globalDNS] = corev1.NodeInternalDNS // GCP provides the search paths to resolve the machine's name, // so we add is as a DNS name // https://cloud.google.com/compute/docs/internal-dns#resolv.conf - addrs[gi.ci.Name] = v1.NodeInternalDNS + addrs[gi.ci.Name] = corev1.NodeInternalDNS return addrs } diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 1f18468ad..dc115822a 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -33,13 +33,13 @@ import ( compute "google.golang.org/api/compute/v1" "google.golang.org/api/googleapi" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/cloudprovider/util" - "k8c.io/machine-controller/pkg/providerconfig" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + "k8c.io/machine-controller/sdk/net" + "k8c.io/machine-controller/sdk/providerconfig" "k8s.io/apimachinery/pkg/types" ) @@ -116,13 +116,13 @@ func (p *Provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec cluste } switch cfg.providerConfig.Network.GetIPFamily() { - case util.IPFamilyUnspecified, util.IPFamilyIPv4: + case net.IPFamilyUnspecified, net.IPFamilyIPv4: // noop - case util.IPFamilyIPv6: - return newError(common.InvalidConfigurationMachineError, util.ErrIPv6OnlyUnsupported) - case util.IPFamilyIPv4IPv6, util.IPFamilyIPv6IPv4: + case net.IPFamilyIPv6: + return newError(common.InvalidConfigurationMachineError, net.ErrIPv6OnlyUnsupported) + case net.IPFamilyIPv4IPv6, net.IPFamilyIPv6IPv4: default: - return newError(common.InvalidConfigurationMachineError, util.ErrUnknownNetworkFamily, cfg.providerConfig.Network.GetIPFamily()) + return newError(common.InvalidConfigurationMachineError, net.ErrUnknownNetworkFamily, cfg.providerConfig.Network.GetIPFamily()) } if cfg.machineType == "" { diff --git a/pkg/cloudprovider/provider/gce/provider_test.go b/pkg/cloudprovider/provider/gce/provider_test.go index 13ccf5368..945776c9c 100644 --- a/pkg/cloudprovider/provider/gce/provider_test.go +++ b/pkg/cloudprovider/provider/gce/provider_test.go @@ -26,8 +26,8 @@ import ( "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" - "k8c.io/machine-controller/pkg/providerconfig" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + "k8c.io/machine-controller/sdk/providerconfig" "k8s.io/apimachinery/pkg/runtime" fake2 "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -126,13 +126,13 @@ func TestValidate(t *testing.T) { p := New(providerconfig.NewConfigVarResolver(context.Background(), fake2.NewClientBuilder().Build())) tests := []struct { name string - mspec v1alpha1.MachineSpec + mspec clusterv1alpha1.MachineSpec expectErr bool }{ { "without IP family", - v1alpha1.MachineSpec{ - ProviderSpec: v1alpha1.ProviderSpec{ + clusterv1alpha1.MachineSpec{ + ProviderSpec: clusterv1alpha1.ProviderSpec{ Value: &runtime.RawExtension{ Raw: rawBytes(testProviderSpec()), }, @@ -142,8 +142,8 @@ func TestValidate(t *testing.T) { }, { "empty IP family", - v1alpha1.MachineSpec{ - ProviderSpec: v1alpha1.ProviderSpec{ + clusterv1alpha1.MachineSpec{ + ProviderSpec: clusterv1alpha1.ProviderSpec{ Value: &runtime.RawExtension{ Raw: rawBytes(testMap(testProviderSpec()). with("network.ipFamily", ""), @@ -155,8 +155,8 @@ func TestValidate(t *testing.T) { }, { "with IP family", - v1alpha1.MachineSpec{ - ProviderSpec: v1alpha1.ProviderSpec{ + clusterv1alpha1.MachineSpec{ + ProviderSpec: clusterv1alpha1.ProviderSpec{ Value: &runtime.RawExtension{ Raw: rawBytes(testMap(testProviderSpec()). with("network.ipFamily", "IPv4+IPv6"), diff --git a/pkg/cloudprovider/provider/gce/types/types.go b/pkg/cloudprovider/provider/gce/types/types.go deleted file mode 100644 index bb99621fa..000000000 --- a/pkg/cloudprovider/provider/gce/types/types.go +++ /dev/null @@ -1,86 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "encoding/json" - "fmt" - - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" - - "k8s.io/apimachinery/pkg/runtime" -) - -// CloudProviderSpec contains the specification of the cloud provider taken -// from the provider configuration. -type CloudProviderSpec struct { - // ServiceAccount must be base64-encoded. - ServiceAccount providerconfigtypes.ConfigVarString `json:"serviceAccount,omitempty"` - Zone providerconfigtypes.ConfigVarString `json:"zone"` - MachineType providerconfigtypes.ConfigVarString `json:"machineType"` - DiskSize int64 `json:"diskSize"` - DiskType providerconfigtypes.ConfigVarString `json:"diskType"` - Network providerconfigtypes.ConfigVarString `json:"network"` - Subnetwork providerconfigtypes.ConfigVarString `json:"subnetwork"` - Preemptible providerconfigtypes.ConfigVarBool `json:"preemptible"` - AutomaticRestart *providerconfigtypes.ConfigVarBool `json:"automaticRestart,omitempty"` - ProvisioningModel *providerconfigtypes.ConfigVarString `json:"provisioningModel,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - Tags []string `json:"tags,omitempty"` - AssignPublicIPAddress *providerconfigtypes.ConfigVarBool `json:"assignPublicIPAddress,omitempty"` - MultiZone providerconfigtypes.ConfigVarBool `json:"multizone"` - Regional providerconfigtypes.ConfigVarBool `json:"regional"` - CustomImage providerconfigtypes.ConfigVarString `json:"customImage,omitempty"` - DisableMachineServiceAccount providerconfigtypes.ConfigVarBool `json:"disableMachineServiceAccount,omitempty"` - EnableNestedVirtualization providerconfigtypes.ConfigVarBool `json:"enableNestedVirtualization,omitempty"` - MinCPUPlatform providerconfigtypes.ConfigVarString `json:"minCPUPlatform,omitempty"` - GuestOSFeatures []string `json:"guestOSFeatures,omitempty"` - ProjectID providerconfigtypes.ConfigVarString `json:"projectID,omitempty"` -} - -// UpdateProviderSpec updates the given provider spec with changed -// configuration values. -func (cpSpec *CloudProviderSpec) UpdateProviderSpec(spec v1alpha1.ProviderSpec) (*runtime.RawExtension, error) { - if spec.Value == nil { - return nil, fmt.Errorf("machine.spec.providerconfig.value is nil") - } - providerConfig := providerconfigtypes.Config{} - err := json.Unmarshal(spec.Value.Raw, &providerConfig) - if err != nil { - return nil, err - } - rawCPSpec, err := json.Marshal(cpSpec) - if err != nil { - return nil, err - } - providerConfig.CloudProviderSpec = runtime.RawExtension{Raw: rawCPSpec} - rawProviderConfig, err := json.Marshal(providerConfig) - if err != nil { - return nil, err - } - return &runtime.RawExtension{Raw: rawProviderConfig}, nil -} - -type RawConfig = CloudProviderSpec - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 93dc87383..3e9635a53 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -27,17 +27,16 @@ import ( "github.com/hetznercloud/hcloud-go/v2/hcloud" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8c.io/machine-controller/pkg/cloudprovider/common/ssh" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - hetznertypes "k8c.io/machine-controller/pkg/cloudprovider/provider/hetzner/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + hetznertypes "k8c.io/machine-controller/sdk/cloudprovider/hetzner" + "k8c.io/machine-controller/sdk/providerconfig" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/rand" ) @@ -69,22 +68,22 @@ type Config struct { AssignIPv6 bool } -func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { +func getNameForOS(os providerconfig.OperatingSystem) (string, error) { switch os { - case providerconfigtypes.OperatingSystemUbuntu: + case providerconfig.OperatingSystemUbuntu: return "ubuntu-24.04", nil - case providerconfigtypes.OperatingSystemRockyLinux: + case providerconfig.OperatingSystemRockyLinux: return "rocky-8", nil } - return "", providerconfigtypes.ErrOSNotSupported + return "", providerconfig.ErrOSNotSupported } func getClient(token string) *hcloud.Client { return hcloud.NewClient(hcloud.WithToken(token)) } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfig.Config, error) { + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, err } @@ -555,20 +554,20 @@ func (s *hetznerServer) ProviderID() string { return fmt.Sprintf("hcloud://%d", s.server.ID) } -func (s *hetznerServer) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{} +func (s *hetznerServer) Addresses() map[string]corev1.NodeAddressType { + addresses := map[string]corev1.NodeAddressType{} for _, fips := range s.server.PublicNet.FloatingIPs { - addresses[fips.IP.String()] = v1.NodeExternalIP + addresses[fips.IP.String()] = corev1.NodeExternalIP } for _, privateNetwork := range s.server.PrivateNet { - addresses[privateNetwork.IP.String()] = v1.NodeInternalIP + addresses[privateNetwork.IP.String()] = corev1.NodeInternalIP } - addresses[s.server.PublicNet.IPv4.IP.String()] = v1.NodeExternalIP + addresses[s.server.PublicNet.IPv4.IP.String()] = corev1.NodeExternalIP // For a given IPv6 network of 2001:db8:1234::/64, the instance address is 2001:db8:1234::1 // Reference: https://github.com/hetznercloud/hcloud-cloud-controller-manager/blob/v1.12.1/hcloud/instances.go#L165-167 if s.server.PublicNet.IPv6.IP != nil && !s.server.PublicNet.IPv6.IP.IsUnspecified() { s.server.PublicNet.IPv6.IP[len(s.server.PublicNet.IPv6.IP)-1] |= 0x01 - addresses[s.server.PublicNet.IPv6.IP.String()] = v1.NodeExternalIP + addresses[s.server.PublicNet.IPv6.IP.String()] = corev1.NodeExternalIP } return addresses } diff --git a/pkg/cloudprovider/provider/hetzner/types/types.go b/pkg/cloudprovider/provider/hetzner/types/types.go deleted file mode 100644 index 0312d2a45..000000000 --- a/pkg/cloudprovider/provider/hetzner/types/types.go +++ /dev/null @@ -1,42 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" -) - -type RawConfig struct { - Token providerconfigtypes.ConfigVarString `json:"token,omitempty"` - ServerType providerconfigtypes.ConfigVarString `json:"serverType"` - Datacenter providerconfigtypes.ConfigVarString `json:"datacenter"` - Image providerconfigtypes.ConfigVarString `json:"image"` - Location providerconfigtypes.ConfigVarString `json:"location"` - PlacementGroupPrefix providerconfigtypes.ConfigVarString `json:"placementGroupPrefix"` - Networks []providerconfigtypes.ConfigVarString `json:"networks"` - Firewalls []providerconfigtypes.ConfigVarString `json:"firewalls"` - Labels map[string]string `json:"labels,omitempty"` - AssignPublicIPv4 providerconfigtypes.ConfigVarBool `json:"assignPublicIPv4,omitempty"` - AssignPublicIPv6 providerconfigtypes.ConfigVarBool `json:"assignPublicIPv6,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index dbcb0dc8c..8b8f1b616 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -28,22 +28,21 @@ import ( "time" "go.uber.org/zap" - kubevirtv1 "kubevirt.io/api/core/v1" - cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" + kubevirtcorev1 "kubevirt.io/api/core/v1" + cdicorev1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - kubevirttypes "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" controllerutil "k8c.io/machine-controller/pkg/controller/util" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + kubevirttypes "k8c.io/machine-controller/sdk/cloudprovider/kubevirt" + "k8c.io/machine-controller/sdk/providerconfig" corev1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -51,14 +50,14 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/utils/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) func init() { - if err := kubevirtv1.AddToScheme(scheme.Scheme); err != nil { + if err := kubevirtcorev1.AddToScheme(scheme.Scheme); err != nil { panic(fmt.Sprintf("failed to add kubevirtv1 to scheme: %v", err)) } - if err := cdiv1beta1.AddToScheme(scheme.Scheme); err != nil { + if err := cdicorev1beta1.AddToScheme(scheme.Scheme); err != nil { panic(fmt.Sprintf("failed to add cdiv1beta1 to scheme: %v", err)) } } @@ -101,13 +100,13 @@ type Config struct { CPUs string Memory string Namespace string - OSImageSource *cdiv1beta1.DataVolumeSource + OSImageSource *cdicorev1beta1.DataVolumeSource StorageTarget StorageTarget StorageClassName string StorageAccessType corev1.PersistentVolumeAccessMode PVCSize resource.Quantity - Instancetype *kubevirtv1.InstancetypeMatcher - Preference *kubevirtv1.PreferenceMatcher + Instancetype *kubevirtcorev1.InstancetypeMatcher + Preference *kubevirtcorev1.PreferenceMatcher SecondaryDisks []SecondaryDisks NodeAffinityPreset NodeAffinityPreset TopologySpreadConstraints []corev1.TopologySpreadConstraint @@ -117,7 +116,7 @@ type Config struct { ExtraHeaders []string ExtraHeadersSecretRef string DataVolumeSecretRef string - EvictionStrategy kubevirtv1.EvictionStrategy + EvictionStrategy kubevirtcorev1.EvictionStrategy ProviderNetworkName string SubnetName string @@ -144,7 +143,7 @@ const ( noAffinityType = "" ) -func (p *provider) affinityType(affinityType providerconfigtypes.ConfigVarString) (AffinityType, error) { +func (p *provider) affinityType(affinityType providerconfig.ConfigVarString) (AffinityType, error) { podAffinityPresetString, err := p.configVarResolver.GetConfigVarStringValue(affinityType) if err != nil { return "", fmt.Errorf(`failed to parse "podAffinityPreset" field: %w`, err) @@ -176,7 +175,7 @@ type SecondaryDisks struct { } type kubeVirtServer struct { - vmi kubevirtv1.VirtualMachineInstance + vmi kubevirtcorev1.VirtualMachineInstance } func (k *kubeVirtServer) Name() string { @@ -205,7 +204,7 @@ func (k *kubeVirtServer) Addresses() map[string]corev1.NodeAddressType { } func (k *kubeVirtServer) Status() instance.Status { - if k.vmi.Status.Phase == kubevirtv1.Running { + if k.vmi.Status.Phase == kubevirtcorev1.Running { return instance.StatusRunning } return instance.StatusUnknown @@ -213,8 +212,8 @@ func (k *kubeVirtServer) Status() instance.Status { var _ instance.Instance = &kubeVirtServer{} -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfig.Config, error) { + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, err } @@ -338,7 +337,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if rawConfig.VirtualMachine.DNSConfig != nil { config.DNSConfig = rawConfig.VirtualMachine.DNSConfig } - infraClient, err := client.New(config.RestConfig, client.Options{}) + infraClient, err := ctrlruntimeclient.New(config.RestConfig, ctrlruntimeclient.Options{}) if err != nil { return nil, nil, fmt.Errorf("failed to get kubevirt client: %w", err) } @@ -368,17 +367,17 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } if rawConfig.VirtualMachine.EvictionStrategy != "" { - config.EvictionStrategy = kubevirtv1.EvictionStrategy(rawConfig.VirtualMachine.EvictionStrategy) + config.EvictionStrategy = kubevirtcorev1.EvictionStrategy(rawConfig.VirtualMachine.EvictionStrategy) } return &config, pconfig, nil } -func (p *provider) getStorageAccessType(ctx context.Context, accessType providerconfigtypes.ConfigVarString, - infraClient client.Client, storageClassName string) (corev1.PersistentVolumeAccessMode, error) { +func (p *provider) getStorageAccessType(ctx context.Context, accessType providerconfig.ConfigVarString, + infraClient ctrlruntimeclient.Client, storageClassName string) (corev1.PersistentVolumeAccessMode, error) { at, _ := p.configVarResolver.GetConfigVarStringValue(accessType) if at == "" { - sp := &cdiv1beta1.StorageProfile{} + sp := &cdicorev1beta1.StorageProfile{} if err := infraClient.Get(ctx, types.NamespacedName{Name: storageClassName}, sp); err != nil { return "", fmt.Errorf(`failed to get cdi storageprofile: %w`, err) } @@ -449,7 +448,7 @@ func (p *provider) parseTopologySpreadConstraint(topologyConstraints []kubevirtt return parsedTopologyConstraints, nil } -func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, config *Config) (*cdiv1beta1.DataVolumeSource, error) { +func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, config *Config) (*cdicorev1beta1.DataVolumeSource, error) { osImage, err := p.configVarResolver.GetConfigVarStringValue(primaryDisk.OsImage) if err != nil { return nil, fmt.Errorf(`failed to get value of "primaryDisk.osImage" field: %w`, err) @@ -468,18 +467,18 @@ func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, con if err != nil { return nil, fmt.Errorf(`failed to get value of "primaryDisk.extraHeaders" field: %w`, err) } - return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage, ExtraHeaders: extraHeaders, SecretRef: config.DataVolumeSecretRef}}, nil + return &cdicorev1beta1.DataVolumeSource{HTTP: &cdicorev1beta1.DataVolumeSourceHTTP{URL: osImage, ExtraHeaders: extraHeaders, SecretRef: config.DataVolumeSecretRef}}, nil case registrySource: return registryDataVolume(osImage, pullMethod), nil case pvcSource: if namespaceAndName := strings.Split(osImage, "/"); len(namespaceAndName) >= 2 { - return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: namespaceAndName[1], Namespace: namespaceAndName[0]}}, nil + return &cdicorev1beta1.DataVolumeSource{PVC: &cdicorev1beta1.DataVolumeSourcePVC{Name: namespaceAndName[1], Namespace: namespaceAndName[0]}}, nil } - return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: config.Namespace}}, nil + return &cdicorev1beta1.DataVolumeSource{PVC: &cdicorev1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: config.Namespace}}, nil default: // handle old API for backward compatibility. if srcURL, err := url.ParseRequestURI(osImage); err == nil { - if srcURL.Scheme == cdiv1beta1.RegistrySchemeDocker || srcURL.Scheme == cdiv1beta1.RegistrySchemeOci { + if srcURL.Scheme == cdicorev1beta1.RegistrySchemeDocker || srcURL.Scheme == cdicorev1beta1.RegistrySchemeOci { return registryDataVolume(osImage, pullMethod), nil } @@ -488,19 +487,19 @@ func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, con return nil, fmt.Errorf(`failed to get value of "primaryDisk.extraHeaders" field: %w`, err) } - return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage, ExtraHeaders: extraHeaders, SecretRef: config.DataVolumeSecretRef}}, nil + return &cdicorev1beta1.DataVolumeSource{HTTP: &cdicorev1beta1.DataVolumeSourceHTTP{URL: osImage, ExtraHeaders: extraHeaders, SecretRef: config.DataVolumeSecretRef}}, nil } if namespaceAndName := strings.Split(osImage, "/"); len(namespaceAndName) >= 2 { - return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: namespaceAndName[1], Namespace: namespaceAndName[0]}}, nil + return &cdicorev1beta1.DataVolumeSource{PVC: &cdicorev1beta1.DataVolumeSourcePVC{Name: namespaceAndName[1], Namespace: namespaceAndName[0]}}, nil } - return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: config.Namespace}}, nil + return &cdicorev1beta1.DataVolumeSource{PVC: &cdicorev1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: config.Namespace}}, nil } } func getHTTPExtraHeaders(config *Config) ([]string, error) { var extraHeaders []string if config.ExtraHeadersSecretRef != "" { - sigClient, err := client.New(config.RestConfig, client.Options{}) + sigClient, err := ctrlruntimeclient.New(config.RestConfig, ctrlruntimeclient.Options{}) if err != nil { return nil, fmt.Errorf("failed to get kubevirt client: %w", err) } @@ -537,24 +536,24 @@ func getNamespace() string { return ns } -func (p *provider) getPullMethod(pullMethod providerconfigtypes.ConfigVarString) (cdiv1beta1.RegistryPullMethod, error) { +func (p *provider) getPullMethod(pullMethod providerconfig.ConfigVarString) (cdicorev1beta1.RegistryPullMethod, error) { resolvedPM, err := p.configVarResolver.GetConfigVarStringValue(pullMethod) if err != nil { return "", err } - switch pm := cdiv1beta1.RegistryPullMethod(resolvedPM); pm { - case cdiv1beta1.RegistryPullNode, cdiv1beta1.RegistryPullPod: + switch pm := cdicorev1beta1.RegistryPullMethod(resolvedPM); pm { + case cdicorev1beta1.RegistryPullNode, cdicorev1beta1.RegistryPullPod: return pm, nil case "": - return cdiv1beta1.RegistryPullNode, nil + return cdicorev1beta1.RegistryPullNode, nil default: return "", fmt.Errorf("unsupported value: %v", resolvedPM) } } -func registryDataVolume(imageURL string, pullMethod cdiv1beta1.RegistryPullMethod) *cdiv1beta1.DataVolumeSource { - return &cdiv1beta1.DataVolumeSource{ - Registry: &cdiv1beta1.DataVolumeSourceRegistry{ +func registryDataVolume(imageURL string, pullMethod cdicorev1beta1.RegistryPullMethod) *cdicorev1beta1.DataVolumeSource { + return &cdicorev1beta1.DataVolumeSource{ + Registry: &cdicorev1beta1.DataVolumeSourceRegistry{ URL: &imageURL, PullMethod: &pullMethod, }, @@ -569,22 +568,22 @@ func (p *provider) Get(ctx context.Context, _ *zap.SugaredLogger, machine *clust Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), } } - sigClient, err := client.New(c.RestConfig, client.Options{}) + sigClient, err := ctrlruntimeclient.New(c.RestConfig, ctrlruntimeclient.Options{}) if err != nil { return nil, fmt.Errorf("failed to get kubevirt client: %w", err) } - virtualMachine := &kubevirtv1.VirtualMachine{} + virtualMachine := &kubevirtcorev1.VirtualMachine{} if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: machine.Name}, virtualMachine); err != nil { - if !kerrors.IsNotFound(err) { + if !apierrors.IsNotFound(err) { return nil, fmt.Errorf("failed to get VirtualMachine %s: %w", machine.Name, err) } return nil, cloudprovidererrors.ErrInstanceNotFound } - virtualMachineInstance := &kubevirtv1.VirtualMachineInstance{} + virtualMachineInstance := &kubevirtcorev1.VirtualMachineInstance{} if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: machine.Name}, virtualMachineInstance); err != nil { - if kerrors.IsNotFound(err) { + if apierrors.IsNotFound(err) { return &kubeVirtServer{}, nil } @@ -622,12 +621,12 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus } } - sigClient, err := client.New(c.RestConfig, client.Options{}) + sigClient, err := ctrlruntimeclient.New(c.RestConfig, ctrlruntimeclient.Options{}) if err != nil { return fmt.Errorf("failed to get kubevirt client: %w", err) } if _, ok := kubevirttypes.SupportedOS[pc.OperatingSystem]; !ok { - return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, providerconfigtypes.ErrOSNotSupported) + return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, providerconfig.ErrOSNotSupported) } if c.DNSPolicy == corev1.DNSNone { if c.DNSConfig == nil || len(c.DNSConfig.Nameservers) == 0 { @@ -635,14 +634,14 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus } } // Check if we can reach the API of the target cluster. - vmi := &kubevirtv1.VirtualMachineInstance{} - if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: "not-expected-to-exist"}, vmi); err != nil && !kerrors.IsNotFound(err) { + vmi := &kubevirtcorev1.VirtualMachineInstance{} + if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: "not-expected-to-exist"}, vmi); err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to request VirtualMachineInstances: %w", err) } if c.EvictionStrategy != "" { - if c.EvictionStrategy != kubevirtv1.EvictionStrategyExternal && - c.EvictionStrategy != kubevirtv1.EvictionStrategyLiveMigrate { + if c.EvictionStrategy != kubevirtcorev1.EvictionStrategyExternal && + c.EvictionStrategy != kubevirtcorev1.EvictionStrategyLiveMigrate { return fmt.Errorf("unsupported vm eviction strategy: %s", c.EvictionStrategy) } } @@ -682,7 +681,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s type machineDeploymentNameGetter func() (string, error) -func machineDeploymentNameAndRevisionForMachineGetter(ctx context.Context, machine *clusterv1alpha1.Machine, c client.Client) machineDeploymentNameGetter { +func machineDeploymentNameAndRevisionForMachineGetter(ctx context.Context, machine *clusterv1alpha1.Machine, c ctrlruntimeclient.Client) machineDeploymentNameGetter { mdName, _, err := controllerutil.GetMachineDeploymentNameAndRevisionForMachine(ctx, machine, c) return func() (string, error) { return mdName, err @@ -698,7 +697,7 @@ func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *cl } } - sigClient, err := client.New(c.RestConfig, client.Options{}) + sigClient, err := ctrlruntimeclient.New(c.RestConfig, ctrlruntimeclient.Options{}) if err != nil { return nil, fmt.Errorf("failed to get kubevirt client: %w", err) } @@ -723,7 +722,7 @@ func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *cl ObjectMeta: metav1.ObjectMeta{ Name: userDataSecretName, Namespace: virtualMachine.Namespace, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(virtualMachine, kubevirtv1.VirtualMachineGroupVersionKind)}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(virtualMachine, kubevirtcorev1.VirtualMachineGroupVersionKind)}, }, Data: map[string][]byte{"userdata": []byte(userdata)}, } @@ -733,19 +732,19 @@ func (p *provider) Create(ctx context.Context, _ *zap.SugaredLogger, machine *cl return &kubeVirtServer{}, nil } -func (p *provider) newVirtualMachine(c *Config, pc *providerconfigtypes.Config, machine *clusterv1alpha1.Machine, - labels map[string]string, userdataSecretName, userdata string, mdNameGetter machineDeploymentNameGetter) (*kubevirtv1.VirtualMachine, error) { +func (p *provider) newVirtualMachine(c *Config, pc *providerconfig.Config, machine *clusterv1alpha1.Machine, + labels map[string]string, userdataSecretName, userdata string, mdNameGetter machineDeploymentNameGetter) (*kubevirtcorev1.VirtualMachine, error) { // We add the timestamp because the secret name must be different when we recreate the VMI // because its pod got deleted // The secret has an ownerRef on the VMI so garbace collection will take care of cleaning up. terminationGracePeriodSeconds := int64(30) - evictionStrategy := kubevirtv1.EvictionStrategyExternal + evictionStrategy := kubevirtcorev1.EvictionStrategyExternal if c.EvictionStrategy != "" { evictionStrategy = c.EvictionStrategy } - resourceRequirements := kubevirtv1.ResourceRequirements{} + resourceRequirements := kubevirtcorev1.ResourceRequirements{} labels["kubevirt.io/vm"] = machine.Name //Add a common label to all VirtualMachines spawned by the same MachineDeployment (= MachineDeployment name). if mdName, err := mdNameGetter(); err == nil { @@ -774,7 +773,7 @@ func (p *provider) newVirtualMachine(c *Config, pc *providerconfigtypes.Config, // Add machineName as prefix to secondaryDisks. addPrefixToSecondaryDisk(c.SecondaryDisks, dataVolumeName) - if pc.OperatingSystem == providerconfigtypes.OperatingSystemFlatcar { + if pc.OperatingSystem == providerconfig.OperatingSystemFlatcar { annotations["kubevirt.io/ignitiondata"] = userdata } @@ -794,37 +793,37 @@ func (p *provider) newVirtualMachine(c *Config, pc *providerconfigtypes.Config, } defaultBridgeNetwork := defaultBridgeNetwork() - runStrategy := kubevirtv1.RunStrategyOnce + runStrategy := kubevirtcorev1.RunStrategyOnce // currently we only support KubeOvn as a ProviderNetwork and KubeOvn has the ability to pin the IP of the VM(static ip) // even if the VMi was stopped or deleted thus we can have the VM always running and in the events of VM restarts the // ip address of the VMI will not change. if c.SubnetName != "" { - runStrategy = kubevirtv1.RunStrategyAlways + runStrategy = kubevirtcorev1.RunStrategyAlways } - virtualMachine := &kubevirtv1.VirtualMachine{ + virtualMachine := &kubevirtcorev1.VirtualMachine{ ObjectMeta: metav1.ObjectMeta{ Name: machine.Name, Namespace: c.Namespace, Labels: labels, }, - Spec: kubevirtv1.VirtualMachineSpec{ + Spec: kubevirtcorev1.VirtualMachineSpec{ RunStrategy: &runStrategy, Instancetype: c.Instancetype, Preference: c.Preference, - Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ + Template: &kubevirtcorev1.VirtualMachineInstanceTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: annotations, Labels: labels, }, - Spec: kubevirtv1.VirtualMachineInstanceSpec{ + Spec: kubevirtcorev1.VirtualMachineInstanceSpec{ EvictionStrategy: &evictionStrategy, - Networks: []kubevirtv1.Network{ - *kubevirtv1.DefaultPodNetwork(), + Networks: []kubevirtcorev1.Network{ + *kubevirtcorev1.DefaultPodNetwork(), }, - Domain: kubevirtv1.DomainSpec{ - Devices: kubevirtv1.Devices{ - Interfaces: []kubevirtv1.Interface{*defaultBridgeNetwork}, + Domain: kubevirtcorev1.DomainSpec{ + Devices: kubevirtcorev1.Devices{ + Interfaces: []kubevirtcorev1.Interface{*defaultBridgeNetwork}, Disks: getVMDisks(c), NetworkInterfaceMultiQueue: ptr.To(c.EnableNetworkMultiQueue), }, @@ -852,14 +851,14 @@ func (p *provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *c Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), } } - sigClient, err := client.New(c.RestConfig, client.Options{}) + sigClient, err := ctrlruntimeclient.New(c.RestConfig, ctrlruntimeclient.Options{}) if err != nil { return false, fmt.Errorf("failed to get kubevirt client: %w", err) } - vm := &kubevirtv1.VirtualMachine{} + vm := &kubevirtcorev1.VirtualMachine{} if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: machine.Name}, vm); err != nil { - if !kerrors.IsNotFound(err) { + if !apierrors.IsNotFound(err) { return false, fmt.Errorf("failed to get VirtualMachineInstance %s: %w", machine.Name, err) } return true, nil @@ -902,44 +901,44 @@ func dnsPolicy(policy string) (corev1.DNSPolicy, error) { return "", fmt.Errorf("unknown dns policy: %s", policy) } -func getVMDisks(config *Config) []kubevirtv1.Disk { - disks := []kubevirtv1.Disk{ +func getVMDisks(config *Config) []kubevirtcorev1.Disk { + disks := []kubevirtcorev1.Disk{ { Name: "datavolumedisk", - DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: "virtio"}}, + DiskDevice: kubevirtcorev1.DiskDevice{Disk: &kubevirtcorev1.DiskTarget{Bus: "virtio"}}, }, { Name: "cloudinitdisk", - DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: "virtio"}}, + DiskDevice: kubevirtcorev1.DiskDevice{Disk: &kubevirtcorev1.DiskTarget{Bus: "virtio"}}, }, } for _, sd := range config.SecondaryDisks { - disks = append(disks, kubevirtv1.Disk{ + disks = append(disks, kubevirtcorev1.Disk{ Name: sd.Name, - DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: "virtio"}}, + DiskDevice: kubevirtcorev1.DiskDevice{Disk: &kubevirtcorev1.DiskTarget{Bus: "virtio"}}, }) } return disks } -func defaultBridgeNetwork() *kubevirtv1.Interface { - return kubevirtv1.DefaultBridgeNetworkInterface() +func defaultBridgeNetwork() *kubevirtcorev1.Interface { + return kubevirtcorev1.DefaultBridgeNetworkInterface() } -func getVMVolumes(config *Config, dataVolumeName string, userDataSecretName string) []kubevirtv1.Volume { - volumes := []kubevirtv1.Volume{ +func getVMVolumes(config *Config, dataVolumeName string, userDataSecretName string) []kubevirtcorev1.Volume { + volumes := []kubevirtcorev1.Volume{ { Name: "datavolumedisk", - VolumeSource: kubevirtv1.VolumeSource{ - DataVolume: &kubevirtv1.DataVolumeSource{ + VolumeSource: kubevirtcorev1.VolumeSource{ + DataVolume: &kubevirtcorev1.DataVolumeSource{ Name: dataVolumeName, }, }, }, { Name: "cloudinitdisk", - VolumeSource: kubevirtv1.VolumeSource{ - CloudInitNoCloud: &kubevirtv1.CloudInitNoCloudSource{ + VolumeSource: kubevirtcorev1.VolumeSource{ + CloudInitNoCloud: &kubevirtcorev1.CloudInitNoCloudSource{ UserDataSecretRef: &corev1.LocalObjectReference{ Name: userDataSecretName, }, @@ -948,10 +947,10 @@ func getVMVolumes(config *Config, dataVolumeName string, userDataSecretName stri }, } for _, sd := range config.SecondaryDisks { - volumes = append(volumes, kubevirtv1.Volume{ + volumes = append(volumes, kubevirtcorev1.Volume{ Name: sd.Name, - VolumeSource: kubevirtv1.VolumeSource{ - DataVolume: &kubevirtv1.DataVolumeSource{ + VolumeSource: kubevirtcorev1.VolumeSource{ + DataVolume: &kubevirtcorev1.DataVolumeSource{ Name: sd.Name, }}, }) @@ -959,15 +958,15 @@ func getVMVolumes(config *Config, dataVolumeName string, userDataSecretName stri return volumes } -func getDataVolumeTemplates(config *Config, dataVolumeName string, annotations map[string]string) []kubevirtv1.DataVolumeTemplateSpec { +func getDataVolumeTemplates(config *Config, dataVolumeName string, annotations map[string]string) []kubevirtcorev1.DataVolumeTemplateSpec { pvcRequest := corev1.ResourceList{corev1.ResourceStorage: config.PVCSize} - dataVolumeTemplates := []kubevirtv1.DataVolumeTemplateSpec{ + dataVolumeTemplates := []kubevirtcorev1.DataVolumeTemplateSpec{ { ObjectMeta: metav1.ObjectMeta{ Name: dataVolumeName, Annotations: annotations, }, - Spec: cdiv1beta1.DataVolumeSpec{ + Spec: cdicorev1beta1.DataVolumeSpec{ Source: config.OSImageSource, }, }, @@ -985,7 +984,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string, annotations m }, } default: - dataVolumeTemplates[0].Spec.Storage = &cdiv1beta1.StorageSpec{ + dataVolumeTemplates[0].Spec.Storage = &cdicorev1beta1.StorageSpec{ StorageClassName: ptr.To(config.StorageClassName), AccessModes: []corev1.PersistentVolumeAccessMode{ config.StorageAccessType, @@ -997,11 +996,11 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string, annotations m } for _, sd := range config.SecondaryDisks { - dataVolumeTemplates = append(dataVolumeTemplates, kubevirtv1.DataVolumeTemplateSpec{ + dataVolumeTemplates = append(dataVolumeTemplates, kubevirtcorev1.DataVolumeTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: sd.Name, }, - Spec: cdiv1beta1.DataVolumeSpec{ + Spec: cdicorev1beta1.DataVolumeSpec{ PVC: &corev1.PersistentVolumeClaimSpec{ StorageClassName: ptr.To(sd.StorageClassName), AccessModes: []corev1.PersistentVolumeAccessMode{ @@ -1109,7 +1108,7 @@ func appendTopologiesLabels(ctx context.Context, c *Config, labels map[string]st } func getStorageTopologies(ctx context.Context, storageClasName string, c *Config, labels map[string]string) error { - kubeClient, err := client.New(c.RestConfig, client.Options{}) + kubeClient, err := ctrlruntimeclient.New(c.RestConfig, ctrlruntimeclient.Options{}) if err != nil { return fmt.Errorf("failed to get kubevirt client: %w", err) } @@ -1154,7 +1153,7 @@ func setOVNAnnotations(c *Config, annotations map[string]string) error { return nil } -func (p *provider) configureStorage(infraClient client.Client, template kubevirttypes.Template) (corev1.PersistentVolumeAccessMode, []SecondaryDisks, error) { +func (p *provider) configureStorage(infraClient ctrlruntimeclient.Client, template kubevirttypes.Template) (corev1.PersistentVolumeAccessMode, []SecondaryDisks, error) { secondaryDisks := make([]SecondaryDisks, 0, len(template.SecondaryDisks)) for i, sd := range template.SecondaryDisks { sdSizeString, err := p.configVarResolver.GetConfigVarStringValue(sd.Size) diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index 4651889d5..6647d84ff 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -25,12 +25,12 @@ import ( "reflect" "testing" - kubevirtv1 "kubevirt.io/api/core/v1" - cdiv1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" + kubevirtcorev1 "kubevirt.io/api/core/v1" + cdicorev1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" - "k8c.io/machine-controller/pkg/cloudprovider/provider/kubevirt/types" cloudprovidertesting "k8c.io/machine-controller/pkg/cloudprovider/testing" - "k8c.io/machine-controller/pkg/providerconfig" + "k8c.io/machine-controller/sdk/cloudprovider/kubevirt" + "k8c.io/machine-controller/sdk/providerconfig" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -47,7 +47,7 @@ var ( vmManifestsFS embed.FS vmDir = "testdata" fakeclient ctrlruntimeclient.WithWatch - expectedVms map[string]*kubevirtv1.VirtualMachine + expectedVms map[string]*kubevirtcorev1.VirtualMachine ) func init() { @@ -58,8 +58,8 @@ func init() { type kubevirtProviderSpecConf struct { OsImageDV string // if OsImage from DV and not from http source - Instancetype *kubevirtv1.InstancetypeMatcher - Preference *kubevirtv1.PreferenceMatcher + Instancetype *kubevirtcorev1.InstancetypeMatcher + Preference *kubevirtcorev1.PreferenceMatcher StorageTarget StorageTarget OperatingSystem string TopologySpreadConstraint bool @@ -68,8 +68,8 @@ type kubevirtProviderSpecConf struct { SecondaryDisks bool OsImageSource imageSource OsImageSourceURL string - PullMethod cdiv1beta1.RegistryPullMethod - ProviderNetwork *types.ProviderNetwork + PullMethod cdicorev1beta1.RegistryPullMethod + ProviderNetwork *kubevirt.ProviderNetwork ExtraHeadersSet bool EvictStrategy string } @@ -106,7 +106,7 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { {{- end }} "virtualMachine": { {{- if .EvictStrategy }} - "evictionStrategy": "LiveMigrate", + "evictionStrategy": "LiveMigrate", {{- end }} {{- if .ProviderNetwork }} "providerNetwork": { @@ -209,11 +209,11 @@ func TestNewVirtualMachine(t *testing.T) { { name: "instancetype-preference-standard", specConf: kubevirtProviderSpecConf{ - Instancetype: &kubevirtv1.InstancetypeMatcher{ + Instancetype: &kubevirtcorev1.InstancetypeMatcher{ Name: "standard-it", Kind: "VirtualMachineInstancetype", }, - Preference: &kubevirtv1.PreferenceMatcher{ + Preference: &kubevirtcorev1.PreferenceMatcher{ Name: "standard-pref", Kind: "VirtualMachinePreference", }, @@ -222,11 +222,11 @@ func TestNewVirtualMachine(t *testing.T) { { name: "instancetype-preference-custom", specConf: kubevirtProviderSpecConf{ - Instancetype: &kubevirtv1.InstancetypeMatcher{ + Instancetype: &kubevirtcorev1.InstancetypeMatcher{ Name: "custom-it", Kind: "VirtualMachineClusterInstancetype", }, - Preference: &kubevirtv1.PreferenceMatcher{ + Preference: &kubevirtcorev1.PreferenceMatcher{ Name: "custom-pref", Kind: "VirtualMachineClusterPreference", }, @@ -235,7 +235,7 @@ func TestNewVirtualMachine(t *testing.T) { { name: "kubeovn-provider-network", specConf: kubevirtProviderSpecConf{ - ProviderNetwork: &types.ProviderNetwork{Name: "KubeOVN", VPC: types.VPC{Name: "test-vpc", Subnet: &types.Subnet{ + ProviderNetwork: &kubevirt.ProviderNetwork{Name: "KubeOVN", VPC: kubevirt.VPC{Name: "test-vpc", Subnet: &kubevirt.Subnet{ Name: "test-subnet", }}}}, }, @@ -273,7 +273,7 @@ func TestNewVirtualMachine(t *testing.T) { }, { name: "registry-image-source-pod", - specConf: kubevirtProviderSpecConf{OsImageSource: registrySource, OsImageSourceURL: "docker://x.y.z.t/ubuntu.img:latest", PullMethod: cdiv1beta1.RegistryPullPod}, + specConf: kubevirtProviderSpecConf{OsImageSource: registrySource, OsImageSourceURL: "docker://x.y.z.t/ubuntu.img:latest", PullMethod: cdicorev1beta1.RegistryPullPod}, }, { name: "pvc-image-source", @@ -307,7 +307,7 @@ func TestNewVirtualMachine(t *testing.T) { // Check the created VirtualMachine vm, _ := p.newVirtualMachine(c, pc, machine, labels, "udsn", userdata, fakeMachineDeploymentNameAndRevisionForMachineGetter()) - vm.TypeMeta.APIVersion, vm.TypeMeta.Kind = kubevirtv1.VirtualMachineGroupVersionKind.ToAPIVersionAndKind() + vm.TypeMeta.APIVersion, vm.TypeMeta.Kind = kubevirtcorev1.VirtualMachineGroupVersionKind.ToAPIVersionAndKind() if !equality.Semantic.DeepEqual(vm, expectedVms[tt.name]) { t.Errorf("Diff %v", diff.ObjectGoPrintDiff(expectedVms[tt.name], vm)) @@ -322,10 +322,10 @@ func fakeMachineDeploymentNameAndRevisionForMachineGetter() machineDeploymentNam } } -func toVirtualMachines(objects []runtime.Object) map[string]*kubevirtv1.VirtualMachine { - vms := make(map[string]*kubevirtv1.VirtualMachine) +func toVirtualMachines(objects []runtime.Object) map[string]*kubevirtcorev1.VirtualMachine { + vms := make(map[string]*kubevirtcorev1.VirtualMachine) for _, o := range objects { - if vm, ok := o.(*kubevirtv1.VirtualMachine); ok { + if vm, ok := o.(*kubevirtcorev1.VirtualMachine); ok { vms[vm.Name] = vm } } diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go deleted file mode 100644 index 79e7100fc..000000000 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ /dev/null @@ -1,167 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - kubevirtv1 "kubevirt.io/api/core/v1" - - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" - - corev1 "k8s.io/api/core/v1" -) - -var SupportedOS = map[providerconfigtypes.OperatingSystem]*struct{}{ - providerconfigtypes.OperatingSystemUbuntu: nil, - providerconfigtypes.OperatingSystemRHEL: nil, - providerconfigtypes.OperatingSystemFlatcar: nil, - providerconfigtypes.OperatingSystemRockyLinux: nil, -} - -type RawConfig struct { - ClusterName providerconfigtypes.ConfigVarString `json:"clusterName"` - Auth Auth `json:"auth,omitempty"` - VirtualMachine VirtualMachine `json:"virtualMachine,omitempty"` - Affinity Affinity `json:"affinity,omitempty"` - TopologySpreadConstraints []TopologySpreadConstraint `json:"topologySpreadConstraints"` -} - -// Auth. -type Auth struct { - Kubeconfig providerconfigtypes.ConfigVarString `json:"kubeconfig,omitempty"` -} - -// VirtualMachine. -type VirtualMachine struct { - // Deprecated: use Instancetype/Preference instead. - Flavor Flavor `json:"flavor,omitempty"` - // Instancetype is optional. - Instancetype *kubevirtv1.InstancetypeMatcher `json:"instancetype,omitempty"` - // Preference is optional. - Preference *kubevirtv1.PreferenceMatcher `json:"preference,omitempty"` - Template Template `json:"template,omitempty"` - DNSPolicy providerconfigtypes.ConfigVarString `json:"dnsPolicy,omitempty"` - DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` - Location *Location `json:"location,omitempty"` - ProviderNetwork *ProviderNetwork `json:"providerNetwork,omitempty"` - EnableNetworkMultiQueue providerconfigtypes.ConfigVarBool `json:"enableNetworkMultiQueue,omitempty"` - EvictionStrategy string `json:"evictionStrategy,omitempty"` -} - -// Flavor. -type Flavor struct { - Name providerconfigtypes.ConfigVarString `json:"name,omitempty"` - Profile providerconfigtypes.ConfigVarString `json:"profile,omitempty"` -} - -// Template. -type Template struct { - CPUs providerconfigtypes.ConfigVarString `json:"cpus,omitempty"` - Memory providerconfigtypes.ConfigVarString `json:"memory,omitempty"` - PrimaryDisk PrimaryDisk `json:"primaryDisk,omitempty"` - SecondaryDisks []SecondaryDisks `json:"secondaryDisks,omitempty"` -} - -// PrimaryDisk. -type PrimaryDisk struct { - Disk - // DataVolumeSecretRef is the name of the secret that will be sent to the CDI data importer pod to read basic auth parameters. - DataVolumeSecretRef providerconfigtypes.ConfigVarString `json:"dataVolumeSecretRef,omitempty"` - // ExtraHeaders is a list of strings containing extra headers to include with HTTP transfer requests - // +optional - ExtraHeaders []string `json:"extraHeaders,omitempty"` - // ExtraHeadersSecretRef is a secret that contains a list of strings containing extra headers to include with HTTP transfer requests - // +optional - ExtraHeadersSecretRef providerconfigtypes.ConfigVarString `json:"extraHeadersSecretRef,omitempty"` - // StorageTarget describes which VirtualMachine storage target will be used in the DataVolumeTemplate. - StorageTarget providerconfigtypes.ConfigVarString `json:"storageTarget,omitempty"` - // OsImage describes the OS that will be installed on the VirtualMachine. - OsImage providerconfigtypes.ConfigVarString `json:"osImage,omitempty"` - // Source describes the VM Disk Image source. - Source providerconfigtypes.ConfigVarString `json:"source,omitempty"` - // PullMethod describes the VM Disk Image source optional pull method for registry source. Defaults to 'node'. - PullMethod providerconfigtypes.ConfigVarString `json:"pullMethod,omitempty"` -} - -// SecondaryDisks. -type SecondaryDisks struct { - Disk -} - -// Disk. -type Disk struct { - Size providerconfigtypes.ConfigVarString `json:"size,omitempty"` - StorageClassName providerconfigtypes.ConfigVarString `json:"storageClassName,omitempty"` - StorageAccessType providerconfigtypes.ConfigVarString `json:"storageAccessType,omitempty"` -} - -// Affinity. -type Affinity struct { - // Deprecated: Use TopologySpreadConstraint instead. - PodAffinityPreset providerconfigtypes.ConfigVarString `json:"podAffinityPreset,omitempty"` - // Deprecated: Use TopologySpreadConstraint instead. - PodAntiAffinityPreset providerconfigtypes.ConfigVarString `json:"podAntiAffinityPreset,omitempty"` - NodeAffinityPreset NodeAffinityPreset `json:"nodeAffinityPreset,omitempty"` -} - -// NodeAffinityPreset. -type NodeAffinityPreset struct { - Type providerconfigtypes.ConfigVarString `json:"type,omitempty"` - Key providerconfigtypes.ConfigVarString `json:"key,omitempty"` - Values []providerconfigtypes.ConfigVarString `json:"values,omitempty"` -} - -// TopologySpreadConstraint describes topology spread constraints for VMs. -type TopologySpreadConstraint struct { - // MaxSkew describes the degree to which VMs may be unevenly distributed. - MaxSkew providerconfigtypes.ConfigVarString `json:"maxSkew,omitempty"` - // TopologyKey is the key of infra-node labels. - TopologyKey providerconfigtypes.ConfigVarString `json:"topologyKey,omitempty"` - // WhenUnsatisfiable indicates how to deal with a VM if it doesn't satisfy - // the spread constraint. - WhenUnsatisfiable providerconfigtypes.ConfigVarString `json:"whenUnsatisfiable,omitempty"` -} - -// Location describes the region and zone where the machines are created at and where the deployed resources will reside. -type Location struct { - Region string `json:"region,omitempty"` - Zone string `json:"zone,omitempty"` -} - -// ProviderNetwork describes the infra cluster network fabric that is being used. -type ProviderNetwork struct { - Name string `json:"name"` - VPC VPC `json:"vpc"` -} - -// VPC is a virtual network dedicated to a single tenant within a KubeVirt, where the resources in the VPC -// is isolated from any other resources within the KubeVirt infra cluster. -type VPC struct { - Name string `json:"name"` - Subnet *Subnet `json:"subnet,omitempty"` -} - -// Subnet a smaller, segmented portion of a larger network, like a Virtual Private Cloud (VPC). -type Subnet struct { - Name string `json:"name"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index 2a8c98acf..dad86a02b 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -32,17 +32,16 @@ import ( "go.uber.org/zap" "golang.org/x/oauth2" - common "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8c.io/machine-controller/pkg/cloudprovider/common/ssh" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - linodetypes "k8c.io/machine-controller/pkg/cloudprovider/provider/linode/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + common "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + linodetypes "k8c.io/machine-controller/sdk/cloudprovider/linode" + "k8c.io/machine-controller/sdk/providerconfig" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" ) @@ -81,12 +80,12 @@ func (t *TokenSource) Token() (*oauth2.Token, error) { return token, nil } -func getSlugForOS(os providerconfigtypes.OperatingSystem) (string, error) { +func getSlugForOS(os providerconfig.OperatingSystem) (string, error) { switch os { - case providerconfigtypes.OperatingSystemUbuntu: + case providerconfig.OperatingSystemUbuntu: return "linode/ubuntu18.04", nil } - return "", providerconfigtypes.ErrOSNotSupported + return "", providerconfig.ErrOSNotSupported } func getClient(ctx context.Context, token string) linodego.Client { @@ -102,8 +101,8 @@ func getClient(ctx context.Context, token string) linodego.Client { return client } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfig.Config, error) { + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, err } @@ -397,12 +396,12 @@ func (d *linodeInstance) ProviderID() string { return fmt.Sprintf("linode://%s", d.ID()) } -func (d *linodeInstance) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{} +func (d *linodeInstance) Addresses() map[string]corev1.NodeAddressType { + addresses := map[string]corev1.NodeAddressType{} for _, n := range d.linode.IPv4 { - addresses[n.String()] = v1.NodeInternalIP + addresses[n.String()] = corev1.NodeInternalIP } - addresses[d.linode.IPv6] = v1.NodeInternalIP + addresses[d.linode.IPv6] = corev1.NodeInternalIP return addresses } diff --git a/pkg/cloudprovider/provider/nutanix/client.go b/pkg/cloudprovider/provider/nutanix/client.go index 205a525a1..d557f2acd 100644 --- a/pkg/cloudprovider/provider/nutanix/client.go +++ b/pkg/cloudprovider/provider/nutanix/client.go @@ -28,10 +28,10 @@ import ( nutanixclient "github.com/nutanix-cloud-native/prism-go-client" nutanixv3 "github.com/nutanix-cloud-native/prism-go-client/v3" - "k8c.io/machine-controller/pkg/apis/cluster/common" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - nutanixtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/nutanix/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + nutanixtypes "k8c.io/machine-controller/sdk/cloudprovider/nutanix" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/wait" diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index e9a4d6f30..ec7eb257a 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -25,14 +25,13 @@ import ( "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - nutanixtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/nutanix/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + nutanixtypes "k8c.io/machine-controller/sdk/cloudprovider/nutanix" + "k8c.io/machine-controller/sdk/providerconfig" corev1 "k8s.io/api/core/v1" ktypes "k8s.io/apimachinery/pkg/types" @@ -109,8 +108,8 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes return provider } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *nutanixtypes.RawConfig, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfig.Config, *nutanixtypes.RawConfig, error) { + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, nil, err } diff --git a/pkg/cloudprovider/provider/opennebula/provider.go b/pkg/cloudprovider/provider/opennebula/provider.go index da79637ff..8495758b0 100644 --- a/pkg/cloudprovider/provider/opennebula/provider.go +++ b/pkg/cloudprovider/provider/opennebula/provider.go @@ -31,16 +31,15 @@ import ( "github.com/OpenNebula/one/src/oca/go/src/goca/schemas/vm/keys" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - opennebulatypes "k8c.io/machine-controller/pkg/cloudprovider/provider/opennebula/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + opennebulatypes "k8c.io/machine-controller/sdk/cloudprovider/opennebula" + "k8c.io/machine-controller/sdk/providerconfig" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" ) @@ -83,8 +82,8 @@ func getClient(config *Config) *goca.Client { return goca.NewDefaultClient(goca.NewConfig(config.Username, config.Password, config.Endpoint)) } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfig.Config, error) { + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, err } @@ -424,12 +423,12 @@ func (i *openNebulaInstance) ProviderID() string { return "opennebula://" + strconv.Itoa(i.vm.ID) } -func (i *openNebulaInstance) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{} +func (i *openNebulaInstance) Addresses() map[string]corev1.NodeAddressType { + addresses := map[string]corev1.NodeAddressType{} for _, nic := range i.vm.Template.GetNICs() { ip, _ := nic.Get(shared.IP) - addresses[ip] = v1.NodeInternalIP + addresses[ip] = corev1.NodeInternalIP } return addresses diff --git a/pkg/cloudprovider/provider/opennebula/types/types.go b/pkg/cloudprovider/provider/opennebula/types/types.go deleted file mode 100644 index 51b5f029f..000000000 --- a/pkg/cloudprovider/provider/opennebula/types/types.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2022 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" -) - -type RawConfig struct { - // Auth details - Username providerconfigtypes.ConfigVarString `json:"username,omitempty"` - Password providerconfigtypes.ConfigVarString `json:"password,omitempty"` - Endpoint providerconfigtypes.ConfigVarString `json:"endpoint,omitempty"` - - // Machine details - CPU *float64 `json:"cpu"` - VCPU *int `json:"vcpu"` - Memory *int `json:"memory"` - Image providerconfigtypes.ConfigVarString `json:"image"` - Datastore providerconfigtypes.ConfigVarString `json:"datastore"` - DiskSize *int `json:"diskSize"` - Network providerconfigtypes.ConfigVarString `json:"network"` - EnableVNC providerconfigtypes.ConfigVarBool `json:"enableVNC"` - VMTemplateExtra map[string]string `json:"vmTemplateExtra,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 861642155..44ae50e1f 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -36,15 +36,14 @@ import ( "github.com/gophercloud/gophercloud/pagination" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - openstacktypes "k8c.io/machine-controller/pkg/cloudprovider/provider/openstack/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" cloudproviderutil "k8c.io/machine-controller/pkg/cloudprovider/util" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + openstacktypes "k8c.io/machine-controller/sdk/cloudprovider/openstack" + "k8c.io/machine-controller/sdk/providerconfig" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -178,8 +177,8 @@ func (p *provider) getConfigAuth(c *Config, rawConfig *openstacktypes.RawConfig) return nil } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *openstacktypes.RawConfig, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfig.Config, *openstacktypes.RawConfig, error) { + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, nil, err } @@ -299,7 +298,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } func setProviderSpec(rawConfig openstacktypes.RawConfig, provSpec clusterv1alpha1.ProviderSpec) (*runtime.RawExtension, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, err } diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index 341791d9c..25f26e132 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -32,10 +32,10 @@ import ( "github.com/gophercloud/gophercloud/testhelper/client" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidertesting "k8c.io/machine-controller/pkg/cloudprovider/testing" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/providerconfig" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + "k8c.io/machine-controller/sdk/providerconfig" "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/ptr" @@ -343,7 +343,7 @@ func TestProjectAuthVarsAreCorrectlyLoaded(t *testing.T) { NewClientBuilder(). Build()), } - conf, _, _, _ := p.getConfig(v1alpha1.ProviderSpec{ + conf, _, _, _ := p.getConfig(clusterv1alpha1.ProviderSpec{ Value: &runtime.RawExtension{ Raw: tt.specConf.rawProviderSpec(t), }, diff --git a/pkg/cloudprovider/provider/openstack/types/types.go b/pkg/cloudprovider/provider/openstack/types/types.go deleted file mode 100644 index 8d5af36d4..000000000 --- a/pkg/cloudprovider/provider/openstack/types/types.go +++ /dev/null @@ -1,64 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" -) - -type RawConfig struct { - // Auth details - IdentityEndpoint providerconfigtypes.ConfigVarString `json:"identityEndpoint,omitempty"` - Username providerconfigtypes.ConfigVarString `json:"username,omitempty"` - Password providerconfigtypes.ConfigVarString `json:"password,omitempty"` - ApplicationCredentialID providerconfigtypes.ConfigVarString `json:"applicationCredentialID,omitempty"` - ApplicationCredentialSecret providerconfigtypes.ConfigVarString `json:"applicationCredentialSecret,omitempty"` - DomainName providerconfigtypes.ConfigVarString `json:"domainName,omitempty"` - ProjectName providerconfigtypes.ConfigVarString `json:"projectName,omitempty"` - ProjectID providerconfigtypes.ConfigVarString `json:"projectID,omitempty"` - TenantName providerconfigtypes.ConfigVarString `json:"tenantName,omitempty"` - TenantID providerconfigtypes.ConfigVarString `json:"tenantID,omitempty"` - TokenID providerconfigtypes.ConfigVarString `json:"tokenId,omitempty"` - Region providerconfigtypes.ConfigVarString `json:"region,omitempty"` - InstanceReadyCheckPeriod providerconfigtypes.ConfigVarString `json:"instanceReadyCheckPeriod,omitempty"` - InstanceReadyCheckTimeout providerconfigtypes.ConfigVarString `json:"instanceReadyCheckTimeout,omitempty"` - ComputeAPIVersion providerconfigtypes.ConfigVarString `json:"computeAPIVersion,omitempty"` - - // Machine details - Image providerconfigtypes.ConfigVarString `json:"image"` - Flavor providerconfigtypes.ConfigVarString `json:"flavor"` - SecurityGroups []providerconfigtypes.ConfigVarString `json:"securityGroups,omitempty"` - Network providerconfigtypes.ConfigVarString `json:"network,omitempty"` - Subnet providerconfigtypes.ConfigVarString `json:"subnet,omitempty"` - FloatingIPPool providerconfigtypes.ConfigVarString `json:"floatingIpPool,omitempty"` - AvailabilityZone providerconfigtypes.ConfigVarString `json:"availabilityZone,omitempty"` - TrustDevicePath providerconfigtypes.ConfigVarBool `json:"trustDevicePath"` - RootDiskSizeGB *int `json:"rootDiskSizeGB"` - RootDiskVolumeType providerconfigtypes.ConfigVarString `json:"rootDiskVolumeType,omitempty"` - NodeVolumeAttachLimit *uint `json:"nodeVolumeAttachLimit"` - ServerGroup providerconfigtypes.ConfigVarString `json:"serverGroup"` - ConfigDrive providerconfigtypes.ConfigVarBool `json:"configDrive,omitempty"` - // This tag is related to server metadata, not compute server's tag - Tags map[string]string `json:"tags,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go index c31dbc874..8cff1e5ac 100644 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ b/pkg/cloudprovider/provider/scaleway/provider.go @@ -27,14 +27,13 @@ import ( "github.com/scaleway/scaleway-sdk-go/validation" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" cloudInstance "k8c.io/machine-controller/pkg/cloudprovider/instance" - scalewaytypes "k8c.io/machine-controller/pkg/cloudprovider/provider/scaleway/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + scalewaytypes "k8c.io/machine-controller/sdk/cloudprovider/scaleway" + "k8c.io/machine-controller/sdk/providerconfig" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -75,18 +74,18 @@ func (c *Config) getInstanceAPI() (*instance.API, error) { return instance.NewAPI(client), nil } -func getImageNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { +func getImageNameForOS(os providerconfig.OperatingSystem) (string, error) { switch os { - case providerconfigtypes.OperatingSystemUbuntu: + case providerconfig.OperatingSystemUbuntu: // ubuntu_focal doesn't work (see https://bugs.launchpad.net/ubuntu/+source/linux-kvm/+bug/1880522) // modprobe ip_vs will fail return "ubuntu_bionic", nil } - return "", providerconfigtypes.ErrOSNotSupported + return "", providerconfig.ErrOSNotSupported } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfig.Config, error) { + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, err } diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go index 65338e64e..1078e13d7 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go @@ -24,10 +24,9 @@ import ( "path" "github.com/vmware/go-vcloud-director/v2/govcd" - "github.com/vmware/go-vcloud-director/v2/types/v56" vcdapitypes "github.com/vmware/go-vcloud-director/v2/types/v56" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" "k8s.io/utils/ptr" ) @@ -77,7 +76,7 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * } // 2. Retrieve Sizing and Placement Compute Policy if required. - var computePolicy *types.ComputePolicy + var computePolicy *vcdapitypes.ComputePolicy if c.SizingPolicy != nil || c.PlacementPolicy != nil { allPolicies, err := org.GetAllVdcComputePolicies(url.Values{}) if err != nil { @@ -89,7 +88,7 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * if sizingPolicy == nil { return fmt.Errorf("sizing policy '%s' doesn't exist", *c.SizingPolicy) } - computePolicy = &types.ComputePolicy{ + computePolicy = &vcdapitypes.ComputePolicy{ VmSizingPolicy: &vcdapitypes.Reference{ HREF: sizingPolicy.VdcComputePolicy.ID, }, @@ -102,7 +101,7 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * return fmt.Errorf("placement policy '%s' doesn't exist", *c.PlacementPolicy) } if computePolicy == nil { - computePolicy = &types.ComputePolicy{} + computePolicy = &vcdapitypes.ComputePolicy{} } computePolicy.VmPlacementPolicy = &vcdapitypes.Reference{ HREF: placementPolicy.VdcComputePolicy.ID, @@ -111,7 +110,7 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * } // 3. Retrieve Storage Profile - var storageProfile *types.Reference + var storageProfile *vcdapitypes.Reference if c.StorageProfile != nil && *c.StorageProfile != defaultStorageProfile { for _, sp := range vdc.Vdc.VdcStorageProfiles.VdcStorageProfile { if sp.Name == *c.StorageProfile || sp.ID == *c.StorageProfile { @@ -131,20 +130,20 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * // // It is not possible to customize compute, disk and network for a VM at initial creation time when we are using templates. So we rely on // vApp re-composition to apply the needed customization, performed at later stages. - vAppRecomposition := &types.ReComposeVAppParams{ - Ovf: types.XMLNamespaceOVF, - Xsi: types.XMLNamespaceXSI, - Xmlns: types.XMLNamespaceVCloud, + vAppRecomposition := &vcdapitypes.ReComposeVAppParams{ + Ovf: vcdapitypes.XMLNamespaceOVF, + Xsi: vcdapitypes.XMLNamespaceXSI, + Xmlns: vcdapitypes.XMLNamespaceVCloud, Deploy: false, Name: vapp.VApp.Name, PowerOn: false, Description: vapp.VApp.Description, - SourcedItem: &types.SourcedCompositionItemParam{ - Source: &types.Reference{ + SourcedItem: &vcdapitypes.SourcedCompositionItemParam{ + Source: &vcdapitypes.Reference{ HREF: templateHref, Name: machine.Name, }, - InstantiationParams: &types.InstantiationParams{ + InstantiationParams: &vcdapitypes.InstantiationParams{ NetworkConnectionSection: &vcdapitypes.NetworkConnectionSection{ NetworkConnection: []*vcdapitypes.NetworkConnection{ { @@ -170,7 +169,7 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * apiEndpoint.Path = path.Join(apiEndpoint.Path, "action/recomposeVApp") task, err := client.VCDClient.Client.ExecuteTaskRequest(apiEndpoint.String(), http.MethodPost, - types.MimeRecomposeVappParams, "error instantiating a new VM: %s", vAppRecomposition) + vcdapitypes.MimeRecomposeVappParams, "error instantiating a new VM: %s", vAppRecomposition) if err != nil { return fmt.Errorf("failed to execute API call to create VM: %w", err) } diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go index 3479bfbe5..351408b51 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -26,13 +26,12 @@ import ( "github.com/vmware/go-vcloud-director/v2/govcd" "go.uber.org/zap" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - vcdtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/vmwareclouddirector/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + vcdtypes "k8c.io/machine-controller/sdk/cloudprovider/vmwareclouddirector" + "k8c.io/machine-controller/sdk/providerconfig" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -255,7 +254,7 @@ func (p *provider) create(_ context.Context, machine *clusterv1alpha1.Machine, u // 5. Before powering on the VM, configure customization to attach userdata with the VM // update guest properties. - err = setUserData(userdata, vm, providerConfig.OperatingSystem == providerconfigtypes.OperatingSystemFlatcar) + err = setUserData(userdata, vm, providerConfig.OperatingSystem == providerconfig.OperatingSystemFlatcar) if err != nil { return nil, err } @@ -309,8 +308,8 @@ func (p *provider) Get(_ context.Context, _ *zap.SugaredLogger, machine *cluster return p.getInstance(vm) } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *vcdtypes.RawConfig, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfig.Config, *vcdtypes.RawConfig, error) { + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, nil, err } @@ -549,7 +548,7 @@ func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec cluste } func setProviderSpec(rawConfig vcdtypes.RawConfig, provSpec clusterv1alpha1.ProviderSpec) (*runtime.RawExtension, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, err } diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 4bcd6686a..7dc1aff5a 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -31,14 +31,13 @@ import ( "github.com/vmware/govmomi/vim25/types" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - vspheretypes "k8c.io/machine-controller/pkg/cloudprovider/provider/vsphere/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + vspheretypes "k8c.io/machine-controller/sdk/cloudprovider/vsphere" + "k8c.io/machine-controller/sdk/providerconfig" corev1 "k8s.io/api/core/v1" ktypes "k8s.io/apimachinery/pkg/types" @@ -120,8 +119,8 @@ func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.Machin return spec, nil } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, *vspheretypes.RawConfig, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfig.Config, *vspheretypes.RawConfig, error) { + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, nil, err } @@ -365,7 +364,7 @@ func (p *provider) create(ctx context.Context, log *zap.SugaredLogger, machine * defer session.Logout(ctx) var containerLinuxUserdata string - if pc.OperatingSystem == providerconfigtypes.OperatingSystemFlatcar { + if pc.OperatingSystem == providerconfig.OperatingSystemFlatcar { containerLinuxUserdata = userdata } @@ -397,7 +396,7 @@ func (p *provider) create(ctx context.Context, log *zap.SugaredLogger, machine * } } - if pc.OperatingSystem != providerconfigtypes.OperatingSystemFlatcar { + if pc.OperatingSystem != providerconfig.OperatingSystemFlatcar { localUserdataIsoFilePath, err := generateLocalUserdataISO(userdata, machine.Spec.Name) if err != nil { return nil, fmt.Errorf("failed to generate local userdadata iso: %w", err) @@ -520,7 +519,7 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine return false, fmt.Errorf("failed to destroy vm %s: %w", virtualMachine.Name(), err) } - if pc.OperatingSystem != providerconfigtypes.OperatingSystemFlatcar { + if pc.OperatingSystem != providerconfig.OperatingSystemFlatcar { filemanager := datastore.NewFileManager(session.Datacenter, false) if err := filemanager.Delete(ctx, virtualMachine.Name()); err != nil { diff --git a/pkg/cloudprovider/provider/vsphere/provider_test.go b/pkg/cloudprovider/provider/vsphere/provider_test.go index a3e15fef6..33fa89e52 100644 --- a/pkg/cloudprovider/provider/vsphere/provider_test.go +++ b/pkg/cloudprovider/provider/vsphere/provider_test.go @@ -28,7 +28,7 @@ import ( "go.uber.org/zap" cloudprovidertesting "k8c.io/machine-controller/pkg/cloudprovider/testing" - "k8c.io/machine-controller/pkg/providerconfig" + "k8c.io/machine-controller/sdk/providerconfig" "k8s.io/utils/ptr" fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" diff --git a/pkg/cloudprovider/provider/vsphere/rule.go b/pkg/cloudprovider/provider/vsphere/rule.go index 94408dc4b..fdb1f6d67 100644 --- a/pkg/cloudprovider/provider/vsphere/rule.go +++ b/pkg/cloudprovider/provider/vsphere/rule.go @@ -29,7 +29,7 @@ import ( "github.com/vmware/govmomi/vim25/types" "go.uber.org/zap" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" "k8s.io/utils/ptr" ) diff --git a/pkg/cloudprovider/provider/vsphere/types/types.go b/pkg/cloudprovider/provider/vsphere/types/types.go deleted file mode 100644 index c8ab0c85a..000000000 --- a/pkg/cloudprovider/provider/vsphere/types/types.go +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" -) - -// RawConfig represents vsphere specific configuration. -type RawConfig struct { - TemplateVMName providerconfigtypes.ConfigVarString `json:"templateVMName"` - // Deprecated: use networks instead. - VMNetName providerconfigtypes.ConfigVarString `json:"vmNetName"` - Networks []providerconfigtypes.ConfigVarString `json:"networks"` - Username providerconfigtypes.ConfigVarString `json:"username"` - Password providerconfigtypes.ConfigVarString `json:"password"` - VSphereURL providerconfigtypes.ConfigVarString `json:"vsphereURL"` - Datacenter providerconfigtypes.ConfigVarString `json:"datacenter"` - - // Cluster defines the cluster to use in vcenter. - // Only needed for vm anti affinity. - Cluster providerconfigtypes.ConfigVarString `json:"cluster"` - - Folder providerconfigtypes.ConfigVarString `json:"folder"` - ResourcePool providerconfigtypes.ConfigVarString `json:"resourcePool"` - - // Either Datastore or DatastoreCluster have to be provided. - DatastoreCluster providerconfigtypes.ConfigVarString `json:"datastoreCluster"` - Datastore providerconfigtypes.ConfigVarString `json:"datastore"` - - CPUs int32 `json:"cpus"` - MemoryMB int64 `json:"memoryMB"` - DiskSizeGB *int64 `json:"diskSizeGB,omitempty"` - Tags []Tag `json:"tags,omitempty"` - AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` - - // Placement rules - VMAntiAffinity providerconfigtypes.ConfigVarBool `json:"vmAntiAffinity"` - VMGroup providerconfigtypes.ConfigVarString `json:"vmGroup,omitempty"` -} - -// Tag represents vsphere tag. -type Tag struct { - Description string `json:"description,omitempty"` - ID string `json:"id,omitempty"` - Name string `json:"name,omitempty"` - CategoryID string `json:"categoryID"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/provider/vsphere/vmgroup.go b/pkg/cloudprovider/provider/vsphere/vmgroup.go index 1b51a5468..5cea8f01c 100644 --- a/pkg/cloudprovider/provider/vsphere/vmgroup.go +++ b/pkg/cloudprovider/provider/vsphere/vmgroup.go @@ -21,10 +21,10 @@ import ( "fmt" "strings" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/vmware/govmomi/vim25/types" "go.uber.org/zap" + + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" ) func (p *provider) addToVMGroup(ctx context.Context, log *zap.SugaredLogger, session *Session, machine *clusterv1alpha1.Machine, config *Config) error { diff --git a/pkg/cloudprovider/provider/vultr/provider.go b/pkg/cloudprovider/provider/vultr/provider.go index d51bac3d2..f308943e1 100644 --- a/pkg/cloudprovider/provider/vultr/provider.go +++ b/pkg/cloudprovider/provider/vultr/provider.go @@ -29,16 +29,15 @@ import ( "go.uber.org/zap" "golang.org/x/oauth2" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" - vultrtypes "k8c.io/machine-controller/pkg/cloudprovider/provider/vultr/types" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + vultrtypes "k8c.io/machine-controller/sdk/cloudprovider/vultr" + "k8c.io/machine-controller/sdk/providerconfig" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" @@ -78,15 +77,15 @@ type Config struct { Vpc2ID []string } -func getIDForOS(os providerconfigtypes.OperatingSystem) (int, error) { +func getIDForOS(os providerconfig.OperatingSystem) (int, error) { switch os { - case providerconfigtypes.OperatingSystemUbuntu: + case providerconfig.OperatingSystemUbuntu: return 1743, nil // name: Rocky Linux 9 x64 - case providerconfigtypes.OperatingSystemRockyLinux: + case providerconfig.OperatingSystemRockyLinux: return 1869, nil } - return 0, providerconfigtypes.ErrOSNotSupported + return 0, providerconfig.ErrOSNotSupported } func getClient(ctx context.Context, apiKey string) *govultr.Client { @@ -95,8 +94,8 @@ func getClient(ctx context.Context, apiKey string) *govultr.Client { return govultr.NewClient(oauth2.NewClient(ctx, ts)) } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { - pconfig, err := providerconfigtypes.GetConfig(provSpec) +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfig.Config, error) { + pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { return nil, nil, err } @@ -585,15 +584,15 @@ func (v *vultrPhysicalMachine) ProviderID() string { return "vultr://" + v.instance.ID } -func (v *vultrVirtualMachine) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{} - addresses[v.instance.MainIP] = v1.NodeExternalIP - addresses[v.instance.InternalIP] = v1.NodeInternalIP +func (v *vultrVirtualMachine) Addresses() map[string]corev1.NodeAddressType { + addresses := map[string]corev1.NodeAddressType{} + addresses[v.instance.MainIP] = corev1.NodeExternalIP + addresses[v.instance.InternalIP] = corev1.NodeInternalIP return addresses } -func (v *vultrPhysicalMachine) Addresses() map[string]v1.NodeAddressType { - addresses := map[string]v1.NodeAddressType{} - addresses[v.instance.MainIP] = v1.NodeExternalIP +func (v *vultrPhysicalMachine) Addresses() map[string]corev1.NodeAddressType { + addresses := map[string]corev1.NodeAddressType{} + addresses[v.instance.MainIP] = corev1.NodeExternalIP return addresses } diff --git a/pkg/cloudprovider/provider/vultr/types/types.go b/pkg/cloudprovider/provider/vultr/types/types.go deleted file mode 100644 index e016d6288..000000000 --- a/pkg/cloudprovider/provider/vultr/types/types.go +++ /dev/null @@ -1,42 +0,0 @@ -/* -Copyright 2023 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package types - -import ( - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" -) - -type RawConfig struct { - PhysicalMachine bool `json:"physicalMachine,omitempty"` - APIKey providerconfigtypes.ConfigVarString `json:"apiKey,omitempty"` - Region providerconfigtypes.ConfigVarString `json:"region"` - Plan providerconfigtypes.ConfigVarString `json:"plan"` - OsID providerconfigtypes.ConfigVarString `json:"osId"` - Tags []string `json:"tags,omitempty"` - VpcID []string `json:"vpcId,omitempty"` - Vpc2ID []string `json:"vpc2Id,omitempty"` - EnableVPC bool `json:"enableVPC,omitempty"` - EnableVPC2 bool `json:"enableVPC2,omitempty"` - EnableIPv6 bool `json:"enableIPv6,omitempty"` -} - -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { - rawConfig := &RawConfig{} - - return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) -} diff --git a/pkg/cloudprovider/testing/testing.go b/pkg/cloudprovider/testing/testing.go index 2069f4407..25ba4f08a 100644 --- a/pkg/cloudprovider/testing/testing.go +++ b/pkg/cloudprovider/testing/testing.go @@ -19,7 +19,7 @@ package testing import ( "testing" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -35,18 +35,18 @@ type Creator struct { ProviderSpecGetter ProviderSpecGetter } -func (c Creator) CreateMachine(t *testing.T) *v1alpha1.Machine { - return &v1alpha1.Machine{ +func (c Creator) CreateMachine(t *testing.T) *clusterv1alpha1.Machine { + return &clusterv1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{ Name: c.Name, Namespace: c.Namespace, }, - Spec: v1alpha1.MachineSpec{ + Spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{ Name: c.Name, Namespace: c.Namespace, }, - ProviderSpec: v1alpha1.ProviderSpec{ + ProviderSpec: clusterv1alpha1.ProviderSpec{ Value: &runtime.RawExtension{ Raw: c.ProviderSpecGetter(t), }, diff --git a/pkg/cloudprovider/types/types.go b/pkg/cloudprovider/types/types.go index c8142d366..e9e3c241e 100644 --- a/pkg/cloudprovider/types/types.go +++ b/pkg/cloudprovider/types/types.go @@ -22,8 +22,8 @@ import ( "go.uber.org/zap" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8c.io/machine-controller/pkg/cloudprovider/instance" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/types" diff --git a/pkg/cloudprovider/util/cloud_init_settings.go b/pkg/cloudprovider/util/cloud_init_settings.go index ed32c6e5a..46f185d16 100644 --- a/pkg/cloudprovider/util/cloud_init_settings.go +++ b/pkg/cloudprovider/util/cloud_init_settings.go @@ -24,7 +24,7 @@ import ( "gopkg.in/yaml.v3" corev1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/clientcmd" @@ -58,7 +58,7 @@ func ExtractTokenAndAPIServer(ctx context.Context, userdata string, client ctrlr func CreateMachineCloudInitSecret(ctx context.Context, userdata, machineName string, client ctrlruntimeclient.Client) error { secret := &corev1.Secret{} if err := client.Get(ctx, types.NamespacedName{Namespace: CloudInitNamespace, Name: machineName}, secret); err != nil { - if kerrors.IsNotFound(err) { + if apierrors.IsNotFound(err) { secret = &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: machineName, diff --git a/pkg/cloudprovider/util/net.go b/pkg/cloudprovider/util/net.go index bdb1fb2a7..fc4ac076f 100644 --- a/pkg/cloudprovider/util/net.go +++ b/pkg/cloudprovider/util/net.go @@ -23,11 +23,6 @@ import ( "net" ) -const ( - ErrIPv6OnlyUnsupported = "IPv6 only network family not supported yet" - ErrUnknownNetworkFamily = "Unknown IP family %q only IPv4,IPv6,IPv4+IPv6 are valid values" -) - func CIDRToIPAndNetMask(ipv4 string) (string, string, int, error) { ip, ipNet, err := net.ParseCIDR(ipv4) if err != nil { @@ -59,32 +54,3 @@ func GenerateRandMAC() (net.HardwareAddr, error) { return mac, nil } - -// IPFamily IPv4 | IPv6 | IPv4+IPv6. -type IPFamily string - -const ( - IPFamilyUnspecified IPFamily = "" // interpreted as IPv4 - IPFamilyIPv4 IPFamily = "IPv4" // IPv4 only - IPFamilyIPv6 IPFamily = "IPv6" // IPv6 only - IPFamilyIPv4IPv6 IPFamily = "IPv4+IPv6" // dualstack with IPv4 as primary - IPFamilyIPv6IPv4 IPFamily = "IPv6+IPv4" // dualstack with IPv6 as primary -) - -func (f IPFamily) HasIPv6() bool { - return f == IPFamilyIPv6 || f == IPFamilyIPv4IPv6 || f == IPFamilyIPv6IPv4 -} - -func (f IPFamily) HasIPv4() bool { - return f == IPFamilyUnspecified || f == IPFamilyIPv4 || f == IPFamilyIPv4IPv6 || f == IPFamilyIPv6IPv4 -} - -func (f IPFamily) IsDualstack() bool { - return f == IPFamilyIPv4IPv6 || f == IPFamilyIPv6IPv4 -} - -// IsLinkLocal checks if given ip address is link local.. -func IsLinkLocal(ipAddr string) bool { - addr := net.ParseIP(ipAddr) - return addr.IsLinkLocalMulticast() || addr.IsLinkLocalUnicast() -} diff --git a/pkg/cloudprovider/util/util.go b/pkg/cloudprovider/util/util.go index 2bf06bdca..2801f835b 100644 --- a/pkg/cloudprovider/util/util.go +++ b/pkg/cloudprovider/util/util.go @@ -19,20 +19,20 @@ package util import ( "fmt" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" kuberneteshelper "k8c.io/machine-controller/pkg/kubernetes" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" ) // RemoveFinalizerOnInstanceNotFound checks whether a finalizer exists and removes it on demand. func RemoveFinalizerOnInstanceNotFound(finalizer string, - machine *v1alpha1.Machine, + machine *clusterv1alpha1.Machine, provider *cloudprovidertypes.ProviderData) (bool, error) { if !kuberneteshelper.HasFinalizer(machine, finalizer) { return true, nil } - if err := provider.Update(machine, func(updatedMachine *v1alpha1.Machine) { + if err := provider.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { updatedMachine.Finalizers = kuberneteshelper.RemoveFinalizer(updatedMachine.Finalizers, finalizer) }); err != nil { return false, fmt.Errorf("failed updating machine %v finzaliers: %w", machine.Name, err) diff --git a/pkg/cloudprovider/util/util_test.go b/pkg/cloudprovider/util/util_test.go index 13734a75a..c6367d2d6 100644 --- a/pkg/cloudprovider/util/util_test.go +++ b/pkg/cloudprovider/util/util_test.go @@ -21,9 +21,8 @@ import ( "reflect" "testing" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -39,7 +38,7 @@ func TestRemoveFinalizerOnInstanceNotFound(t *testing.T) { var fakeClient = fakectrlruntimeclient. NewClientBuilder(). WithScheme(scheme.Scheme). - WithObjects(&v1alpha1.Machine{ + WithObjects(&clusterv1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{ Name: "test_machine", Finalizers: []string{ @@ -51,13 +50,13 @@ func TestRemoveFinalizerOnInstanceNotFound(t *testing.T) { var testCases = []struct { name string - machine *v1alpha1.Machine - expectedMachine *v1alpha1.Machine + machine *clusterv1alpha1.Machine + expectedMachine *clusterv1alpha1.Machine providerData *cloudprovidertypes.ProviderData }{ { name: "Test remove machine finalizer", - machine: &v1alpha1.Machine{ + machine: &clusterv1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{ UID: "123456", Name: "test_machine", @@ -66,7 +65,7 @@ func TestRemoveFinalizerOnInstanceNotFound(t *testing.T) { "test_finalizer_2"}, }, }, - expectedMachine: &v1alpha1.Machine{ + expectedMachine: &clusterv1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{ UID: "123456", Name: "test_machine", @@ -87,7 +86,7 @@ func TestRemoveFinalizerOnInstanceNotFound(t *testing.T) { t.Fatalf("failed removing finalizer: %v", err) } - foundMachine := &v1alpha1.Machine{} + foundMachine := &clusterv1alpha1.Machine{} if err := fakeClient.Get( context.Background(), types.NamespacedName{Name: "test_machine"}, diff --git a/pkg/cloudprovider/validationwrapper.go b/pkg/cloudprovider/validationwrapper.go index a3bb979a8..8f98e54f8 100644 --- a/pkg/cloudprovider/validationwrapper.go +++ b/pkg/cloudprovider/validationwrapper.go @@ -23,9 +23,9 @@ import ( "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8c.io/machine-controller/pkg/cloudprovider/instance" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" "k8s.io/apimachinery/pkg/types" ) @@ -40,13 +40,13 @@ func NewValidationCacheWrappingCloudProvider(actualProvider cloudprovidertypes.P } // AddDefaults just calls the underlying cloudproviders AddDefaults. -func (w *cachingValidationWrapper) AddDefaults(log *zap.SugaredLogger, spec v1alpha1.MachineSpec) (v1alpha1.MachineSpec, error) { +func (w *cachingValidationWrapper) AddDefaults(log *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { return w.actualProvider.AddDefaults(log, spec) } // Validate tries to get the validation result from the cache and if not found, calls the // cloudproviders Validate and saves that to the cache. -func (w *cachingValidationWrapper) Validate(ctx context.Context, log *zap.SugaredLogger, spec v1alpha1.MachineSpec) error { +func (w *cachingValidationWrapper) Validate(ctx context.Context, log *zap.SugaredLogger, spec clusterv1alpha1.MachineSpec) error { result, exists, err := cache.Get(spec) if err != nil { return fmt.Errorf("error getting validation result from cache: %w", err) @@ -71,30 +71,30 @@ func (w *cachingValidationWrapper) Validate(ctx context.Context, log *zap.Sugare } // Get just calls the underlying cloudproviders Get. -func (w *cachingValidationWrapper) Get(ctx context.Context, log *zap.SugaredLogger, machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { +func (w *cachingValidationWrapper) Get(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (instance.Instance, error) { return w.actualProvider.Get(ctx, log, machine, data) } // Create just calls the underlying cloudproviders Create. -func (w *cachingValidationWrapper) Create(ctx context.Context, log *zap.SugaredLogger, machine *v1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (w *cachingValidationWrapper) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { return w.actualProvider.Create(ctx, log, machine, data, userdata) } // Cleanup just calls the underlying cloudproviders Cleanup. -func (w *cachingValidationWrapper) Cleanup(ctx context.Context, log *zap.SugaredLogger, m *v1alpha1.Machine, mcd *cloudprovidertypes.ProviderData) (bool, error) { +func (w *cachingValidationWrapper) Cleanup(ctx context.Context, log *zap.SugaredLogger, m *clusterv1alpha1.Machine, mcd *cloudprovidertypes.ProviderData) (bool, error) { return w.actualProvider.Cleanup(ctx, log, m, mcd) } // MigrateUID just calls the underlying cloudproviders MigrateUID. -func (w *cachingValidationWrapper) MigrateUID(ctx context.Context, log *zap.SugaredLogger, m *v1alpha1.Machine, newUID types.UID) error { +func (w *cachingValidationWrapper) MigrateUID(ctx context.Context, log *zap.SugaredLogger, m *clusterv1alpha1.Machine, newUID types.UID) error { return w.actualProvider.MigrateUID(ctx, log, m, newUID) } // MachineMetricsLabels just calls the underlying cloudproviders MachineMetricsLabels. -func (w *cachingValidationWrapper) MachineMetricsLabels(machine *v1alpha1.Machine) (map[string]string, error) { +func (w *cachingValidationWrapper) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { return w.actualProvider.MachineMetricsLabels(machine) } -func (w *cachingValidationWrapper) SetMetricsForMachines(machines v1alpha1.MachineList) error { +func (w *cachingValidationWrapper) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { return w.actualProvider.SetMetricsForMachines(machines) } diff --git a/pkg/controller/machine/controller.go b/pkg/controller/machine/controller.go index ea9addf71..00f57f68b 100644 --- a/pkg/controller/machine/controller.go +++ b/pkg/controller/machine/controller.go @@ -31,9 +31,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/common" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" - "k8c.io/machine-controller/pkg/bootstrap" "k8c.io/machine-controller/pkg/cloudprovider" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" "k8c.io/machine-controller/pkg/cloudprovider/instance" @@ -43,14 +40,16 @@ import ( kuberneteshelper "k8c.io/machine-controller/pkg/kubernetes" "k8c.io/machine-controller/pkg/node/eviction" "k8c.io/machine-controller/pkg/node/poddeletion" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" "k8c.io/machine-controller/pkg/rhsm" - "k8c.io/machine-controller/pkg/userdata/rhel" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + "k8c.io/machine-controller/sdk/bootstrap" + "k8c.io/machine-controller/sdk/providerconfig" + "k8c.io/machine-controller/sdk/userdata/rhel" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" - kerrors "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" @@ -364,7 +363,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( machine := &clusterv1alpha1.Machine{} if err := r.client.Get(ctx, request.NamespacedName, machine); err != nil { - if kerrors.IsNotFound(err) { + if apierrors.IsNotFound(err) { return reconcile.Result{}, nil } log.Errorw("Failed to get Machine", zap.Error(err)) @@ -406,7 +405,7 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, mach machine.Spec.Name = machine.Name } - providerConfig, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) + providerConfig, err := providerconfig.GetConfig(machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("failed to get provider config: %w", err) } @@ -432,7 +431,7 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, mach node, err := r.getNodeByNodeRef(ctx, machine.Status.NodeRef) if err != nil { // In case we cannot find a node for the NodeRef we must remove the NodeRef & recreate an instance on the next sync - if kerrors.IsNotFound(err) { + if apierrors.IsNotFound(err) { log.Info("Found invalid NodeRef on machine; deleting reference...") return nil, r.updateMachine(machine, func(m *clusterv1alpha1.Machine) { m.Status.NodeRef = nil @@ -457,7 +456,7 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, mach } // case 3.2: if the node exists and both external and internal CCM are not available. Then set the provider-id for the node. - inTree := providerconfigtypes.IntreeCloudProviderImplementationSupported(providerConfig.CloudProvider) + inTree := providerconfig.IntreeCloudProviderImplementationSupported(providerConfig.CloudProvider) if !inTree && !r.nodeSettings.ExternalCloudProvider && node.Spec.ProviderID == "" { providerID := fmt.Sprintf(ProviderIDPattern, providerConfig.CloudProvider, machine.UID) if err := r.updateNode(ctx, node, func(n *corev1.Node) { @@ -496,7 +495,7 @@ func (r *Reconciler) machineHasValidNode(ctx context.Context, machine *clusterv1 node := &corev1.Node{} if err := r.client.Get(ctx, types.NamespacedName{Name: machine.Status.NodeRef.Name}, node); err != nil { - if kerrors.IsNotFound(err) { + if apierrors.IsNotFound(err) { return false, nil } @@ -506,9 +505,9 @@ func (r *Reconciler) machineHasValidNode(ctx context.Context, machine *clusterv1 return true, nil } -func (r *Reconciler) shouldCleanupVolumes(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, providerName providerconfigtypes.CloudProvider) (bool, error) { +func (r *Reconciler) shouldCleanupVolumes(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, providerName providerconfig.CloudProvider) (bool, error) { // we need to wait for volumeAttachments clean up only for vSphere - if providerName != providerconfigtypes.CloudProviderVsphere { + if providerName != providerconfig.CloudProviderVsphere { return false, nil } @@ -581,7 +580,7 @@ func (r *Reconciler) deleteMachine( ctx context.Context, log *zap.SugaredLogger, prov cloudprovidertypes.Provider, - providerName providerconfigtypes.CloudProvider, + providerName providerconfig.CloudProvider, machine *clusterv1alpha1.Machine, skipEviction bool, ) (*reconcile.Result, error) { @@ -657,7 +656,7 @@ func (r *Reconciler) retrieveNodesRelatedToMachine(ctx context.Context, log *zap objKey := ctrlruntimeclient.ObjectKey{Name: machine.Status.NodeRef.Name} node := &corev1.Node{} if err := r.client.Get(ctx, objKey, node); err != nil { - if !kerrors.IsNotFound(err) { + if !apierrors.IsNotFound(err) { return nil, fmt.Errorf("failed to get node %s: %w", machine.Status.NodeRef.Name, err) } log.Debugw("Node does not longer exist for machine", "node", machine.Status.NodeRef.Name) @@ -704,19 +703,19 @@ func (r *Reconciler) deleteCloudProviderInstance(ctx context.Context, log *zap.S return &reconcile.Result{RequeueAfter: deletionRetryWaitPeriod}, nil } - machineConfig, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) + machineConfig, err := providerconfig.GetConfig(machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("failed to get provider config: %w", err) } - if machineConfig.OperatingSystem == providerconfigtypes.OperatingSystemRHEL { + if machineConfig.OperatingSystem == providerconfig.OperatingSystemRHEL { rhelConfig, err := rhel.LoadConfig(machineConfig.OperatingSystemSpec) if err != nil { return nil, fmt.Errorf("failed to get rhel os specs: %w", err) } machineName := machine.Name - if machineConfig.CloudProvider == providerconfigtypes.CloudProviderAWS { + if machineConfig.CloudProvider == providerconfig.CloudProviderAWS { for _, address := range machine.Status.Addresses { if address.Type == corev1.NodeInternalDNS { machineName = address.Address @@ -755,7 +754,7 @@ func (r *Reconciler) deleteCloudProviderInstance(ctx context.Context, log *zap.S // removed by an administrator or an external service. This is because the machine controller lacks access to cloud // instances and cannot ensure their deletion. If the external service fails to delete the instance, it may result // in orphaned resources or nodes without a machine reference. - if machineConfig.CloudProvider != providerconfigtypes.CloudProviderExternal { + if machineConfig.CloudProvider != providerconfig.CloudProviderExternal { finalizers.Delete(FinalizerDeleteInstance) m.Finalizers = finalizers.List() } @@ -766,7 +765,7 @@ func (r *Reconciler) deleteNodeForMachine(ctx context.Context, log *zap.SugaredL // iterates on all nodes and delete them. Finally, remove the finalizer on the machine for _, node := range nodes { if err := r.client.Delete(ctx, node); err != nil { - if !kerrors.IsNotFound(err) { + if !apierrors.IsNotFound(err) { return err } log.Infow("Node does not longer exist for machine", "node", machine.Status.NodeRef.Name) @@ -788,7 +787,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( log *zap.SugaredLogger, prov cloudprovidertypes.Provider, machine *clusterv1alpha1.Machine, - providerConfig *providerconfigtypes.Config, + providerConfig *providerconfig.Config, ) (*reconcile.Result, error) { log.Debug("Requesting instance for machine from cloudprovider because no associated node with status ready found...") @@ -832,7 +831,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( message := fmt.Sprintf("%v. Failed to create a machine.", err) return nil, r.updateMachineErrorIfTerminalError(machine, common.CreateMachineError, message, err, "failed to create machine at cloudprovider") } - if providerConfig.OperatingSystem == providerconfigtypes.OperatingSystemRHEL { + if providerConfig.OperatingSystem == providerconfig.OperatingSystemRHEL { if err := rhsm.AddRHELSubscriptionFinalizer(machine, r.updateMachine); err != nil { return nil, fmt.Errorf("failed to add redhat subscription finalizer: %w", err) } @@ -899,7 +898,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( var providerID string if machine.Spec.ProviderID == nil { - inTree := providerconfigtypes.IntreeCloudProviderImplementationSupported(providerConfig.CloudProvider) + inTree := providerconfig.IntreeCloudProviderImplementationSupported(providerConfig.CloudProvider) // If both external and internal CCM are not available. We set provider-id for the machine explicitly. if !inTree && !r.nodeSettings.ExternalCloudProvider { providerID = fmt.Sprintf(ProviderIDPattern, providerConfig.CloudProvider, machine.UID) @@ -918,7 +917,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine( return r.ensureNodeOwnerRef(ctx, log, providerInstance, machine, providerConfig) } -func (r *Reconciler) ensureNodeOwnerRef(ctx context.Context, log *zap.SugaredLogger, providerInstance instance.Instance, machine *clusterv1alpha1.Machine, providerConfig *providerconfigtypes.Config) (*reconcile.Result, error) { +func (r *Reconciler) ensureNodeOwnerRef(ctx context.Context, log *zap.SugaredLogger, providerInstance instance.Instance, machine *clusterv1alpha1.Machine, providerConfig *providerconfig.Config) (*reconcile.Result, error) { node, exists, err := r.getNode(ctx, log, providerInstance, providerConfig.CloudProvider) if err != nil { return nil, fmt.Errorf("failed to get node for machine %s: %w", machine.Name, err) @@ -1043,7 +1042,7 @@ func (r *Reconciler) updateMachineStatus(machine *clusterv1alpha1.Machine, node return nil } -func (r *Reconciler) getNode(ctx context.Context, log *zap.SugaredLogger, instance instance.Instance, provider providerconfigtypes.CloudProvider) (node *corev1.Node, exists bool, err error) { +func (r *Reconciler) getNode(ctx context.Context, log *zap.SugaredLogger, instance instance.Instance, provider providerconfig.CloudProvider) (node *corev1.Node, exists bool, err error) { if instance == nil { return nil, false, fmt.Errorf("getNode called with nil provider instance") } @@ -1081,7 +1080,7 @@ func (r *Reconciler) getNode(ctx context.Context, log *zap.SugaredLogger, instan // TODO: We should do this for other providers, but there are providers where // the node and the instance names will not match, so it requires further // investigation (e.g. AWS). - if provider == providerconfigtypes.CloudProviderHetzner && node.Name != instance.Name() { + if provider == providerconfig.CloudProviderHetzner && node.Name != instance.Name() { continue } if nodeAddress.Address == instanceAddress { @@ -1094,7 +1093,7 @@ func (r *Reconciler) getNode(ctx context.Context, log *zap.SugaredLogger, instan return nil, false, nil } -func findNodeByProviderID(instance instance.Instance, provider providerconfigtypes.CloudProvider, nodes []corev1.Node) *corev1.Node { +func findNodeByProviderID(instance instance.Instance, provider providerconfig.CloudProvider, nodes []corev1.Node) *corev1.Node { providerID := instance.ProviderID() if providerID == "" { return nil @@ -1109,7 +1108,7 @@ func findNodeByProviderID(instance instance.Instance, provider providerconfigtyp // * aws://// // * aws:/// // The first case is handled above, while the second here is handled here. - if provider == providerconfigtypes.CloudProviderAWS { + if provider == providerconfig.CloudProviderAWS { pid := strings.Split(node.Spec.ProviderID, "aws:///") if len(pid) == 2 && pid[1] == instance.ID() { return node.DeepCopy() @@ -1205,7 +1204,7 @@ func (r *Reconciler) handleNodeFailuresWithExternalCCM( ctx context.Context, log *zap.SugaredLogger, prov cloudprovidertypes.Provider, - provConfig *providerconfigtypes.Config, + provConfig *providerconfig.Config, node *corev1.Node, machine *clusterv1alpha1.Machine, ) (*reconcile.Result, error) { @@ -1223,10 +1222,10 @@ func (r *Reconciler) handleNodeFailuresWithExternalCCM( return nil, err } else if taintExists(node, taintShutdown) { switch provConfig.CloudProvider { - case providerconfigtypes.CloudProviderKubeVirt: + case providerconfig.CloudProviderKubeVirt: log.Infof("Deleting a shut-down machine %q that cannot recover", machine.Name) skipEviction := true - return r.deleteMachine(ctx, log, prov, providerconfigtypes.CloudProviderKubeVirt, machine, skipEviction) + return r.deleteMachine(ctx, log, prov, providerconfig.CloudProviderKubeVirt, machine, skipEviction) } } diff --git a/pkg/controller/machine/controller_test.go b/pkg/controller/machine/controller_test.go index f505c2434..c5d9834d7 100644 --- a/pkg/controller/machine/controller_test.go +++ b/pkg/controller/machine/controller_test.go @@ -25,19 +25,18 @@ import ( "github.com/go-test/deep" "go.uber.org/zap" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8c.io/machine-controller/pkg/cloudprovider/instance" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + providerconfigtypes "k8c.io/machine-controller/sdk/providerconfig" corev1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" - ctrlruntimefake "sigs.k8s.io/controller-runtime/pkg/client/fake" fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -319,7 +318,7 @@ func TestControllerDeletesMachinesOnJoinTimeout(t *testing.T) { } err := client.Get(ctx, types.NamespacedName{Name: machine.Name}, &clusterv1alpha1.Machine{}) - wasDeleted := kerrors.IsNotFound(err) + wasDeleted := apierrors.IsNotFound(err) if wasDeleted != test.getsDeleted { t.Errorf("Machine was deleted: %v, but expectedDeletion: %v", wasDeleted, test.getsDeleted) @@ -464,7 +463,7 @@ func TestControllerShouldEvict(t *testing.T) { objects = append(objects, test.existingNodes...) objects = append(objects, test.additionalMachines...) - client := ctrlruntimefake.NewClientBuilder(). + client := fakectrlruntimeclient.NewClientBuilder(). WithScheme(scheme.Scheme). WithObjects(objects...). Build() @@ -642,7 +641,7 @@ func TestControllerDeleteNodeForMachine(t *testing.T) { if test.shouldDeleteNode != "" { err = client.Get(ctx, types.NamespacedName{Name: test.shouldDeleteNode}, &corev1.Node{}) - if !kerrors.IsNotFound(err) { + if !apierrors.IsNotFound(err) { t.Errorf("expected node %q to be deleted, but got: %v", test.shouldDeleteNode, err) } } else { diff --git a/pkg/controller/machine/metrics.go b/pkg/controller/machine/metrics.go index b5f681bdb..64399c6dd 100644 --- a/pkg/controller/machine/metrics.go +++ b/pkg/controller/machine/metrics.go @@ -23,10 +23,9 @@ import ( "github.com/prometheus/client_golang/prometheus" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8c.io/machine-controller/pkg/cloudprovider" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + "k8c.io/machine-controller/sdk/providerconfig" "k8s.io/apimachinery/pkg/api/equality" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -80,8 +79,8 @@ type MachineCollector struct { type machineMetricLabels struct { KubeletVersion string - CloudProvider providerconfigtypes.CloudProvider - OperatingSystem providerconfigtypes.OperatingSystem + CloudProvider providerconfig.CloudProvider + OperatingSystem providerconfig.OperatingSystem ProviderLabels map[string]string } @@ -139,9 +138,9 @@ func NewMachineCollector(ctx context.Context, client ctrlruntimeclient.Client) * return } - providerMachineMap := map[providerconfigtypes.CloudProvider]*clusterv1alpha1.MachineList{} + providerMachineMap := map[providerconfig.CloudProvider]*clusterv1alpha1.MachineList{} for _, machine := range machines.Items { - providerConfig, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) + providerConfig, err := providerconfig.GetConfig(machine.Spec.ProviderSpec) if err != nil { utilruntime.HandleError(fmt.Errorf("failed to get providerSpec for SetMetricsForMachines: %w", err)) continue @@ -226,7 +225,7 @@ func (mc MachineCollector) Collect(ch chan<- prometheus.Metric) { ) } - providerConfig, err := providerconfigtypes.GetConfig(machine.Spec.ProviderSpec) + providerConfig, err := providerconfig.GetConfig(machine.Spec.ProviderSpec) if err != nil { utilruntime.HandleError(fmt.Errorf("failed to determine providerSpec for machine %s: %w", machine.Name, err)) continue diff --git a/pkg/controller/machinedeployment/controller.go b/pkg/controller/machinedeployment/controller.go index 81753d7a4..5c623860e 100644 --- a/pkg/controller/machinedeployment/controller.go +++ b/pkg/controller/machinedeployment/controller.go @@ -25,8 +25,8 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/common" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -36,7 +36,7 @@ import ( "k8s.io/client-go/tools/record" ctrlruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/client" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -48,12 +48,12 @@ const controllerName = "machinedeployment-controller" var ( // controllerKind contains the schema.GroupVersionKind for this controller type. - controllerKind = v1alpha1.SchemeGroupVersion.WithKind("MachineDeployment") + controllerKind = clusterv1alpha1.SchemeGroupVersion.WithKind("MachineDeployment") ) // ReconcileMachineDeployment reconciles a MachineDeployment object. type ReconcileMachineDeployment struct { - client.Client + ctrlruntimeclient.Client log *zap.SugaredLogger scheme *runtime.Scheme recorder record.EventRecorder @@ -86,13 +86,13 @@ func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) err }, }). // Watch for changes to MachineDeployment. - For(&v1alpha1.MachineDeployment{}). + For(&clusterv1alpha1.MachineDeployment{}). // Watch for changes to MachineSet and reconcile the owner MachineDeployment. - Owns(&v1alpha1.MachineSet{}). + Owns(&clusterv1alpha1.MachineSet{}). // Watch for changes to MachineSets using a mapping function to MachineDeployment. // This watcher is required for use cases like adoption. In case a MachineSet doesn't have // a controller reference, it'll look for potential matching MachineDeployments to reconcile. - Watches(&v1alpha1.MachineSet{}, handler.EnqueueRequestsFromMapFunc(mapFn)). + Watches(&clusterv1alpha1.MachineSet{}, handler.EnqueueRequestsFromMapFunc(mapFn)). Build(r) return err @@ -107,7 +107,7 @@ func (r *ReconcileMachineDeployment) Reconcile(ctx context.Context, request reco log.Debug("Reconciling") // Fetch the MachineDeployment instance - deployment := &v1alpha1.MachineDeployment{} + deployment := &clusterv1alpha1.MachineDeployment{} if err := r.Get(ctx, request.NamespacedName, deployment); err != nil { if apierrors.IsNotFound(err) { // Object not found, return. Created objects are automatically garbage collected. @@ -134,8 +134,8 @@ func (r *ReconcileMachineDeployment) Reconcile(ctx context.Context, request reco return result, err } -func (r *ReconcileMachineDeployment) reconcile(ctx context.Context, log *zap.SugaredLogger, d *v1alpha1.MachineDeployment) (reconcile.Result, error) { - v1alpha1.PopulateDefaultsMachineDeployment(d) +func (r *ReconcileMachineDeployment) reconcile(ctx context.Context, log *zap.SugaredLogger, d *clusterv1alpha1.MachineDeployment) (reconcile.Result, error) { + clusterv1alpha1.PopulateDefaultsMachineDeployment(d) everything := metav1.LabelSelector{} if reflect.DeepEqual(d.Spec.Selector, &everything) { @@ -191,18 +191,18 @@ func (r *ReconcileMachineDeployment) reconcile(ctx context.Context, log *zap.Sug } // getMachineSetsForDeployment returns a list of MachineSets associated with a MachineDeployment. -func (r *ReconcileMachineDeployment) getMachineSetsForDeployment(ctx context.Context, log *zap.SugaredLogger, d *v1alpha1.MachineDeployment) ([]*v1alpha1.MachineSet, error) { +func (r *ReconcileMachineDeployment) getMachineSetsForDeployment(ctx context.Context, log *zap.SugaredLogger, d *clusterv1alpha1.MachineDeployment) ([]*clusterv1alpha1.MachineSet, error) { // List all MachineSets to find those we own but that no longer match our selector. - machineSets := &v1alpha1.MachineSetList{} - listOptions := &client.ListOptions{Namespace: d.Namespace} + machineSets := &clusterv1alpha1.MachineSetList{} + listOptions := &ctrlruntimeclient.ListOptions{Namespace: d.Namespace} if err := r.Client.List(ctx, machineSets, listOptions); err != nil { return nil, err } - filtered := make([]*v1alpha1.MachineSet, 0, len(machineSets.Items)) + filtered := make([]*clusterv1alpha1.MachineSet, 0, len(machineSets.Items)) for idx := range machineSets.Items { ms := &machineSets.Items[idx] - msLog := log.With("machineset", client.ObjectKeyFromObject(ms)) + msLog := log.With("machineset", ctrlruntimeclient.ObjectKeyFromObject(ms)) selector, err := metav1.LabelSelectorAsSelector(&d.Spec.Selector) if err != nil { @@ -240,27 +240,27 @@ func (r *ReconcileMachineDeployment) getMachineSetsForDeployment(ctx context.Con } // adoptOrphan sets the MachineDeployment as a controller OwnerReference to the MachineSet. -func (r *ReconcileMachineDeployment) adoptOrphan(ctx context.Context, deployment *v1alpha1.MachineDeployment, machineSet *v1alpha1.MachineSet) error { +func (r *ReconcileMachineDeployment) adoptOrphan(ctx context.Context, deployment *clusterv1alpha1.MachineDeployment, machineSet *clusterv1alpha1.MachineSet) error { newRef := *metav1.NewControllerRef(deployment, controllerKind) machineSet.OwnerReferences = append(machineSet.OwnerReferences, newRef) return r.Client.Update(ctx, machineSet) } // getMachineDeploymentsForMachineSet returns a list of MachineDeployments that could potentially match a MachineSet. -func (r *ReconcileMachineDeployment) getMachineDeploymentsForMachineSet(ctx context.Context, log *zap.SugaredLogger, ms *v1alpha1.MachineSet) []*v1alpha1.MachineDeployment { +func (r *ReconcileMachineDeployment) getMachineDeploymentsForMachineSet(ctx context.Context, log *zap.SugaredLogger, ms *clusterv1alpha1.MachineSet) []*clusterv1alpha1.MachineDeployment { if len(ms.Labels) == 0 { log.Info("No MachineDeployments found for MachineSet because it has no labels") return nil } - dList := &v1alpha1.MachineDeploymentList{} - listOptions := &client.ListOptions{Namespace: ms.Namespace} + dList := &clusterv1alpha1.MachineDeploymentList{} + listOptions := &ctrlruntimeclient.ListOptions{Namespace: ms.Namespace} if err := r.Client.List(ctx, dList, listOptions); err != nil { log.Errorw("Failed to list MachineDeployments", zap.Error(err)) return nil } - deployments := make([]*v1alpha1.MachineDeployment, 0, len(dList.Items)) + deployments := make([]*clusterv1alpha1.MachineDeployment, 0, len(dList.Items)) for idx, d := range dList.Items { selector, err := metav1.LabelSelectorAsSelector(&d.Spec.Selector) if err != nil { @@ -281,11 +281,11 @@ func (r *ReconcileMachineDeployment) getMachineDeploymentsForMachineSet(ctx cont // MachineSetTodeployments is a handler.MapFunc to be used to enqeue requests for reconciliation // for MachineDeployments that might adopt an orphaned MachineSet. func (r *ReconcileMachineDeployment) MachineSetToDeployments() handler.MapFunc { - return func(ctx context.Context, o client.Object) []ctrlruntime.Request { + return func(ctx context.Context, o ctrlruntimeclient.Object) []ctrlruntime.Request { result := []reconcile.Request{} - ms := &v1alpha1.MachineSet{} - key := client.ObjectKey{Namespace: o.GetNamespace(), Name: o.GetName()} + ms := &clusterv1alpha1.MachineSet{} + key := ctrlruntimeclient.ObjectKey{Namespace: o.GetNamespace(), Name: o.GetName()} if err := r.Client.Get(ctx, key, ms); err != nil { if !apierrors.IsNotFound(err) { r.log.Errorw("Failed to retrieve MachineSet for possible MachineDeployment adoption", "machineset", key, zap.Error(err)) @@ -308,7 +308,7 @@ func (r *ReconcileMachineDeployment) MachineSetToDeployments() handler.MapFunc { } for _, md := range mds { - name := client.ObjectKey{Namespace: md.Namespace, Name: md.Name} + name := ctrlruntimeclient.ObjectKey{Namespace: md.Namespace, Name: md.Name} result = append(result, reconcile.Request{NamespacedName: name}) } diff --git a/pkg/controller/machinedeployment/metrics.go b/pkg/controller/machinedeployment/metrics.go index 767ea4d89..2b6a98169 100644 --- a/pkg/controller/machinedeployment/metrics.go +++ b/pkg/controller/machinedeployment/metrics.go @@ -20,7 +20,9 @@ import ( "context" "github.com/prometheus/client_golang/prometheus" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -74,7 +76,7 @@ func (c *Collector) Describe(desc chan<- *prometheus.Desc) { // Collect implements the prometheus.Collector interface. func (c *Collector) Collect(metrics chan<- prometheus.Metric) { - machineDeployments := &v1alpha1.MachineDeploymentList{} + machineDeployments := &clusterv1alpha1.MachineDeploymentList{} if err := c.client.List(c.ctx, machineDeployments); err != nil { return } diff --git a/pkg/controller/machinedeployment/rolling.go b/pkg/controller/machinedeployment/rolling.go index 000fb1ad6..cc3703adf 100644 --- a/pkg/controller/machinedeployment/rolling.go +++ b/pkg/controller/machinedeployment/rolling.go @@ -23,15 +23,15 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" - dutil "k8c.io/machine-controller/pkg/controller/util" + "k8c.io/machine-controller/pkg/controller/util" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" "k8s.io/utils/integer" - "sigs.k8s.io/controller-runtime/pkg/client" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) // rolloutRolling implements the logic for rolling a new machine set. -func (r *ReconcileMachineDeployment) rolloutRolling(ctx context.Context, log *zap.SugaredLogger, d *v1alpha1.MachineDeployment, msList []*v1alpha1.MachineSet) error { +func (r *ReconcileMachineDeployment) rolloutRolling(ctx context.Context, log *zap.SugaredLogger, d *clusterv1alpha1.MachineDeployment, msList []*clusterv1alpha1.MachineSet) error { newMS, oldMSs, err := r.getAllMachineSetsAndSyncRevision(ctx, log, d, msList, true) if err != nil { return err @@ -56,7 +56,7 @@ func (r *ReconcileMachineDeployment) rolloutRolling(ctx context.Context, log *za } // Scale down, if we can. - if err := r.reconcileOldMachineSets(ctx, log.With("newmachineset", client.ObjectKeyFromObject(newMS)), allMSs, oldMSs, newMS, d); err != nil { + if err := r.reconcileOldMachineSets(ctx, log.With("newmachineset", ctrlruntimeclient.ObjectKeyFromObject(newMS)), allMSs, oldMSs, newMS, d); err != nil { return err } @@ -64,7 +64,7 @@ func (r *ReconcileMachineDeployment) rolloutRolling(ctx context.Context, log *za return err } - if dutil.DeploymentComplete(d, &d.Status) { + if util.DeploymentComplete(d, &d.Status) { if err := r.cleanupDeployment(ctx, log, oldMSs, d); err != nil { return err } @@ -73,7 +73,7 @@ func (r *ReconcileMachineDeployment) rolloutRolling(ctx context.Context, log *za return nil } -func (r *ReconcileMachineDeployment) reconcileNewMachineSet(ctx context.Context, allMSs []*v1alpha1.MachineSet, newMS *v1alpha1.MachineSet, deployment *v1alpha1.MachineDeployment) error { +func (r *ReconcileMachineDeployment) reconcileNewMachineSet(ctx context.Context, allMSs []*clusterv1alpha1.MachineSet, newMS *clusterv1alpha1.MachineSet, deployment *clusterv1alpha1.MachineDeployment) error { if deployment.Spec.Replicas == nil { return errors.Errorf("spec replicas for deployment set %v is nil, this is unexpected", deployment.Name) } @@ -93,7 +93,7 @@ func (r *ReconcileMachineDeployment) reconcileNewMachineSet(ctx context.Context, return err } - newReplicasCount, err := dutil.NewMSNewReplicas(deployment, allMSs, newMS) + newReplicasCount, err := util.NewMSNewReplicas(deployment, allMSs, newMS) if err != nil { return err } @@ -101,7 +101,7 @@ func (r *ReconcileMachineDeployment) reconcileNewMachineSet(ctx context.Context, return err } -func (r *ReconcileMachineDeployment) reconcileOldMachineSets(ctx context.Context, log *zap.SugaredLogger, allMSs []*v1alpha1.MachineSet, oldMSs []*v1alpha1.MachineSet, newMS *v1alpha1.MachineSet, deployment *v1alpha1.MachineDeployment) error { +func (r *ReconcileMachineDeployment) reconcileOldMachineSets(ctx context.Context, log *zap.SugaredLogger, allMSs []*clusterv1alpha1.MachineSet, oldMSs []*clusterv1alpha1.MachineSet, newMS *clusterv1alpha1.MachineSet, deployment *clusterv1alpha1.MachineDeployment) error { if deployment.Spec.Replicas == nil { return errors.Errorf("spec replicas for deployment set %v is nil, this is unexpected", deployment.Name) } @@ -110,15 +110,15 @@ func (r *ReconcileMachineDeployment) reconcileOldMachineSets(ctx context.Context return errors.Errorf("spec replicas for machine set %v is nil, this is unexpected", newMS.Name) } - oldMachinesCount := dutil.GetReplicaCountForMachineSets(oldMSs) + oldMachinesCount := util.GetReplicaCountForMachineSets(oldMSs) if oldMachinesCount == 0 { // Can't scale down further return nil } - allMachinesCount := dutil.GetReplicaCountForMachineSets(allMSs) + allMachinesCount := util.GetReplicaCountForMachineSets(allMSs) log.Debugw("New machine set status", "replicas", newMS.Status.AvailableReplicas) - maxUnavailable := dutil.MaxUnavailable(*deployment) + maxUnavailable := util.MaxUnavailable(*deployment) // Check if we can scale down. We can scale down in the following 2 cases: // * Some old machine sets have unhealthy replicas, we could safely scale down those unhealthy replicas since that won't further @@ -178,8 +178,8 @@ func (r *ReconcileMachineDeployment) reconcileOldMachineSets(ctx context.Context } // cleanupUnhealthyReplicas will scale down old machine sets with unhealthy replicas, so that all unhealthy replicas will be deleted. -func (r *ReconcileMachineDeployment) cleanupUnhealthyReplicas(ctx context.Context, log *zap.SugaredLogger, oldMSs []*v1alpha1.MachineSet, deployment *v1alpha1.MachineDeployment, maxCleanupCount int32) ([]*v1alpha1.MachineSet, int32, error) { - sort.Sort(dutil.MachineSetsByCreationTimestamp(oldMSs)) +func (r *ReconcileMachineDeployment) cleanupUnhealthyReplicas(ctx context.Context, log *zap.SugaredLogger, oldMSs []*clusterv1alpha1.MachineSet, deployment *clusterv1alpha1.MachineDeployment, maxCleanupCount int32) ([]*clusterv1alpha1.MachineSet, int32, error) { + sort.Sort(util.MachineSetsByCreationTimestamp(oldMSs)) // Safely scale down all old machine sets with unhealthy replicas. Replica set will sort the machines in the order // such that not-ready < ready, unscheduled < scheduled, and pending < running. This ensures that unhealthy replicas will @@ -202,7 +202,7 @@ func (r *ReconcileMachineDeployment) cleanupUnhealthyReplicas(ctx context.Contex } oldMSAvailableReplicas := targetMS.Status.AvailableReplicas - log.Debugw("Available machines in old MachineSet", "oldmachineset", client.ObjectKeyFromObject(targetMS), "replicas", oldMSAvailableReplicas) + log.Debugw("Available machines in old MachineSet", "oldmachineset", ctrlruntimeclient.ObjectKeyFromObject(targetMS), "replicas", oldMSAvailableReplicas) if oldMSReplicas == oldMSAvailableReplicas { // no unhealthy replicas found, no scaling required. continue @@ -229,18 +229,18 @@ func (r *ReconcileMachineDeployment) cleanupUnhealthyReplicas(ctx context.Contex // scaleDownOldMachineSetsForRollingUpdate scales down old machine sets when deployment strategy is "RollingUpdate". // Need check maxUnavailable to ensure availability. -func (r *ReconcileMachineDeployment) scaleDownOldMachineSetsForRollingUpdate(ctx context.Context, log *zap.SugaredLogger, allMSs []*v1alpha1.MachineSet, oldMSs []*v1alpha1.MachineSet, deployment *v1alpha1.MachineDeployment) (int32, error) { +func (r *ReconcileMachineDeployment) scaleDownOldMachineSetsForRollingUpdate(ctx context.Context, log *zap.SugaredLogger, allMSs []*clusterv1alpha1.MachineSet, oldMSs []*clusterv1alpha1.MachineSet, deployment *clusterv1alpha1.MachineDeployment) (int32, error) { if deployment.Spec.Replicas == nil { return 0, errors.Errorf("spec replicas for deployment %v is nil, this is unexpected", deployment.Name) } - maxUnavailable := dutil.MaxUnavailable(*deployment) + maxUnavailable := util.MaxUnavailable(*deployment) // Check if we can scale down. minAvailable := *(deployment.Spec.Replicas) - maxUnavailable // Find the number of available machines. - availableMachineCount := dutil.GetAvailableReplicaCountForMachineSets(allMSs) + availableMachineCount := util.GetAvailableReplicaCountForMachineSets(allMSs) if availableMachineCount <= minAvailable { // Cannot scale down. return 0, nil @@ -248,7 +248,7 @@ func (r *ReconcileMachineDeployment) scaleDownOldMachineSetsForRollingUpdate(ctx log.Debugw("Found available machines, scaling down old MachineSets", "replicas", availableMachineCount) - sort.Sort(dutil.MachineSetsByCreationTimestamp(oldMSs)) + sort.Sort(util.MachineSetsByCreationTimestamp(oldMSs)) totalScaledDown := int32(0) totalScaleDownCount := availableMachineCount - minAvailable diff --git a/pkg/controller/machinedeployment/sync.go b/pkg/controller/machinedeployment/sync.go index 4c10bac48..8dbf04765 100644 --- a/pkg/controller/machinedeployment/sync.go +++ b/pkg/controller/machinedeployment/sync.go @@ -26,8 +26,8 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" dutil "k8c.io/machine-controller/pkg/controller/util" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -36,7 +36,7 @@ import ( apirand "k8s.io/apimachinery/pkg/util/rand" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/retry" - "sigs.k8s.io/controller-runtime/pkg/client" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) // sync is responsible for reconciling deployments on scaling events or when they @@ -184,7 +184,7 @@ func (r *ReconcileMachineDeployment) getNewMachineSet(ctx context.Context, log * alreadyExists = true ms := &clusterv1alpha1.MachineSet{} - msErr := r.Get(ctx, client.ObjectKey{Namespace: newMS.Namespace, Name: newMS.Name}, ms) + msErr := r.Get(ctx, ctrlruntimeclient.ObjectKey{Namespace: newMS.Namespace, Name: newMS.Name}, ms) if msErr != nil { return nil, msErr } @@ -201,12 +201,12 @@ func (r *ReconcileMachineDeployment) getNewMachineSet(ctx context.Context, log * return nil, err case err != nil: - log.Errorw("Failed to create new MachineSet", "machineset", client.ObjectKeyFromObject(&newMS), zap.Error(err)) + log.Errorw("Failed to create new MachineSet", "machineset", ctrlruntimeclient.ObjectKeyFromObject(&newMS), zap.Error(err)) return nil, err } if !alreadyExists { - log.Debugw("Created new MachineSet", "machineset", client.ObjectKeyFromObject(createdMS)) + log.Debugw("Created new MachineSet", "machineset", ctrlruntimeclient.ObjectKeyFromObject(createdMS)) } err = r.updateMachineDeployment(ctx, d, func(md *clusterv1alpha1.MachineDeployment) { @@ -277,7 +277,7 @@ func (r *ReconcileMachineDeployment) scale(ctx context.Context, log *zap.Sugared for i := range allMSs { ms := allMSs[i] if ms.Spec.Replicas == nil { - log.Errorw("spec.replicas for MachineSet is nil, this is unexpected.", "machineset", client.ObjectKeyFromObject(ms)) + log.Errorw("spec.replicas for MachineSet is nil, this is unexpected.", "machineset", ctrlruntimeclient.ObjectKeyFromObject(ms)) continue } @@ -429,7 +429,7 @@ func (r *ReconcileMachineDeployment) cleanupDeployment(ctx context.Context, log continue } - log.Debugw("Trying to cleanup MachineSet for MachineDeployment", "machineset", client.ObjectKeyFromObject(ms)) + log.Debugw("Trying to cleanup MachineSet for MachineDeployment", "machineset", ctrlruntimeclient.ObjectKeyFromObject(ms)) if err := r.Delete(ctx, ms); err != nil && !apierrors.IsNotFound(err) { // Return error instead of aggregating and continuing DELETEs on the theory // that we may be overloading the api server. @@ -445,7 +445,7 @@ func (r *ReconcileMachineDeployment) updateMachineDeployment(ctx context.Context } // We have this as standalone variant to be able to use it from the tests. -func updateMachineDeployment(ctx context.Context, c client.Client, d *clusterv1alpha1.MachineDeployment, modify func(*clusterv1alpha1.MachineDeployment)) error { +func updateMachineDeployment(ctx context.Context, c ctrlruntimeclient.Client, d *clusterv1alpha1.MachineDeployment, modify func(*clusterv1alpha1.MachineDeployment)) error { dCopy := d.DeepCopy() modify(dCopy) if equality.Semantic.DeepEqual(dCopy, d) { diff --git a/pkg/controller/machineset/controller.go b/pkg/controller/machineset/controller.go index 000d37179..cf1a18ca7 100644 --- a/pkg/controller/machineset/controller.go +++ b/pkg/controller/machineset/controller.go @@ -28,7 +28,7 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -39,7 +39,7 @@ import ( "k8s.io/client-go/tools/record" ctrlruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/client" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -103,7 +103,7 @@ func add(mgr manager.Manager, r reconcile.Reconciler, mapFn handler.MapFunc) err // ReconcileMachineSet reconciles a MachineSet object. type ReconcileMachineSet struct { - client.Client + ctrlruntimeclient.Client log *zap.SugaredLogger scheme *runtime.Scheme recorder record.EventRecorder @@ -149,7 +149,7 @@ func (r *ReconcileMachineSet) reconcile(ctx context.Context, log *zap.SugaredLog log.Debug("Reconcile MachineSet") allMachines := &clusterv1alpha1.MachineList{} - if err := r.Client.List(ctx, allMachines, client.InNamespace(machineSet.Namespace)); err != nil { + if err := r.Client.List(ctx, allMachines, ctrlruntimeclient.InNamespace(machineSet.Namespace)); err != nil { return reconcile.Result{}, errors.Wrap(err, "failed to list machines") } @@ -185,7 +185,7 @@ func (r *ReconcileMachineSet) reconcile(ctx context.Context, log *zap.SugaredLog filteredMachines := make([]*clusterv1alpha1.Machine, 0, len(allMachines.Items)) for idx := range allMachines.Items { machine := &allMachines.Items[idx] - machineLog := log.With("machine", client.ObjectKeyFromObject(machine)) + machineLog := log.With("machine", ctrlruntimeclient.ObjectKeyFromObject(machine)) if shouldExcludeMachine(machineLog, machineSet, machine) { continue @@ -260,7 +260,7 @@ func (r *ReconcileMachineSet) syncReplicas(ctx context.Context, log *zap.Sugared machine := r.createMachine(ms) if err := r.Client.Create(ctx, machine); err != nil { - log.Errorw("Failed to create Machine", "machine", client.ObjectKeyFromObject(machine), zap.Error(err)) + log.Errorw("Failed to create Machine", "machine", ctrlruntimeclient.ObjectKeyFromObject(machine), zap.Error(err)) errstrings = append(errstrings, err.Error()) continue } @@ -293,7 +293,7 @@ func (r *ReconcileMachineSet) syncReplicas(ctx context.Context, log *zap.Sugared defer wg.Done() err := r.Client.Delete(ctx, targetMachine) if err != nil { - log.Errorw("Failed to delete Machine", "machine", client.ObjectKeyFromObject(targetMachine), zap.Error(err)) + log.Errorw("Failed to delete Machine", "machine", ctrlruntimeclient.ObjectKeyFromObject(targetMachine), zap.Error(err)) errCh <- err } }(machine) @@ -362,7 +362,7 @@ func (r *ReconcileMachineSet) adoptOrphan(ctx context.Context, machineSet *clust func (r *ReconcileMachineSet) waitForMachineCreation(ctx context.Context, log *zap.SugaredLogger, machineList []*clusterv1alpha1.Machine) error { for _, machine := range machineList { pollErr := wait.PollUntilContextTimeout(ctx, stateConfirmationInterval, stateConfirmationTimeout, false, func(ctx context.Context) (bool, error) { - key := client.ObjectKey{Namespace: machine.Namespace, Name: machine.Name} + key := ctrlruntimeclient.ObjectKey{Namespace: machine.Namespace, Name: machine.Name} if err := r.Client.Get(ctx, key, &clusterv1alpha1.Machine{}); err != nil { if apierrors.IsNotFound(err) { @@ -387,7 +387,7 @@ func (r *ReconcileMachineSet) waitForMachineDeletion(ctx context.Context, machin for _, machine := range machineList { pollErr := wait.PollUntilContextTimeout(ctx, stateConfirmationInterval, stateConfirmationTimeout, false, func(ctx context.Context) (bool, error) { m := &clusterv1alpha1.Machine{} - key := client.ObjectKey{Namespace: machine.Namespace, Name: machine.Name} + key := ctrlruntimeclient.ObjectKey{Namespace: machine.Namespace, Name: machine.Name} err := r.Client.Get(ctx, key, m) if apierrors.IsNotFound(err) || !m.DeletionTimestamp.IsZero() { @@ -407,11 +407,11 @@ func (r *ReconcileMachineSet) waitForMachineDeletion(ctx context.Context, machin // MachineToMachineSets is a handler.ToRequestsFunc to be used to enqeue requests for reconciliation // for MachineSets that might adopt an orphaned Machine. func (r *ReconcileMachineSet) MachineToMachineSets() handler.MapFunc { - return func(ctx context.Context, o client.Object) []ctrlruntime.Request { + return func(ctx context.Context, o ctrlruntimeclient.Object) []ctrlruntime.Request { result := []reconcile.Request{} m := &clusterv1alpha1.Machine{} - key := client.ObjectKey{Namespace: o.GetNamespace(), Name: o.GetName()} + key := ctrlruntimeclient.ObjectKey{Namespace: o.GetNamespace(), Name: o.GetName()} machineLog := r.log.With("machine", key) if err := r.Client.Get(ctx, key, m); err != nil { @@ -436,7 +436,7 @@ func (r *ReconcileMachineSet) MachineToMachineSets() handler.MapFunc { } for _, ms := range mss { - name := client.ObjectKey{Namespace: ms.Namespace, Name: ms.Name} + name := ctrlruntimeclient.ObjectKey{Namespace: ms.Namespace, Name: ms.Name} result = append(result, reconcile.Request{NamespacedName: name}) } diff --git a/pkg/controller/machineset/delete_policy.go b/pkg/controller/machineset/delete_policy.go index c631a70e5..73f87f990 100644 --- a/pkg/controller/machineset/delete_policy.go +++ b/pkg/controller/machineset/delete_policy.go @@ -22,14 +22,14 @@ import ( "github.com/pkg/errors" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type ( deletePriority float64 - deletePriorityFunc func(machine *v1alpha1.Machine) deletePriority + deletePriorityFunc func(machine *clusterv1alpha1.Machine) deletePriority ) const ( @@ -47,7 +47,7 @@ const ( ) // maps the creation timestamp onto the 0-100 priority range. -func oldestDeletePriority(machine *v1alpha1.Machine) deletePriority { +func oldestDeletePriority(machine *clusterv1alpha1.Machine) deletePriority { if machine.DeletionTimestamp != nil && !machine.DeletionTimestamp.IsZero() { return mustDelete } @@ -67,7 +67,7 @@ func oldestDeletePriority(machine *v1alpha1.Machine) deletePriority { return deletePriority(float64(mustDelete) * (1.0 - math.Exp(-d.Seconds()/secondsPerTenDays))) } -func newestDeletePriority(machine *v1alpha1.Machine) deletePriority { +func newestDeletePriority(machine *clusterv1alpha1.Machine) deletePriority { if machine.DeletionTimestamp != nil && !machine.DeletionTimestamp.IsZero() { return mustDelete } @@ -80,7 +80,7 @@ func newestDeletePriority(machine *v1alpha1.Machine) deletePriority { return mustDelete - oldestDeletePriority(machine) } -func randomDeletePolicy(machine *v1alpha1.Machine) deletePriority { +func randomDeletePolicy(machine *clusterv1alpha1.Machine) deletePriority { if machine.DeletionTimestamp != nil && !machine.DeletionTimestamp.IsZero() { return mustDelete } @@ -94,7 +94,7 @@ func randomDeletePolicy(machine *v1alpha1.Machine) deletePriority { } type sortableMachines struct { - machines []*v1alpha1.Machine + machines []*clusterv1alpha1.Machine priority deletePriorityFunc } @@ -106,11 +106,11 @@ func (m sortableMachines) Less(i, j int) bool { return m.priority(m.machines[j]) < m.priority(m.machines[i]) // high to low } -func getMachinesToDeletePrioritized(filteredMachines []*v1alpha1.Machine, diff int, fun deletePriorityFunc) []*v1alpha1.Machine { +func getMachinesToDeletePrioritized(filteredMachines []*clusterv1alpha1.Machine, diff int, fun deletePriorityFunc) []*clusterv1alpha1.Machine { if diff >= len(filteredMachines) { return filteredMachines } else if diff <= 0 { - return []*v1alpha1.Machine{} + return []*clusterv1alpha1.Machine{} } sortable := sortableMachines{ @@ -122,14 +122,14 @@ func getMachinesToDeletePrioritized(filteredMachines []*v1alpha1.Machine, diff i return sortable.machines[:diff] } -func getDeletePriorityFunc(ms *v1alpha1.MachineSet) (deletePriorityFunc, error) { +func getDeletePriorityFunc(ms *clusterv1alpha1.MachineSet) (deletePriorityFunc, error) { // Map the Spec.DeletePolicy value to the appropriate delete priority function - switch msdp := v1alpha1.MachineSetDeletePolicy(ms.Spec.DeletePolicy); msdp { - case v1alpha1.RandomMachineSetDeletePolicy: + switch msdp := clusterv1alpha1.MachineSetDeletePolicy(ms.Spec.DeletePolicy); msdp { + case clusterv1alpha1.RandomMachineSetDeletePolicy: return randomDeletePolicy, nil - case v1alpha1.NewestMachineSetDeletePolicy: + case clusterv1alpha1.NewestMachineSetDeletePolicy: return newestDeletePriority, nil - case v1alpha1.OldestMachineSetDeletePolicy: + case clusterv1alpha1.OldestMachineSetDeletePolicy: return oldestDeletePriority, nil case "": return randomDeletePolicy, nil diff --git a/pkg/controller/machineset/machine.go b/pkg/controller/machineset/machine.go index 7c24891f0..501b83652 100644 --- a/pkg/controller/machineset/machine.go +++ b/pkg/controller/machineset/machine.go @@ -21,21 +21,21 @@ import ( "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) -func (c *ReconcileMachineSet) getMachineSetsForMachine(ctx context.Context, machineLog *zap.SugaredLogger, m *v1alpha1.Machine) []*v1alpha1.MachineSet { +func (c *ReconcileMachineSet) getMachineSetsForMachine(ctx context.Context, machineLog *zap.SugaredLogger, m *clusterv1alpha1.Machine) []*clusterv1alpha1.MachineSet { if len(m.Labels) == 0 { machineLog.Infow("No MachineSets found for Machine because it has no labels") return nil } - msList := &v1alpha1.MachineSetList{} - listOptions := &client.ListOptions{ + msList := &clusterv1alpha1.MachineSetList{} + listOptions := &ctrlruntimeclient.ListOptions{ Namespace: m.Namespace, } @@ -45,7 +45,7 @@ func (c *ReconcileMachineSet) getMachineSetsForMachine(ctx context.Context, mach return nil } - var mss []*v1alpha1.MachineSet + var mss []*clusterv1alpha1.MachineSet for idx := range msList.Items { ms := &msList.Items[idx] if hasMatchingLabels(machineLog, ms, m) { @@ -56,7 +56,7 @@ func (c *ReconcileMachineSet) getMachineSetsForMachine(ctx context.Context, mach return mss } -func hasMatchingLabels(machineLog *zap.SugaredLogger, machineSet *v1alpha1.MachineSet, machine *v1alpha1.Machine) bool { +func hasMatchingLabels(machineLog *zap.SugaredLogger, machineSet *clusterv1alpha1.MachineSet, machine *clusterv1alpha1.Machine) bool { selector, err := metav1.LabelSelectorAsSelector(&machineSet.Spec.Selector) if err != nil { machineLog.Errorw("Failed to convert selector", zap.Error(err)) diff --git a/pkg/controller/machineset/status.go b/pkg/controller/machineset/status.go index 0d724816b..d4162e817 100644 --- a/pkg/controller/machineset/status.go +++ b/pkg/controller/machineset/status.go @@ -23,12 +23,12 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/controller-runtime/pkg/client" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -36,7 +36,7 @@ const ( statusUpdateRetries = 1 ) -func (c *ReconcileMachineSet) calculateStatus(ctx context.Context, log *zap.SugaredLogger, ms *v1alpha1.MachineSet, filteredMachines []*v1alpha1.Machine) v1alpha1.MachineSetStatus { +func (c *ReconcileMachineSet) calculateStatus(ctx context.Context, log *zap.SugaredLogger, ms *clusterv1alpha1.MachineSet, filteredMachines []*clusterv1alpha1.Machine) clusterv1alpha1.MachineSetStatus { newStatus := ms.Status // Count the number of machines that have labels matching the labels of the machine // template of the replica set, the matching machines may have more @@ -53,7 +53,7 @@ func (c *ReconcileMachineSet) calculateStatus(ctx context.Context, log *zap.Suga } node, err := c.getMachineNode(ctx, machine) if err != nil { - log.Debugw("Failed to get node for machine", "machine", client.ObjectKeyFromObject(machine), zap.Error(err)) + log.Debugw("Failed to get node for machine", "machine", ctrlruntimeclient.ObjectKeyFromObject(machine), zap.Error(err)) continue } if isNodeReady(node) { @@ -72,7 +72,7 @@ func (c *ReconcileMachineSet) calculateStatus(ctx context.Context, log *zap.Suga } // updateMachineSetStatus attempts to update the Status.Replicas of the given MachineSet, with a single GET/PUT retry. -func updateMachineSetStatus(ctx context.Context, log *zap.SugaredLogger, c client.Client, ms *v1alpha1.MachineSet, newStatus v1alpha1.MachineSetStatus) (*v1alpha1.MachineSet, error) { +func updateMachineSetStatus(ctx context.Context, log *zap.SugaredLogger, c ctrlruntimeclient.Client, ms *clusterv1alpha1.MachineSet, newStatus clusterv1alpha1.MachineSetStatus) (*clusterv1alpha1.MachineSet, error) { // This is the steady state. It happens when the MachineSet doesn't have any expectations, since // we do a periodic relist every 30s. If the generations differ but the replicas are // the same, a caller might've resized to the same replica count. @@ -121,7 +121,7 @@ func updateMachineSetStatus(ctx context.Context, log *zap.SugaredLogger, c clien break } // Update the MachineSet with the latest resource version for the next poll - if getErr = c.Get(ctx, client.ObjectKey{Namespace: ms.Namespace, Name: ms.Name}, ms); getErr != nil { + if getErr = c.Get(ctx, ctrlruntimeclient.ObjectKey{Namespace: ms.Namespace, Name: ms.Name}, ms); getErr != nil { // If the GET fails we can't trust status.Replicas anymore. This error // is bound to be more interesting than the update failure. return nil, getErr @@ -131,14 +131,14 @@ func updateMachineSetStatus(ctx context.Context, log *zap.SugaredLogger, c clien return nil, updateErr } -func (c *ReconcileMachineSet) getMachineNode(ctx context.Context, machine *v1alpha1.Machine) (*corev1.Node, error) { +func (c *ReconcileMachineSet) getMachineNode(ctx context.Context, machine *clusterv1alpha1.Machine) (*corev1.Node, error) { nodeRef := machine.Status.NodeRef if nodeRef == nil { return nil, errors.New("machine has no node ref") } node := &corev1.Node{} - err := c.Client.Get(ctx, client.ObjectKey{Name: nodeRef.Name}, node) + err := c.Client.Get(ctx, ctrlruntimeclient.ObjectKey{Name: nodeRef.Name}, node) return node, err } diff --git a/pkg/controller/nodecsrapprover/controller.go b/pkg/controller/nodecsrapprover/controller.go index b894ff46b..f8713bed9 100644 --- a/pkg/controller/nodecsrapprover/controller.go +++ b/pkg/controller/nodecsrapprover/controller.go @@ -27,16 +27,16 @@ import ( "github.com/go-logr/zapr" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" certificatesv1 "k8s.io/api/certificates/v1" corev1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" certificatesv1client "k8s.io/client-go/kubernetes/typed/certificates/v1" "sigs.k8s.io/controller-runtime/pkg/builder" - "sigs.k8s.io/controller-runtime/pkg/client" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -62,7 +62,7 @@ var ( ) type reconciler struct { - client.Client + ctrlruntimeclient.Client log *zap.SugaredLogger // Have to use the typed client because csr approval is a subresource // the dynamic client does not approve @@ -102,7 +102,7 @@ func (r *reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( // Get the CSR object csr := &certificatesv1.CertificateSigningRequest{} if err := r.Get(ctx, request.NamespacedName, csr); err != nil { - if kerrors.IsNotFound(err) { + if apierrors.IsNotFound(err) { return reconcile.Result{}, nil } log.Errorw("Failed to get CertificateSigningRequest", zap.Error(err)) @@ -209,7 +209,7 @@ func (r *reconciler) validateCSRObject(csr *certificatesv1.CertificateSigningReq // validateX509CSR validates the certificate request by comparing CN with username, // and organization with groups. -func (r *reconciler) validateX509CSR(csr *certificatesv1.CertificateSigningRequest, certReq *x509.CertificateRequest, machine v1alpha1.Machine) error { +func (r *reconciler) validateX509CSR(csr *certificatesv1.CertificateSigningRequest, certReq *x509.CertificateRequest, machine clusterv1alpha1.Machine) error { // Validate Subject CommonName. if certReq.Subject.CommonName != csr.Spec.Username { return fmt.Errorf("commonName '%s' is different then CSR username '%s'", certReq.Subject.CommonName, csr.Spec.Username) @@ -251,11 +251,11 @@ func (r *reconciler) validateX509CSR(csr *certificatesv1.CertificateSigningReque return nil } -func (r *reconciler) getMachineForNode(ctx context.Context, nodeName string) (v1alpha1.Machine, bool, error) { +func (r *reconciler) getMachineForNode(ctx context.Context, nodeName string) (clusterv1alpha1.Machine, bool, error) { // List all Machines in all namespaces. - machines := &v1alpha1.MachineList{} + machines := &clusterv1alpha1.MachineList{} if err := r.Client.List(ctx, machines); err != nil { - return v1alpha1.Machine{}, false, fmt.Errorf("failed to list all machine objects: %w", err) + return clusterv1alpha1.Machine{}, false, fmt.Errorf("failed to list all machine objects: %w", err) } for _, machine := range machines.Items { @@ -264,7 +264,7 @@ func (r *reconciler) getMachineForNode(ctx context.Context, nodeName string) (v1 } } - return v1alpha1.Machine{}, false, fmt.Errorf("failed to get machine for given node name '%s'", nodeName) + return clusterv1alpha1.Machine{}, false, fmt.Errorf("failed to get machine for given node name '%s'", nodeName) } func isUsageInUsageList(usage certificatesv1.KeyUsage, usageList []certificatesv1.KeyUsage) bool { diff --git a/pkg/controller/nodecsrapprover/controller_test.go b/pkg/controller/nodecsrapprover/controller_test.go index 970045250..82ca2c7c5 100644 --- a/pkg/controller/nodecsrapprover/controller_test.go +++ b/pkg/controller/nodecsrapprover/controller_test.go @@ -22,7 +22,7 @@ import ( "fmt" "testing" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" certificatesv1 "k8s.io/api/certificates/v1" corev1 "k8s.io/api/core/v1" @@ -331,13 +331,13 @@ func TestValidateCSRObject(t *testing.T) { } func TestValidateX509CSR(t *testing.T) { - machine := v1alpha1.Machine{ + machine := clusterv1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{ Name: "test-machine", Namespace: metav1.NamespaceSystem, }, - Spec: v1alpha1.MachineSpec{}, - Status: v1alpha1.MachineStatus{ + Spec: clusterv1alpha1.MachineSpec{}, + Status: clusterv1alpha1.MachineStatus{ NodeRef: &corev1.ObjectReference{ APIVersion: "v1", Kind: "Node", @@ -359,7 +359,7 @@ func TestValidateX509CSR(t *testing.T) { testCases := []struct { name string csr *certificatesv1.CertificateSigningRequest - machine v1alpha1.Machine + machine clusterv1alpha1.Machine err error }{ { @@ -407,13 +407,13 @@ func TestValidateX509CSR(t *testing.T) { }, }, }, - machine: v1alpha1.Machine{ + machine: clusterv1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{ Name: "test-machine", Namespace: metav1.NamespaceSystem, }, - Spec: v1alpha1.MachineSpec{}, - Status: v1alpha1.MachineStatus{ + Spec: clusterv1alpha1.MachineSpec{}, + Status: clusterv1alpha1.MachineStatus{ NodeRef: &corev1.ObjectReference{ APIVersion: "v1", Kind: "Node", diff --git a/pkg/controller/util/machine.go b/pkg/controller/util/machine.go index 848308f9a..cee33f100 100644 --- a/pkg/controller/util/machine.go +++ b/pkg/controller/util/machine.go @@ -20,13 +20,13 @@ import ( "context" "fmt" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) -func GetMachineDeploymentNameAndRevisionForMachine(ctx context.Context, machine *clusterv1alpha1.Machine, c client.Client) (string, string, error) { +func GetMachineDeploymentNameAndRevisionForMachine(ctx context.Context, machine *clusterv1alpha1.Machine, c ctrlruntimeclient.Client) (string, string, error) { var ( machineSetName string machineDeploymentName string diff --git a/pkg/controller/util/machine_deployment.go b/pkg/controller/util/machine_deployment.go index a6b06365a..f6b0ef24b 100644 --- a/pkg/controller/util/machine_deployment.go +++ b/pkg/controller/util/machine_deployment.go @@ -27,17 +27,17 @@ import ( "github.com/davecgh/go-spew/spew" "go.uber.org/zap" - "k8c.io/machine-controller/pkg/apis/cluster/common" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + "k8c.io/machine-controller/sdk/apis/cluster/common" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" intstrutil "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/integer" - "sigs.k8s.io/controller-runtime/pkg/client" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -72,7 +72,7 @@ const ( ) // MachineSetsByCreationTimestamp sorts a list of MachineSet by creation timestamp, using their names as a tie breaker. -type MachineSetsByCreationTimestamp []*v1alpha1.MachineSet +type MachineSetsByCreationTimestamp []*clusterv1alpha1.MachineSet func (o MachineSetsByCreationTimestamp) Len() int { return len(o) } @@ -87,7 +87,7 @@ func (o MachineSetsByCreationTimestamp) Less(i, j int) bool { // MachineSetsBySizeOlder sorts a list of MachineSet by size in descending order, using their creation timestamp or name as a tie breaker. // By using the creation timestamp, this sorts from old to new machine sets. -type MachineSetsBySizeOlder []*v1alpha1.MachineSet +type MachineSetsBySizeOlder []*clusterv1alpha1.MachineSet func (o MachineSetsBySizeOlder) Len() int { return len(o) } @@ -102,7 +102,7 @@ func (o MachineSetsBySizeOlder) Less(i, j int) bool { // MachineSetsBySizeNewer sorts a list of MachineSet by size in descending order, using their creation timestamp or name as a tie breaker. // By using the creation timestamp, this sorts from new to old machine sets. -type MachineSetsBySizeNewer []*v1alpha1.MachineSet +type MachineSetsBySizeNewer []*clusterv1alpha1.MachineSet func (o MachineSetsBySizeNewer) Len() int { return len(o) } @@ -116,7 +116,7 @@ func (o MachineSetsBySizeNewer) Less(i, j int) bool { } // SetDeploymentRevision updates the revision for a deployment. -func SetDeploymentRevision(deployment *v1alpha1.MachineDeployment, revision string) bool { +func SetDeploymentRevision(deployment *clusterv1alpha1.MachineDeployment, revision string) bool { updated := false if deployment.Annotations == nil { @@ -131,13 +131,13 @@ func SetDeploymentRevision(deployment *v1alpha1.MachineDeployment, revision stri } // MaxRevision finds the highest revision in the machine sets. -func MaxRevision(log *zap.SugaredLogger, allMSs []*v1alpha1.MachineSet) int64 { +func MaxRevision(log *zap.SugaredLogger, allMSs []*clusterv1alpha1.MachineSet) int64 { maxRev := int64(0) for _, ms := range allMSs { if v, err := Revision(ms); err != nil { log.Debugw( "Failed to parse revision for MachineSet, deployment controller will skip it when reconciling revisions", - "machinset", client.ObjectKeyFromObject(ms), + "machinset", ctrlruntimeclient.ObjectKeyFromObject(ms), zap.Error(err), ) } else if v > maxRev { @@ -161,11 +161,11 @@ func Revision(obj runtime.Object) (int64, error) { } var annotationsToSkip = map[string]bool{ - v1.LastAppliedConfigAnnotation: true, - RevisionAnnotation: true, - RevisionHistoryAnnotation: true, - DesiredReplicasAnnotation: true, - MaxReplicasAnnotation: true, + corev1.LastAppliedConfigAnnotation: true, + RevisionAnnotation: true, + RevisionHistoryAnnotation: true, + DesiredReplicasAnnotation: true, + MaxReplicasAnnotation: true, } // skipCopyAnnotation returns true if we should skip copying the annotation with the given annotation key @@ -179,7 +179,7 @@ func skipCopyAnnotation(key string) bool { // copyDeploymentAnnotationsToMachineSet copies deployment's annotations to machine set's annotations, // and returns true if machine set's annotation is changed. // Note that apply and revision annotations are not copied. -func copyDeploymentAnnotationsToMachineSet(deployment *v1alpha1.MachineDeployment, ms *v1alpha1.MachineSet) bool { +func copyDeploymentAnnotationsToMachineSet(deployment *clusterv1alpha1.MachineDeployment, ms *clusterv1alpha1.MachineSet) bool { msAnnotationsChanged := false if ms.Annotations == nil { ms.Annotations = make(map[string]string) @@ -198,15 +198,15 @@ func copyDeploymentAnnotationsToMachineSet(deployment *v1alpha1.MachineDeploymen } // GetDesiredReplicasAnnotation returns the number of desired replicas. -func GetDesiredReplicasAnnotation(log *zap.SugaredLogger, ms *v1alpha1.MachineSet) (int32, bool) { +func GetDesiredReplicasAnnotation(log *zap.SugaredLogger, ms *clusterv1alpha1.MachineSet) (int32, bool) { return getIntFromAnnotation(log, ms, DesiredReplicasAnnotation) } -func getMaxReplicasAnnotation(log *zap.SugaredLogger, ms *v1alpha1.MachineSet) (int32, bool) { +func getMaxReplicasAnnotation(log *zap.SugaredLogger, ms *clusterv1alpha1.MachineSet) (int32, bool) { return getIntFromAnnotation(log, ms, MaxReplicasAnnotation) } -func getIntFromAnnotation(log *zap.SugaredLogger, ms *v1alpha1.MachineSet, annotationKey string) (int32, bool) { +func getIntFromAnnotation(log *zap.SugaredLogger, ms *clusterv1alpha1.MachineSet, annotationKey string) (int32, bool) { annotationValue, ok := ms.Annotations[annotationKey] if !ok { return int32(0), false @@ -221,7 +221,7 @@ func getIntFromAnnotation(log *zap.SugaredLogger, ms *v1alpha1.MachineSet, annot // SetNewMachineSetAnnotations sets new machine set's annotations appropriately by updating its revision and // copying required deployment annotations to it; it returns true if machine set's annotation is changed. -func SetNewMachineSetAnnotations(mdLog *zap.SugaredLogger, deployment *v1alpha1.MachineDeployment, newMS *v1alpha1.MachineSet, newRevision string, exists bool) bool { +func SetNewMachineSetAnnotations(mdLog *zap.SugaredLogger, deployment *clusterv1alpha1.MachineDeployment, newMS *clusterv1alpha1.MachineSet, newRevision string, exists bool) bool { // First, copy deployment's annotations (except for apply and revision annotations) annotationChanged := copyDeploymentAnnotationsToMachineSet(deployment, newMS) // Then, update machine set's revision annotation @@ -233,7 +233,7 @@ func SetNewMachineSetAnnotations(mdLog *zap.SugaredLogger, deployment *v1alpha1. // of all old MSes + 1). However, it's possible that some of the old MSes are deleted after the newMS revision being updated, and // newRevision becomes smaller than newMS's revision. We should only update newMS revision when it's smaller than newRevision. - msLog := mdLog.With("machineset", client.ObjectKeyFromObject(newMS)) + msLog := mdLog.With("machineset", ctrlruntimeclient.ObjectKeyFromObject(newMS)) oldRevisionInt, err := strconv.ParseInt(oldRevision, 10, 64) if err != nil { @@ -281,7 +281,7 @@ func SetNewMachineSetAnnotations(mdLog *zap.SugaredLogger, deployment *v1alpha1. // FindOneActiveOrLatest returns the only active or the latest machine set in case there is at most one active // machine set. If there are more than one active machine sets, return nil so machine sets can be scaled down // to the point where there is only one active machine set. -func FindOneActiveOrLatest(newMS *v1alpha1.MachineSet, oldMSs []*v1alpha1.MachineSet) *v1alpha1.MachineSet { +func FindOneActiveOrLatest(newMS *clusterv1alpha1.MachineSet, oldMSs []*clusterv1alpha1.MachineSet) *clusterv1alpha1.MachineSet { if newMS == nil && len(oldMSs) == 0 { return nil } @@ -304,7 +304,7 @@ func FindOneActiveOrLatest(newMS *v1alpha1.MachineSet, oldMSs []*v1alpha1.Machin } // SetReplicasAnnotations sets the desiredReplicas and maxReplicas into the annotations. -func SetReplicasAnnotations(ms *v1alpha1.MachineSet, desiredReplicas, maxReplicas int32) bool { +func SetReplicasAnnotations(ms *clusterv1alpha1.MachineSet, desiredReplicas, maxReplicas int32) bool { updated := false if ms.Annotations == nil { ms.Annotations = make(map[string]string) @@ -323,7 +323,7 @@ func SetReplicasAnnotations(ms *v1alpha1.MachineSet, desiredReplicas, maxReplica } // AnnotationsNeedUpdate return true if ReplicasAnnotations need to be updated. -func ReplicasAnnotationsNeedUpdate(ms *v1alpha1.MachineSet, desiredReplicas, maxReplicas int32) bool { +func ReplicasAnnotationsNeedUpdate(ms *clusterv1alpha1.MachineSet, desiredReplicas, maxReplicas int32) bool { if ms.Annotations == nil { return true } @@ -339,7 +339,7 @@ func ReplicasAnnotationsNeedUpdate(ms *v1alpha1.MachineSet, desiredReplicas, max } // MaxUnavailable returns the maximum unavailable machines a rolling deployment can take. -func MaxUnavailable(deployment v1alpha1.MachineDeployment) int32 { +func MaxUnavailable(deployment clusterv1alpha1.MachineDeployment) int32 { if !IsRollingUpdate(&deployment) || *(deployment.Spec.Replicas) == 0 { return int32(0) } @@ -352,7 +352,7 @@ func MaxUnavailable(deployment v1alpha1.MachineDeployment) int32 { } // MaxSurge returns the maximum surge machines a rolling deployment can take. -func MaxSurge(deployment v1alpha1.MachineDeployment) int32 { +func MaxSurge(deployment clusterv1alpha1.MachineDeployment) int32 { if !IsRollingUpdate(&deployment) { return int32(0) } @@ -364,7 +364,7 @@ func MaxSurge(deployment v1alpha1.MachineDeployment) int32 { // GetProportion will estimate the proportion for the provided machine set using 1. the current size // of the parent deployment, 2. the replica count that needs be added on the machine sets of the // deployment, and 3. the total replicas added in the machine sets of the deployment so far. -func GetProportion(log *zap.SugaredLogger, ms *v1alpha1.MachineSet, d v1alpha1.MachineDeployment, deploymentReplicasToAdd, deploymentReplicasAdded int32) int32 { +func GetProportion(log *zap.SugaredLogger, ms *clusterv1alpha1.MachineSet, d clusterv1alpha1.MachineDeployment, deploymentReplicasToAdd, deploymentReplicasAdded int32) int32 { if ms == nil || *(ms.Spec.Replicas) == 0 || deploymentReplicasToAdd == 0 || deploymentReplicasToAdd == deploymentReplicasAdded { return int32(0) } @@ -386,7 +386,7 @@ func GetProportion(log *zap.SugaredLogger, ms *v1alpha1.MachineSet, d v1alpha1.M // getMachineSetFraction estimates the fraction of replicas a machine set can have in // 1. a scaling event during a rollout or 2. when scaling a paused deployment. -func getMachineSetFraction(log *zap.SugaredLogger, ms v1alpha1.MachineSet, d v1alpha1.MachineDeployment) int32 { +func getMachineSetFraction(log *zap.SugaredLogger, ms clusterv1alpha1.MachineSet, d clusterv1alpha1.MachineDeployment) int32 { // If we are scaling down to zero then the fraction of this machine set is its whole size (negative) if *(d.Spec.Replicas) == int32(0) { return -*(ms.Spec.Replicas) @@ -413,7 +413,7 @@ func getMachineSetFraction(log *zap.SugaredLogger, ms v1alpha1.MachineSet, d v1a // 1. The hash result would be different upon machineTemplateSpec API changes // (e.g. the addition of a new field will cause the hash code to change) // 2. The deployment template won't have hash labels. -func EqualIgnoreHash(template1, template2 *v1alpha1.MachineTemplateSpec) bool { +func EqualIgnoreHash(template1, template2 *clusterv1alpha1.MachineTemplateSpec) bool { t1Copy := template1.DeepCopy() t2Copy := template2.DeepCopy() // Remove hash labels from template.Labels before comparing. @@ -423,7 +423,7 @@ func EqualIgnoreHash(template1, template2 *v1alpha1.MachineTemplateSpec) bool { } // FindNewMachineSet returns the new MS this given deployment targets (the one with the same machine template). -func FindNewMachineSet(deployment *v1alpha1.MachineDeployment, msList []*v1alpha1.MachineSet) *v1alpha1.MachineSet { +func FindNewMachineSet(deployment *clusterv1alpha1.MachineDeployment, msList []*clusterv1alpha1.MachineSet) *clusterv1alpha1.MachineSet { sort.Sort(MachineSetsByCreationTimestamp(msList)) for i := range msList { if EqualIgnoreHash(&msList[i].Spec.Template, &deployment.Spec.Template) { @@ -442,9 +442,9 @@ func FindNewMachineSet(deployment *v1alpha1.MachineDeployment, msList []*v1alpha // Returns two list of machine sets // - the first contains all old machine sets with all non-zero replicas // - the second contains all old machine sets -func FindOldMachineSets(deployment *v1alpha1.MachineDeployment, msList []*v1alpha1.MachineSet) ([]*v1alpha1.MachineSet, []*v1alpha1.MachineSet) { - var requiredMSs []*v1alpha1.MachineSet - allMSs := make([]*v1alpha1.MachineSet, 0, len(msList)) +func FindOldMachineSets(deployment *clusterv1alpha1.MachineDeployment, msList []*clusterv1alpha1.MachineSet) ([]*clusterv1alpha1.MachineSet, []*clusterv1alpha1.MachineSet) { + var requiredMSs []*clusterv1alpha1.MachineSet + allMSs := make([]*clusterv1alpha1.MachineSet, 0, len(msList)) newMS := FindNewMachineSet(deployment, msList) for _, ms := range msList { // Filter out new machine set @@ -460,7 +460,7 @@ func FindOldMachineSets(deployment *v1alpha1.MachineDeployment, msList []*v1alph } // GetReplicaCountForMachineSets returns the sum of Replicas of the given machine sets. -func GetReplicaCountForMachineSets(machineSets []*v1alpha1.MachineSet) int32 { +func GetReplicaCountForMachineSets(machineSets []*clusterv1alpha1.MachineSet) int32 { totalReplicas := int32(0) for _, ms := range machineSets { if ms != nil { @@ -471,7 +471,7 @@ func GetReplicaCountForMachineSets(machineSets []*v1alpha1.MachineSet) int32 { } // GetActualReplicaCountForMachineSets returns the sum of actual replicas of the given machine sets. -func GetActualReplicaCountForMachineSets(machineSets []*v1alpha1.MachineSet) int32 { +func GetActualReplicaCountForMachineSets(machineSets []*clusterv1alpha1.MachineSet) int32 { totalActualReplicas := int32(0) for _, ms := range machineSets { if ms != nil { @@ -482,7 +482,7 @@ func GetActualReplicaCountForMachineSets(machineSets []*v1alpha1.MachineSet) int } // GetReadyReplicaCountForMachineSets returns the number of ready machines corresponding to the given machine sets. -func GetReadyReplicaCountForMachineSets(machineSets []*v1alpha1.MachineSet) int32 { +func GetReadyReplicaCountForMachineSets(machineSets []*clusterv1alpha1.MachineSet) int32 { totalReadyReplicas := int32(0) for _, ms := range machineSets { if ms != nil { @@ -493,7 +493,7 @@ func GetReadyReplicaCountForMachineSets(machineSets []*v1alpha1.MachineSet) int3 } // GetAvailableReplicaCountForMachineSets returns the number of available machines corresponding to the given machine sets. -func GetAvailableReplicaCountForMachineSets(machineSets []*v1alpha1.MachineSet) int32 { +func GetAvailableReplicaCountForMachineSets(machineSets []*clusterv1alpha1.MachineSet) int32 { totalAvailableReplicas := int32(0) for _, ms := range machineSets { if ms != nil { @@ -504,13 +504,13 @@ func GetAvailableReplicaCountForMachineSets(machineSets []*v1alpha1.MachineSet) } // IsRollingUpdate returns true if the strategy type is a rolling update. -func IsRollingUpdate(deployment *v1alpha1.MachineDeployment) bool { +func IsRollingUpdate(deployment *clusterv1alpha1.MachineDeployment) bool { return deployment.Spec.Strategy.Type == common.RollingUpdateMachineDeploymentStrategyType } // DeploymentComplete considers a deployment to be complete once all of its desired replicas // are updated and available, and no old machines are running. -func DeploymentComplete(deployment *v1alpha1.MachineDeployment, newStatus *v1alpha1.MachineDeploymentStatus) bool { +func DeploymentComplete(deployment *clusterv1alpha1.MachineDeployment, newStatus *clusterv1alpha1.MachineDeploymentStatus) bool { return newStatus.UpdatedReplicas == *(deployment.Spec.Replicas) && newStatus.Replicas == *(deployment.Spec.Replicas) && newStatus.AvailableReplicas == *(deployment.Spec.Replicas) && @@ -521,7 +521,7 @@ func DeploymentComplete(deployment *v1alpha1.MachineDeployment, newStatus *v1alp // When one of the following is true, we're rolling out the deployment; otherwise, we're scaling it. // 1) The new MS is saturated: newMS's replicas == deployment's replicas // 2) Max number of machines allowed is reached: deployment's replicas + maxSurge == all MSs' replicas. -func NewMSNewReplicas(deployment *v1alpha1.MachineDeployment, allMSs []*v1alpha1.MachineSet, newMS *v1alpha1.MachineSet) (int32, error) { +func NewMSNewReplicas(deployment *clusterv1alpha1.MachineDeployment, allMSs []*clusterv1alpha1.MachineSet, newMS *clusterv1alpha1.MachineSet) (int32, error) { switch deployment.Spec.Strategy.Type { case common.RollingUpdateMachineDeploymentStrategyType: // Check if we can scale up. @@ -567,7 +567,7 @@ func NewMSNewReplicas(deployment *v1alpha1.MachineDeployment, allMSs []*v1alpha1 // Both the deployment and the machine set have to believe this machine set can own all of the desired // replicas in the deployment and the annotation helps in achieving that. All machines of the MachineSet // need to be available. -func IsSaturated(deployment *v1alpha1.MachineDeployment, ms *v1alpha1.MachineSet) bool { +func IsSaturated(deployment *clusterv1alpha1.MachineDeployment, ms *clusterv1alpha1.MachineSet) bool { if ms == nil { return false } @@ -612,18 +612,18 @@ func ResolveFenceposts(maxSurge, maxUnavailable *intstrutil.IntOrString, desired } // FilterActiveMachineSets returns machine sets that have (or at least ought to have) machines. -func FilterActiveMachineSets(machineSets []*v1alpha1.MachineSet) []*v1alpha1.MachineSet { - activeFilter := func(ms *v1alpha1.MachineSet) bool { +func FilterActiveMachineSets(machineSets []*clusterv1alpha1.MachineSet) []*clusterv1alpha1.MachineSet { + activeFilter := func(ms *clusterv1alpha1.MachineSet) bool { return ms != nil && ms.Spec.Replicas != nil && *(ms.Spec.Replicas) > 0 } return FilterMachineSets(machineSets, activeFilter) } -type filterMS func(ms *v1alpha1.MachineSet) bool +type filterMS func(ms *clusterv1alpha1.MachineSet) bool // FilterMachineSets returns machine sets that are filtered by filterFn (all returned ones should match filterFn). -func FilterMachineSets(MSes []*v1alpha1.MachineSet, filterFn filterMS) []*v1alpha1.MachineSet { - var filtered []*v1alpha1.MachineSet +func FilterMachineSets(MSes []*clusterv1alpha1.MachineSet, filterFn filterMS) []*clusterv1alpha1.MachineSet { + var filtered []*clusterv1alpha1.MachineSet for i := range MSes { if filterFn(MSes[i]) { filtered = append(filtered, MSes[i]) @@ -702,7 +702,7 @@ func DeepHashObject(hasher hash.Hash, objectToWrite interface{}) { printer.Fprintf(hasher, "%#v", objectToWrite) } -func ComputeHash(template *v1alpha1.MachineTemplateSpec) uint32 { +func ComputeHash(template *clusterv1alpha1.MachineTemplateSpec) uint32 { machineTemplateSpecHasher := fnv.New32a() DeepHashObject(machineTemplateSpecHasher, *template) diff --git a/pkg/apis/cluster/v1alpha1/migrations/migrations.go b/pkg/migrations/migrations.go similarity index 96% rename from pkg/apis/cluster/v1alpha1/migrations/migrations.go rename to pkg/migrations/migrations.go index f2e3e2f3a..bfebff658 100644 --- a/pkg/apis/cluster/v1alpha1/migrations/migrations.go +++ b/pkg/migrations/migrations.go @@ -25,20 +25,19 @@ import ( "go.uber.org/zap" machinecontrolleradmission "k8c.io/machine-controller/pkg/admission" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1/conversions" "k8c.io/machine-controller/pkg/cloudprovider" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" machinecontroller "k8c.io/machine-controller/pkg/controller/machine" - "k8c.io/machine-controller/pkg/machines" - machinesv1alpha1 "k8c.io/machine-controller/pkg/machines/v1alpha1" - "k8c.io/machine-controller/pkg/providerconfig" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1/conversions" + "k8c.io/machine-controller/sdk/apis/machines" + machinesv1alpha1 "k8c.io/machine-controller/sdk/apis/machines/v1alpha1" + "k8c.io/machine-controller/sdk/providerconfig" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/equality" - kerrors "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -159,7 +158,7 @@ func MigrateMachinesv1Alpha1MachineToClusterv1Alpha1MachineIfNecessary( err := wait.PollUntilContextTimeout(ctx, cachePopulatingInterval, cachePopulatingTimeout, false, func(ctx context.Context) (bool, error) { err := client.Get(ctx, types.NamespacedName{Name: machines.CRDName}, &apiextensionsv1.CustomResourceDefinition{}) if err != nil { - if kerrors.IsNotFound(err) { + if apierrors.IsNotFound(err) { noMigrationNeed = true return true, nil } @@ -231,7 +230,7 @@ func migrateMachines(ctx context.Context, log *zap.SugaredLogger, client ctrlrun // Some providers need to update the provider instance to the new UID, we get the provider as early as possible // to not fail in a half-migrated state when the providerconfig is invalid - providerConfig, err := providerconfigtypes.GetConfig(convertedClusterv1alpha1Machine.Spec.ProviderSpec) + providerConfig, err := providerconfig.GetConfig(convertedClusterv1alpha1Machine.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to get provider config: %w", err) } @@ -260,7 +259,7 @@ func migrateMachines(ctx context.Context, log *zap.SugaredLogger, client ctrlrun existingClusterV1alpha1Machine) if err != nil { // Some random error occurred - if !kerrors.IsNotFound(err) { + if !apierrors.IsNotFound(err) { return fmt.Errorf("failed to check if converted machine %s already exists: %w", convertedClusterv1alpha1Machine.Name, err) } @@ -347,7 +346,7 @@ func ensureClusterV1Alpha1NodeOwnership(ctx context.Context, machineLog *zap.Sug for _, nodeName := range nodeNameCandidates { node := &corev1.Node{} if err := client.Get(ctx, types.NamespacedName{Name: nodeName}, node); err != nil { - if kerrors.IsNotFound(err) { + if apierrors.IsNotFound(err) { machineLog.Info("No node for machines found") continue } @@ -399,7 +398,7 @@ func deleteMachinesV1Alpha1Machine(ctx context.Context, func isMachinesV1Alpha1MachineDeleted(ctx context.Context, name string, client ctrlruntimeclient.Client) (bool, error) { if err := client.Get(ctx, types.NamespacedName{Name: name}, &machinesv1alpha1.Machine{}); err != nil { - if kerrors.IsNotFound(err) { + if apierrors.IsNotFound(err) { return true, nil } return false, err diff --git a/pkg/node/eviction/eviction.go b/pkg/node/eviction/eviction.go index 740bafe87..f41c27007 100644 --- a/pkg/node/eviction/eviction.go +++ b/pkg/node/eviction/eviction.go @@ -23,12 +23,12 @@ import ( "go.uber.org/zap" - evictiontypes "k8c.io/machine-controller/pkg/node/eviction/types" "k8c.io/machine-controller/pkg/node/nodemanager" + nodetypes "k8c.io/machine-controller/sdk/node" corev1 "k8s.io/api/core/v1" - policy "k8s.io/api/policy/v1beta1" - kerrors "k8s.io/apimachinery/pkg/api/errors" + policyv1beta1 "k8s.io/api/policy/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/kubernetes" @@ -58,8 +58,8 @@ func (ne *NodeEviction) Run(ctx context.Context, log *zap.SugaredLogger) (bool, if err != nil { return false, fmt.Errorf("failed to get node from lister: %w", err) } - if _, exists := node.Annotations[evictiontypes.SkipEvictionAnnotationKey]; exists { - nodeLog.Infof("Skipping eviction for node as it has a %s annotation", evictiontypes.SkipEvictionAnnotationKey) + if _, exists := node.Annotations[nodetypes.SkipEvictionAnnotationKey]; exists { + nodeLog.Infof("Skipping eviction for node as it has a %s annotation", nodetypes.SkipEvictionAnnotationKey) return false, nil } @@ -134,10 +134,10 @@ func (ne *NodeEviction) evictPods(ctx context.Context, log *zap.SugaredLogger, p return } err := ne.evictPod(ctx, &p) - if err == nil || kerrors.IsNotFound(err) { + if err == nil || apierrors.IsNotFound(err) { log.Debugw("Successfully evicted pod on node", "pod", ctrlruntimeclient.ObjectKeyFromObject(&p)) return - } else if kerrors.IsTooManyRequests(err) { + } else if apierrors.IsTooManyRequests(err) { // PDB prevents eviction, return and make the controller retry later return } @@ -162,7 +162,7 @@ func (ne *NodeEviction) evictPods(ctx context.Context, log *zap.SugaredLogger, p } func (ne *NodeEviction) evictPod(ctx context.Context, pod *corev1.Pod) error { - eviction := &policy.Eviction{ + eviction := &policyv1beta1.Eviction{ ObjectMeta: metav1.ObjectMeta{ Name: pod.Name, Namespace: pod.Namespace, diff --git a/pkg/node/poddeletion/pod_deletion.go b/pkg/node/poddeletion/pod_deletion.go index bc3af41c4..47dec7e35 100644 --- a/pkg/node/poddeletion/pod_deletion.go +++ b/pkg/node/poddeletion/pod_deletion.go @@ -26,7 +26,7 @@ import ( "k8c.io/machine-controller/pkg/node/nodemanager" corev1 "k8s.io/api/core/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -123,7 +123,7 @@ func (vc *NodeVolumeAttachmentsCleanup) getFilteredPods(ctx context.Context) ([] defer wg.Done() pods, err := vc.kubeClient.CoreV1().Pods(pvc.Namespace).List(ctx, metav1.ListOptions{}) switch { - case kerrors.IsTooManyRequests(err): + case apierrors.IsTooManyRequests(err): return case err != nil: errCh <- fmt.Errorf("failed to list pod: %w", err) @@ -183,10 +183,10 @@ func (vc *NodeVolumeAttachmentsCleanup) deletePods(ctx context.Context, log *zap return } err := vc.kubeClient.CoreV1().Pods(p.Namespace).Delete(ctx, p.Name, metav1.DeleteOptions{}) - if err == nil || kerrors.IsNotFound(err) { + if err == nil || apierrors.IsNotFound(err) { log.Debugw("Successfully deleted pod on node", "pod", ctrlruntimeclient.ObjectKeyFromObject(&p)) return - } else if kerrors.IsTooManyRequests(err) { + } else if apierrors.IsTooManyRequests(err) { // PDB prevents pod deletion, return and make the controller retry later. return } diff --git a/pkg/rhsm/util.go b/pkg/rhsm/util.go index 6e8fcb7a9..823c85ee2 100644 --- a/pkg/rhsm/util.go +++ b/pkg/rhsm/util.go @@ -17,9 +17,9 @@ limitations under the License. package rhsm import ( - "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" "k8c.io/machine-controller/pkg/cloudprovider/types" kuberneteshelper "k8c.io/machine-controller/pkg/kubernetes" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" ) const ( @@ -27,9 +27,9 @@ const ( ) // AddRHELSubscriptionFinalizer adds finalizer RedhatSubscriptionFinalizer to the machine object on rhel machine creation. -func AddRHELSubscriptionFinalizer(machine *v1alpha1.Machine, update types.MachineUpdater) error { +func AddRHELSubscriptionFinalizer(machine *clusterv1alpha1.Machine, update types.MachineUpdater) error { if !kuberneteshelper.HasFinalizer(machine, RedhatSubscriptionFinalizer) { - if err := update(machine, func(m *v1alpha1.Machine) { + if err := update(machine, func(m *clusterv1alpha1.Machine) { m.Finalizers = append(m.Finalizers, RedhatSubscriptionFinalizer) }); err != nil { return err @@ -40,9 +40,9 @@ func AddRHELSubscriptionFinalizer(machine *v1alpha1.Machine, update types.Machin } // RemoveRHELSubscriptionFinalizer removes finalizer RedhatSubscriptionFinalizer to the machine object on rhel machine deletion. -func RemoveRHELSubscriptionFinalizer(machine *v1alpha1.Machine, update types.MachineUpdater) error { +func RemoveRHELSubscriptionFinalizer(machine *clusterv1alpha1.Machine, update types.MachineUpdater) error { if kuberneteshelper.HasFinalizer(machine, RedhatSubscriptionFinalizer) { - if err := update(machine, func(m *v1alpha1.Machine) { + if err := update(machine, func(m *clusterv1alpha1.Machine) { m.Finalizers = kuberneteshelper.RemoveFinalizer(m.Finalizers, RedhatSubscriptionFinalizer) }); err != nil { return err diff --git a/sdk/.golangci.yml b/sdk/.golangci.yml new file mode 100644 index 000000000..10a3f40e2 --- /dev/null +++ b/sdk/.golangci.yml @@ -0,0 +1,39 @@ +# Copyright 2022 The Kubermatic Kubernetes Platform contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# This file contains *additional* linting rules that just apply to the SDK. +# When running `make lint`, the SDK is linted twice, once with the repository +# root's .golangci.yml and once with the SDK's config file. +# + +run: + timeout: 10m + modules-download-mode: readonly + +linters: + enable: + - depguard + disable-all: true + +linters-settings: + depguard: + rules: + noreverse: + deny: + - { pkg: k8c.io/machine-controller/pkg, desc: SDK must not depend on the main module } + +issues: + exclude-files: + - zz_generated.*.go diff --git a/sdk/LICENSE b/sdk/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/sdk/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/sdk/Makefile b/sdk/Makefile new file mode 100644 index 000000000..d49b0491f --- /dev/null +++ b/sdk/Makefile @@ -0,0 +1,18 @@ +# Copyright 2025 The Machine Controller Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.PHONY: lint +lint: + golangci-lint run --verbose --print-resources-usage ./... + golangci-lint run --verbose --print-resources-usage --config ../.golangci.yml ./... diff --git a/pkg/apis/cluster/common/consts.go b/sdk/apis/cluster/common/consts.go similarity index 100% rename from pkg/apis/cluster/common/consts.go rename to sdk/apis/cluster/common/consts.go diff --git a/pkg/apis/cluster/common/plugins.go b/sdk/apis/cluster/common/plugins.go similarity index 93% rename from pkg/apis/cluster/common/plugins.go rename to sdk/apis/cluster/common/plugins.go index 9439a5e53..7d5a7d774 100644 --- a/pkg/apis/cluster/common/plugins.go +++ b/sdk/apis/cluster/common/plugins.go @@ -19,8 +19,6 @@ package common import ( "fmt" "sync" - - "github.com/pkg/errors" ) var ( @@ -44,7 +42,7 @@ func ClusterProvisioner(name string) (interface{}, error) { defer providersMutex.Unlock() provisioner, found := providers[name] if !found { - return nil, errors.Errorf("failed to find provisioner for %s", name) + return nil, fmt.Errorf("failed to find provisioner for %s", name) } return provisioner, nil } diff --git a/pkg/apis/cluster/v1alpha1/common_types.go b/sdk/apis/cluster/v1alpha1/common_types.go similarity index 100% rename from pkg/apis/cluster/v1alpha1/common_types.go rename to sdk/apis/cluster/v1alpha1/common_types.go diff --git a/pkg/apis/cluster/v1alpha1/conversions/conversions.go b/sdk/apis/cluster/v1alpha1/conversions/conversions.go similarity index 89% rename from pkg/apis/cluster/v1alpha1/conversions/conversions.go rename to sdk/apis/cluster/v1alpha1/conversions/conversions.go index d5a729f9b..88a649681 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/conversions.go +++ b/sdk/apis/cluster/v1alpha1/conversions/conversions.go @@ -20,8 +20,8 @@ import ( "encoding/json" "fmt" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" - machinesv1alpha1 "k8c.io/machine-controller/pkg/machines/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + machinesv1alpha1 "k8c.io/machine-controller/sdk/apis/machines/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" @@ -36,8 +36,8 @@ func Convert_MachinesV1alpha1Machine_To_ClusterV1alpha1Machine(in *machinesv1alp out.CreationTimestamp = metav1.Time{} out.ObjectMeta.Namespace = metav1.NamespaceSystem - // k8c.io/machine-controller/pkg/apis/cluster/v1alpha1.MachineStatus and - // pkg/machines/v1alpha1.MachineStatus are semantically identical, the former + // k8c.io/machine-controller/sdk/apis/cluster/v1alpha1.MachineStatus and + // sdk/apis/machines/v1alpha1.MachineStatus are semantically identical, the former // only has one additional field, so we cast by serializing and deserializing inStatusJSON, err := json.Marshal(in.Status) if err != nil { diff --git a/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go b/sdk/apis/cluster/v1alpha1/conversions/conversions_test.go similarity index 94% rename from pkg/apis/cluster/v1alpha1/conversions/conversions_test.go rename to sdk/apis/cluster/v1alpha1/conversions/conversions_test.go index 563b2cdd8..52a57404a 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go +++ b/sdk/apis/cluster/v1alpha1/conversions/conversions_test.go @@ -23,8 +23,8 @@ import ( "os" "testing" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" - machinesv1alpha1 "k8c.io/machine-controller/pkg/machines/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + machinesv1alpha1 "k8c.io/machine-controller/sdk/apis/machines/v1alpha1" kyaml "k8s.io/apimachinery/pkg/util/yaml" "sigs.k8s.io/yaml" diff --git a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec.go b/sdk/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec.go similarity index 99% rename from pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec.go rename to sdk/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec.go index eca0097e6..c7e26fbb0 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec.go +++ b/sdk/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec.go @@ -20,7 +20,7 @@ import ( "encoding/json" "fmt" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" diff --git a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go b/sdk/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go similarity index 98% rename from pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go rename to sdk/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go index 56f906404..feb220858 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go +++ b/sdk/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go @@ -22,7 +22,7 @@ import ( "os" "testing" - testhelper "k8c.io/machine-controller/pkg/test" + testhelper "k8c.io/machine-controller/sdk/internal/test" "sigs.k8s.io/yaml" ) diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/aws.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/aws.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/aws.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/aws.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/hetzner.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/hetzner.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/hetzner.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/clusterv1alpha1machineWithProviderConfig/hetzner.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/aws.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/aws.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/aws.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/aws.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/azure.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/azure.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/azure.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/azure.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/digitalocean.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/digitalocean.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/digitalocean.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/digitalocean.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/hetzner.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/hetzner.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/hetzner.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/hetzner.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/linode.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/linode.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/linode.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/linode.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere-static-ip.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere-static-ip.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere-static-ip.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere-static-ip.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/vsphere.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/aws.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/aws.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/aws.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/aws.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/azure.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/azure.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/azure.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/azure.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/digitalocean.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/digitalocean.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/digitalocean.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/digitalocean.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/hetzner.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/hetzner.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/hetzner.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/hetzner.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/linode.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/linode.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/linode.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/linode.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/openstack.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/openstack.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/openstack.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/openstack.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere-static-ip.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere-static-ip.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere-static-ip.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere-static-ip.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machine/vsphere.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineDeploymentWithProviderConfig/hetzner.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineSetWithProviderConfig/hetzner.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/aws.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/aws.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/aws.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/aws.yaml diff --git a/pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/hetzner.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/hetzner.yaml similarity index 100% rename from pkg/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/hetzner.yaml rename to sdk/apis/cluster/v1alpha1/conversions/testdata/migrated_clusterv1alpha1machineWithProviderConfig/hetzner.yaml diff --git a/pkg/apis/cluster/v1alpha1/defaults.go b/sdk/apis/cluster/v1alpha1/defaults.go similarity index 97% rename from pkg/apis/cluster/v1alpha1/defaults.go rename to sdk/apis/cluster/v1alpha1/defaults.go index ad61b3313..a022f33ce 100644 --- a/pkg/apis/cluster/v1alpha1/defaults.go +++ b/sdk/apis/cluster/v1alpha1/defaults.go @@ -17,7 +17,7 @@ limitations under the License. package v1alpha1 import ( - "k8c.io/machine-controller/pkg/apis/cluster/common" + "k8c.io/machine-controller/sdk/apis/cluster/common" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" diff --git a/pkg/apis/cluster/v1alpha1/doc.go b/sdk/apis/cluster/v1alpha1/doc.go similarity index 92% rename from pkg/apis/cluster/v1alpha1/doc.go rename to sdk/apis/cluster/v1alpha1/doc.go index 3a530de12..3fe0390c2 100644 --- a/pkg/apis/cluster/v1alpha1/doc.go +++ b/sdk/apis/cluster/v1alpha1/doc.go @@ -17,7 +17,7 @@ limitations under the License. // Package v1alpha1 contains API Schema definitions for the cluster v1alpha1 API group // +k8s:openapi-gen=true // +k8s:deepcopy-gen=package,register -// +k8s:conversion-gen=k8c.io/machine-controller/pkg/apis/cluster +// +k8s:conversion-gen=k8c.io/machine-controller/sdk/apis/cluster // +k8s:defaulter-gen=TypeMeta // +groupName=cluster.k8s.io package v1alpha1 diff --git a/pkg/apis/cluster/v1alpha1/machine_types.go b/sdk/apis/cluster/v1alpha1/machine_types.go similarity index 98% rename from pkg/apis/cluster/v1alpha1/machine_types.go rename to sdk/apis/cluster/v1alpha1/machine_types.go index 01bdb8fb8..6371ef270 100644 --- a/pkg/apis/cluster/v1alpha1/machine_types.go +++ b/sdk/apis/cluster/v1alpha1/machine_types.go @@ -17,7 +17,7 @@ limitations under the License. package v1alpha1 import ( - "k8c.io/machine-controller/pkg/apis/cluster/common" + "k8c.io/machine-controller/sdk/apis/cluster/common" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -248,7 +248,3 @@ type MachineList struct { metav1.ListMeta `json:"metadata,omitempty"` Items []Machine `json:"items"` } - -func init() { - SchemeBuilder.Register(&Machine{}, &MachineList{}) -} diff --git a/pkg/apis/cluster/v1alpha1/machineclass_types.go b/sdk/apis/cluster/v1alpha1/machineclass_types.go similarity index 97% rename from pkg/apis/cluster/v1alpha1/machineclass_types.go rename to sdk/apis/cluster/v1alpha1/machineclass_types.go index b73553fbd..f1e30f07b 100644 --- a/pkg/apis/cluster/v1alpha1/machineclass_types.go +++ b/sdk/apis/cluster/v1alpha1/machineclass_types.go @@ -73,7 +73,3 @@ type MachineClassList struct { metav1.ListMeta `json:"metadata,omitempty"` Items []MachineClass `json:"items"` } - -func init() { - SchemeBuilder.Register(&MachineClass{}, &MachineClassList{}) -} diff --git a/pkg/apis/cluster/v1alpha1/machinedeployment_types.go b/sdk/apis/cluster/v1alpha1/machinedeployment_types.go similarity index 98% rename from pkg/apis/cluster/v1alpha1/machinedeployment_types.go rename to sdk/apis/cluster/v1alpha1/machinedeployment_types.go index 110dc9175..8faad9d9a 100644 --- a/pkg/apis/cluster/v1alpha1/machinedeployment_types.go +++ b/sdk/apis/cluster/v1alpha1/machinedeployment_types.go @@ -17,7 +17,7 @@ limitations under the License. package v1alpha1 import ( - "k8c.io/machine-controller/pkg/apis/cluster/common" + "k8c.io/machine-controller/sdk/apis/cluster/common" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -189,7 +189,3 @@ type MachineDeploymentList struct { metav1.ListMeta `json:"metadata,omitempty"` Items []MachineDeployment `json:"items"` } - -func init() { - SchemeBuilder.Register(&MachineDeployment{}, &MachineDeploymentList{}) -} diff --git a/pkg/apis/cluster/v1alpha1/machineset_types.go b/sdk/apis/cluster/v1alpha1/machineset_types.go similarity index 98% rename from pkg/apis/cluster/v1alpha1/machineset_types.go rename to sdk/apis/cluster/v1alpha1/machineset_types.go index 2bce745e5..169c25f8a 100644 --- a/pkg/apis/cluster/v1alpha1/machineset_types.go +++ b/sdk/apis/cluster/v1alpha1/machineset_types.go @@ -19,7 +19,7 @@ package v1alpha1 import ( "log" - "k8c.io/machine-controller/pkg/apis/cluster/common" + "k8c.io/machine-controller/sdk/apis/cluster/common" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" @@ -217,7 +217,3 @@ type MachineSetList struct { metav1.ListMeta `json:"metadata,omitempty"` Items []MachineSet `json:"items"` } - -func init() { - SchemeBuilder.Register(&MachineSet{}, &MachineSetList{}) -} diff --git a/pkg/apis/cluster/v1alpha1/register.go b/sdk/apis/cluster/v1alpha1/register.go similarity index 54% rename from pkg/apis/cluster/v1alpha1/register.go rename to sdk/apis/cluster/v1alpha1/register.go index f6fd7b6db..3e6941d2b 100644 --- a/pkg/apis/cluster/v1alpha1/register.go +++ b/sdk/apis/cluster/v1alpha1/register.go @@ -25,25 +25,41 @@ limitations under the License. package v1alpha1 import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/scheme" ) var ( - // SchemeGroupVersion is group version used to register these objects. - SchemeGroupVersion = schema.GroupVersion{Group: "cluster.k8s.io", Version: "v1alpha1"} + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + AddToScheme = SchemeBuilder.AddToScheme +) - // SchemeBuilder is used to add go types to the GroupVersionKind scheme. - SchemeBuilder = &scheme.Builder{GroupVersion: SchemeGroupVersion} +// GroupName is the group name use in this package. +const GroupName = "cluster.k8s.io" +const GroupVersion = "v1alpha1" - // AddToScheme adds registered types to the builder. - // Required by pkg/client/... - // TODO(pwittrock): Remove this after removing pkg/client/... - AddToScheme = SchemeBuilder.AddToScheme -) +// SchemeGroupVersion is group version used to register these objects. +var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: GroupVersion} -// Required by pkg/client/listers/... -// TODO(pwittrock): Remove this after removing pkg/client/... +// Resource takes an unqualified resource and returns a Group qualified GroupResource. func Resource(resource string) schema.GroupResource { return SchemeGroupVersion.WithResource(resource).GroupResource() } + +// Adds the list of known types to api.Scheme. +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &Machine{}, + &MachineList{}, + &MachineClass{}, + &MachineClassList{}, + &MachineDeployment{}, + &MachineDeploymentList{}, + &MachineSet{}, + &MachineSetList{}, + ) + + metav1.AddToGroupVersion(scheme, SchemeGroupVersion) + return nil +} diff --git a/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go b/sdk/apis/cluster/v1alpha1/zz_generated.deepcopy.go similarity index 98% rename from pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go rename to sdk/apis/cluster/v1alpha1/zz_generated.deepcopy.go index e9a7bd9eb..c4d6068e2 100644 --- a/pkg/apis/cluster/v1alpha1/zz_generated.deepcopy.go +++ b/sdk/apis/cluster/v1alpha1/zz_generated.deepcopy.go @@ -22,8 +22,8 @@ limitations under the License. package v1alpha1 import ( - common "k8c.io/machine-controller/pkg/apis/cluster/common" - v1 "k8s.io/api/core/v1" + common "k8c.io/machine-controller/sdk/apis/cluster/common" + corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" intstr "k8s.io/apimachinery/pkg/util/intstr" ) @@ -156,7 +156,7 @@ func (in *MachineClassRef) DeepCopyInto(out *MachineClassRef) { *out = *in if in.ObjectReference != nil { in, out := &in.ObjectReference, &out.ObjectReference - *out = new(v1.ObjectReference) + *out = new(corev1.ObjectReference) **out = **in } return @@ -488,7 +488,7 @@ func (in *MachineSpec) DeepCopyInto(out *MachineSpec) { in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) if in.Taints != nil { in, out := &in.Taints, &out.Taints - *out = make([]v1.Taint, len(*in)) + *out = make([]corev1.Taint, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -497,7 +497,7 @@ func (in *MachineSpec) DeepCopyInto(out *MachineSpec) { out.Versions = in.Versions if in.ConfigSource != nil { in, out := &in.ConfigSource, &out.ConfigSource - *out = new(v1.NodeConfigSource) + *out = new(corev1.NodeConfigSource) (*in).DeepCopyInto(*out) } if in.ProviderID != nil { @@ -523,7 +523,7 @@ func (in *MachineStatus) DeepCopyInto(out *MachineStatus) { *out = *in if in.NodeRef != nil { in, out := &in.NodeRef, &out.NodeRef - *out = new(v1.ObjectReference) + *out = new(corev1.ObjectReference) **out = **in } if in.LastUpdated != nil { @@ -552,12 +552,12 @@ func (in *MachineStatus) DeepCopyInto(out *MachineStatus) { } if in.Addresses != nil { in, out := &in.Addresses, &out.Addresses - *out = make([]v1.NodeAddress, len(*in)) + *out = make([]corev1.NodeAddress, len(*in)) copy(*out, *in) } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions - *out = make([]v1.NodeCondition, len(*in)) + *out = make([]corev1.NodeCondition, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } diff --git a/pkg/machines/register.go b/sdk/apis/machines/register.go similarity index 93% rename from pkg/machines/register.go rename to sdk/apis/machines/register.go index 76e3f862e..69e53518d 100644 --- a/pkg/machines/register.go +++ b/sdk/apis/machines/register.go @@ -19,7 +19,7 @@ package machines import ( "reflect" - "k8c.io/machine-controller/pkg/machines/v1alpha1" + "k8c.io/machine-controller/sdk/apis/machines/v1alpha1" ) type resource struct { diff --git a/pkg/machines/v1alpha1/defaults.go b/sdk/apis/machines/v1alpha1/defaults.go similarity index 100% rename from pkg/machines/v1alpha1/defaults.go rename to sdk/apis/machines/v1alpha1/defaults.go diff --git a/pkg/machines/v1alpha1/doc.go b/sdk/apis/machines/v1alpha1/doc.go similarity index 100% rename from pkg/machines/v1alpha1/doc.go rename to sdk/apis/machines/v1alpha1/doc.go diff --git a/pkg/machines/v1alpha1/register.go b/sdk/apis/machines/v1alpha1/register.go similarity index 100% rename from pkg/machines/v1alpha1/register.go rename to sdk/apis/machines/v1alpha1/register.go diff --git a/pkg/machines/v1alpha1/types.go b/sdk/apis/machines/v1alpha1/types.go similarity index 100% rename from pkg/machines/v1alpha1/types.go rename to sdk/apis/machines/v1alpha1/types.go diff --git a/pkg/machines/v1alpha1/zz_generated.deepcopy.go b/sdk/apis/machines/v1alpha1/zz_generated.deepcopy.go similarity index 97% rename from pkg/machines/v1alpha1/zz_generated.deepcopy.go rename to sdk/apis/machines/v1alpha1/zz_generated.deepcopy.go index 2510c81ce..8dcc314b6 100644 --- a/pkg/machines/v1alpha1/zz_generated.deepcopy.go +++ b/sdk/apis/machines/v1alpha1/zz_generated.deepcopy.go @@ -22,7 +22,7 @@ limitations under the License. package v1alpha1 import ( - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -93,7 +93,7 @@ func (in *MachineSpec) DeepCopyInto(out *MachineSpec) { in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) if in.Taints != nil { in, out := &in.Taints, &out.Taints - *out = make([]v1.Taint, len(*in)) + *out = make([]corev1.Taint, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -107,7 +107,7 @@ func (in *MachineSpec) DeepCopyInto(out *MachineSpec) { out.Versions = in.Versions if in.ConfigSource != nil { in, out := &in.ConfigSource, &out.ConfigSource - *out = new(v1.NodeConfigSource) + *out = new(corev1.NodeConfigSource) (*in).DeepCopyInto(*out) } return @@ -128,7 +128,7 @@ func (in *MachineStatus) DeepCopyInto(out *MachineStatus) { *out = *in if in.NodeRef != nil { in, out := &in.NodeRef, &out.NodeRef - *out = new(v1.ObjectReference) + *out = new(corev1.ObjectReference) **out = **in } in.LastUpdated.DeepCopyInto(&out.LastUpdated) diff --git a/pkg/bootstrap/doc.go b/sdk/bootstrap/doc.go similarity index 100% rename from pkg/bootstrap/doc.go rename to sdk/bootstrap/doc.go diff --git a/pkg/bootstrap/types.go b/sdk/bootstrap/types.go similarity index 100% rename from pkg/bootstrap/types.go rename to sdk/bootstrap/types.go diff --git a/sdk/cloudprovider/alibaba/types.go b/sdk/cloudprovider/alibaba/types.go new file mode 100644 index 000000000..1d4192538 --- /dev/null +++ b/sdk/cloudprovider/alibaba/types.go @@ -0,0 +1,42 @@ +/* +Copyright 2019 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package alibaba + +import ( + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" +) + +type RawConfig struct { + AccessKeyID providerconfig.ConfigVarString `json:"accessKeyID,omitempty"` + AccessKeySecret providerconfig.ConfigVarString `json:"accessKeySecret,omitempty"` + RegionID providerconfig.ConfigVarString `json:"regionID,omitempty"` + InstanceName providerconfig.ConfigVarString `json:"instanceName,omitempty"` + InstanceType providerconfig.ConfigVarString `json:"instanceType,omitempty"` + VSwitchID providerconfig.ConfigVarString `json:"vSwitchID,omitempty"` + InternetMaxBandwidthOut providerconfig.ConfigVarString `json:"internetMaxBandwidthOut,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + ZoneID providerconfig.ConfigVarString `json:"zoneID,omitempty"` + DiskType providerconfig.ConfigVarString `json:"diskType,omitempty"` + DiskSize providerconfig.ConfigVarString `json:"diskSize,omitempty"` +} + +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/anexia/types/types.go b/sdk/cloudprovider/anexia/types.go similarity index 62% rename from pkg/cloudprovider/provider/anexia/types/types.go rename to sdk/cloudprovider/anexia/types.go index 9c01eabf3..008ccb7e2 100644 --- a/pkg/cloudprovider/provider/anexia/types/types.go +++ b/sdk/cloudprovider/anexia/types.go @@ -14,15 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -package types +package anexia import ( + "errors" "time" - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( @@ -40,31 +41,39 @@ const ( MachinePoweredOn = "poweredOn" ) +var ( + // ErrConfigDiskSizeAndDisks is returned when the config has both DiskSize and Disks set, which is unsupported. + ErrConfigDiskSizeAndDisks = errors.New("both the deprecated DiskSize and new Disks attribute are set") + + // ErrConfigVlanIDAndNetworks is returned when the config has both VlanID and Networks set, which is unsupported. + ErrConfigVlanIDAndNetworks = errors.New("both the deprecated VlanID and new Networks attribute are set") +) + // RawDisk specifies a single disk, with some values maybe being fetched from secrets. type RawDisk struct { - Size int `json:"size"` - PerformanceType providerconfigtypes.ConfigVarString `json:"performanceType"` + Size int `json:"size"` + PerformanceType providerconfig.ConfigVarString `json:"performanceType"` } // RawNetwork specifies a single network interface. type RawNetwork struct { // Identifier of the VLAN to attach this network interface to. - VlanID providerconfigtypes.ConfigVarString `json:"vlan"` + VlanID providerconfig.ConfigVarString `json:"vlan"` // IDs of prefixes to reserve IP addresses from for each Machine on network interface. // // Empty list means that no IPs will be reserved, but the interface will still be added. - PrefixIDs []providerconfigtypes.ConfigVarString `json:"prefixes"` + PrefixIDs []providerconfig.ConfigVarString `json:"prefixes"` } // RawConfig contains all the configuration values for VMs to create, with some values maybe being fetched from secrets. type RawConfig struct { - Token providerconfigtypes.ConfigVarString `json:"token,omitempty"` - LocationID providerconfigtypes.ConfigVarString `json:"locationID"` + Token providerconfig.ConfigVarString `json:"token,omitempty"` + LocationID providerconfig.ConfigVarString `json:"locationID"` - TemplateID providerconfigtypes.ConfigVarString `json:"templateID"` - Template providerconfigtypes.ConfigVarString `json:"template"` - TemplateBuild providerconfigtypes.ConfigVarString `json:"templateBuild"` + TemplateID providerconfig.ConfigVarString `json:"templateID"` + Template providerconfig.ConfigVarString `json:"template"` + TemplateBuild providerconfig.ConfigVarString `json:"templateBuild"` CPUs int `json:"cpus"` CPUPerformanceType string `json:"cpuPerformanceType"` @@ -76,7 +85,7 @@ type RawConfig struct { Disks []RawDisk `json:"disks"` // Deprecated, use Networks instead. - VlanID providerconfigtypes.ConfigVarString `json:"vlanID"` + VlanID providerconfig.ConfigVarString `json:"vlanID"` // Configuration of the network interfaces. At least one entry with at // least one Prefix is required. @@ -95,16 +104,16 @@ type NetworkStatus struct { } type ProviderStatus struct { - InstanceID string `json:"instanceID"` - ProvisioningID string `json:"provisioningID"` - DeprovisioningID string `json:"deprovisioningID"` - Conditions []v1.Condition `json:"conditions,omitempty"` + InstanceID string `json:"instanceID"` + ProvisioningID string `json:"provisioningID"` + DeprovisioningID string `json:"deprovisioningID"` + Conditions []metav1.Condition `json:"conditions,omitempty"` // each entry belongs to the config.Networks entry at the same index Networks []NetworkStatus `json:"networkStatus,omitempty"` } -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { rawConfig := &RawConfig{} return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) diff --git a/sdk/cloudprovider/aws/types.go b/sdk/cloudprovider/aws/types.go new file mode 100644 index 000000000..96b9820fb --- /dev/null +++ b/sdk/cloudprovider/aws/types.go @@ -0,0 +1,69 @@ +/* +Copyright 2019 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" +) + +type RawConfig struct { + AccessKeyID providerconfig.ConfigVarString `json:"accessKeyId,omitempty"` + SecretAccessKey providerconfig.ConfigVarString `json:"secretAccessKey,omitempty"` + + AssumeRoleARN providerconfig.ConfigVarString `json:"assumeRoleARN,omitempty"` + AssumeRoleExternalID providerconfig.ConfigVarString `json:"assumeRoleExternalID,omitempty"` + + Region providerconfig.ConfigVarString `json:"region"` + AvailabilityZone providerconfig.ConfigVarString `json:"availabilityZone,omitempty"` + VpcID providerconfig.ConfigVarString `json:"vpcId"` + SubnetID providerconfig.ConfigVarString `json:"subnetId"` + SecurityGroupIDs []providerconfig.ConfigVarString `json:"securityGroupIDs,omitempty"` + InstanceProfile providerconfig.ConfigVarString `json:"instanceProfile,omitempty"` + InstanceType providerconfig.ConfigVarString `json:"instanceType,omitempty"` + AMI providerconfig.ConfigVarString `json:"ami,omitempty"` + DiskSize int32 `json:"diskSize"` + DiskType providerconfig.ConfigVarString `json:"diskType,omitempty"` + DiskIops *int32 `json:"diskIops,omitempty"` + EBSVolumeEncrypted providerconfig.ConfigVarBool `json:"ebsVolumeEncrypted"` + Tags map[string]string `json:"tags,omitempty"` + AssignPublicIP *bool `json:"assignPublicIP,omitempty"` + + IsSpotInstance *bool `json:"isSpotInstance,omitempty"` + SpotInstanceConfig *SpotInstanceConfig `json:"spotInstanceConfig,omitempty"` +} + +type SpotInstanceConfig struct { + MaxPrice providerconfig.ConfigVarString `json:"maxPrice,omitempty"` + PersistentRequest providerconfig.ConfigVarBool `json:"persistentRequest,omitempty"` + InterruptionBehavior providerconfig.ConfigVarString `json:"interruptionBehavior,omitempty"` +} + +// CPUArchitecture defines processor architectures returned by the AWS API. +type CPUArchitecture string + +const ( + CPUArchitectureARM64 CPUArchitecture = "arm64" + CPUArchitectureX86_64 CPUArchitecture = "x86_64" + CPUArchitectureI386 CPUArchitecture = "i386" +) + +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/sdk/cloudprovider/azure/types.go b/sdk/cloudprovider/azure/types.go new file mode 100644 index 000000000..ae3266ab1 --- /dev/null +++ b/sdk/cloudprovider/azure/types.go @@ -0,0 +1,77 @@ +/* +Copyright 2019 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" +) + +// RawConfig is a direct representation of an Azure machine object's configuration. +type RawConfig struct { + SubscriptionID providerconfig.ConfigVarString `json:"subscriptionID,omitempty"` + TenantID providerconfig.ConfigVarString `json:"tenantID,omitempty"` + ClientID providerconfig.ConfigVarString `json:"clientID,omitempty"` + ClientSecret providerconfig.ConfigVarString `json:"clientSecret,omitempty"` + + Location providerconfig.ConfigVarString `json:"location"` + ResourceGroup providerconfig.ConfigVarString `json:"resourceGroup"` + VNetResourceGroup providerconfig.ConfigVarString `json:"vnetResourceGroup"` + VMSize providerconfig.ConfigVarString `json:"vmSize"` + VNetName providerconfig.ConfigVarString `json:"vnetName"` + SubnetName providerconfig.ConfigVarString `json:"subnetName"` + LoadBalancerSku providerconfig.ConfigVarString `json:"loadBalancerSku"` + RouteTableName providerconfig.ConfigVarString `json:"routeTableName"` + AvailabilitySet providerconfig.ConfigVarString `json:"availabilitySet"` + AssignAvailabilitySet *bool `json:"assignAvailabilitySet"` + SecurityGroupName providerconfig.ConfigVarString `json:"securityGroupName"` + Zones []string `json:"zones"` + ImagePlan *ImagePlan `json:"imagePlan,omitempty"` + ImageReference *ImageReference `json:"imageReference,omitempty"` + EnableAcceleratedNetworking *bool `json:"enableAcceleratedNetworking"` + EnableBootDiagnostics *bool `json:"enableBootDiagnostics,omitempty"` + + ImageID providerconfig.ConfigVarString `json:"imageID"` + OSDiskSize int32 `json:"osDiskSize"` + OSDiskSKU *string `json:"osDiskSKU,omitempty"` + DataDiskSize int32 `json:"dataDiskSize"` + DataDiskSKU *string `json:"dataDiskSKU,omitempty"` + AssignPublicIP providerconfig.ConfigVarBool `json:"assignPublicIP"` + PublicIPSKU *string `json:"publicIPSKU,omitempty"` + Tags map[string]string `json:"tags,omitempty"` +} + +// ImagePlan contains azure OS Plan fields for the marketplace images. +type ImagePlan struct { + Name string `json:"name,omitempty"` + Publisher string `json:"publisher,omitempty"` + Product string `json:"product,omitempty"` +} + +// ImageReference specifies information about the image to use. +type ImageReference struct { + Publisher string `json:"publisher,omitempty"` + Offer string `json:"offer,omitempty"` + Sku string `json:"sku,omitempty"` + Version string `json:"version,omitempty"` +} + +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/sdk/cloudprovider/baremetal/plugins/plugins.go b/sdk/cloudprovider/baremetal/plugins/plugins.go new file mode 100644 index 000000000..7be2b56bb --- /dev/null +++ b/sdk/cloudprovider/baremetal/plugins/plugins.go @@ -0,0 +1,21 @@ +/* +Copyright 2021 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package plugins + +type Driver string + +const Tinkerbell Driver = "tinkerbell" diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/types.go b/sdk/cloudprovider/baremetal/plugins/tinkerbell/types.go similarity index 86% rename from pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/types.go rename to sdk/cloudprovider/baremetal/plugins/tinkerbell/types.go index 012300cc5..225887dbb 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/types/types.go +++ b/sdk/cloudprovider/baremetal/plugins/tinkerbell/types.go @@ -14,26 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */ -package types +package tinkerbell import ( - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + providerconfigtypes "k8c.io/machine-controller/sdk/providerconfig" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" ) -const ( - Unknown string = "Unknown" - Staged string = "Staged" - Provisioned string = "Provisioned" - Decommissioned string = "Decommissioned" - - HardwareRefLabel = "app.kubernetes.io/hardware-reference" -) - // TinkerbellPluginSpec defines the required information for the Tinkerbell plugin. type TinkerbellPluginSpec struct { - // ClusterName specifies the name of the Tinkerbell cluster. This is used to identify // the cluster within a larger infrastructure or across multiple clusters. ClusterName providerconfigtypes.ConfigVarString `json:"clusterName"` diff --git a/pkg/cloudprovider/provider/baremetal/types/types.go b/sdk/cloudprovider/baremetal/types.go similarity index 69% rename from pkg/cloudprovider/provider/baremetal/types/types.go rename to sdk/cloudprovider/baremetal/types.go index 80feebb40..93e4e3cf4 100644 --- a/pkg/cloudprovider/provider/baremetal/types/types.go +++ b/sdk/cloudprovider/baremetal/types.go @@ -14,21 +14,21 @@ See the License for the specific language governing permissions and limitations under the License. */ -package types +package baremetal import ( - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" "k8s.io/apimachinery/pkg/runtime" ) type RawConfig struct { - Driver providerconfigtypes.ConfigVarString `json:"driver"` - DriverSpec runtime.RawExtension `json:"driverSpec"` + Driver providerconfig.ConfigVarString `json:"driver"` + DriverSpec runtime.RawExtension `json:"driverSpec"` } -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { rawConfig := &RawConfig{} return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) diff --git a/sdk/cloudprovider/digitalocean/types.go b/sdk/cloudprovider/digitalocean/types.go new file mode 100644 index 000000000..2eac6101c --- /dev/null +++ b/sdk/cloudprovider/digitalocean/types.go @@ -0,0 +1,39 @@ +/* +Copyright 2019 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package digitalocean + +import ( + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" +) + +type RawConfig struct { + Token providerconfig.ConfigVarString `json:"token,omitempty"` + Region providerconfig.ConfigVarString `json:"region"` + Size providerconfig.ConfigVarString `json:"size"` + Backups providerconfig.ConfigVarBool `json:"backups"` + IPv6 providerconfig.ConfigVarBool `json:"ipv6"` + PrivateNetworking providerconfig.ConfigVarBool `json:"private_networking"` + Monitoring providerconfig.ConfigVarBool `json:"monitoring"` + Tags []providerconfig.ConfigVarString `json:"tags,omitempty"` +} + +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/equinixmetal/types/types.go b/sdk/cloudprovider/equinixmetal/types.go similarity index 51% rename from pkg/cloudprovider/provider/equinixmetal/types/types.go rename to sdk/cloudprovider/equinixmetal/types.go index 3cbcbb7dd..a941b58a8 100644 --- a/pkg/cloudprovider/provider/equinixmetal/types/types.go +++ b/sdk/cloudprovider/equinixmetal/types.go @@ -14,24 +14,24 @@ See the License for the specific language governing permissions and limitations under the License. */ -package types +package equinixmetal import ( - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" ) type RawConfig struct { - Token providerconfigtypes.ConfigVarString `json:"token,omitempty"` - ProjectID providerconfigtypes.ConfigVarString `json:"projectID,omitempty"` - BillingCycle providerconfigtypes.ConfigVarString `json:"billingCycle"` - InstanceType providerconfigtypes.ConfigVarString `json:"instanceType"` - Metro providerconfigtypes.ConfigVarString `json:"metro,omitempty"` - Facilities []providerconfigtypes.ConfigVarString `json:"facilities,omitempty"` - Tags []providerconfigtypes.ConfigVarString `json:"tags,omitempty"` + Token providerconfig.ConfigVarString `json:"token,omitempty"` + ProjectID providerconfig.ConfigVarString `json:"projectID,omitempty"` + BillingCycle providerconfig.ConfigVarString `json:"billingCycle"` + InstanceType providerconfig.ConfigVarString `json:"instanceType"` + Metro providerconfig.ConfigVarString `json:"metro,omitempty"` + Facilities []providerconfig.ConfigVarString `json:"facilities,omitempty"` + Tags []providerconfig.ConfigVarString `json:"tags,omitempty"` } -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { rawConfig := &RawConfig{} return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) diff --git a/sdk/cloudprovider/gce/types.go b/sdk/cloudprovider/gce/types.go new file mode 100644 index 000000000..c4f05580c --- /dev/null +++ b/sdk/cloudprovider/gce/types.go @@ -0,0 +1,86 @@ +/* +Copyright 2019 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gce + +import ( + "encoding/json" + "fmt" + + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" + + "k8s.io/apimachinery/pkg/runtime" +) + +// CloudProviderSpec contains the specification of the cloud provider taken +// from the provider configuration. +type CloudProviderSpec struct { + // ServiceAccount must be base64-encoded. + ServiceAccount providerconfig.ConfigVarString `json:"serviceAccount,omitempty"` + Zone providerconfig.ConfigVarString `json:"zone"` + MachineType providerconfig.ConfigVarString `json:"machineType"` + DiskSize int64 `json:"diskSize"` + DiskType providerconfig.ConfigVarString `json:"diskType"` + Network providerconfig.ConfigVarString `json:"network"` + Subnetwork providerconfig.ConfigVarString `json:"subnetwork"` + Preemptible providerconfig.ConfigVarBool `json:"preemptible"` + AutomaticRestart *providerconfig.ConfigVarBool `json:"automaticRestart,omitempty"` + ProvisioningModel *providerconfig.ConfigVarString `json:"provisioningModel,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + Tags []string `json:"tags,omitempty"` + AssignPublicIPAddress *providerconfig.ConfigVarBool `json:"assignPublicIPAddress,omitempty"` + MultiZone providerconfig.ConfigVarBool `json:"multizone"` + Regional providerconfig.ConfigVarBool `json:"regional"` + CustomImage providerconfig.ConfigVarString `json:"customImage,omitempty"` + DisableMachineServiceAccount providerconfig.ConfigVarBool `json:"disableMachineServiceAccount,omitempty"` + EnableNestedVirtualization providerconfig.ConfigVarBool `json:"enableNestedVirtualization,omitempty"` + MinCPUPlatform providerconfig.ConfigVarString `json:"minCPUPlatform,omitempty"` + GuestOSFeatures []string `json:"guestOSFeatures,omitempty"` + ProjectID providerconfig.ConfigVarString `json:"projectID,omitempty"` +} + +// UpdateProviderSpec updates the given provider spec with changed +// configuration values. +func (cpSpec *CloudProviderSpec) UpdateProviderSpec(spec clusterv1alpha1.ProviderSpec) (*runtime.RawExtension, error) { + if spec.Value == nil { + return nil, fmt.Errorf("machine.spec.providerconfig.value is nil") + } + providerConfig := providerconfig.Config{} + err := json.Unmarshal(spec.Value.Raw, &providerConfig) + if err != nil { + return nil, err + } + rawCPSpec, err := json.Marshal(cpSpec) + if err != nil { + return nil, err + } + providerConfig.CloudProviderSpec = runtime.RawExtension{Raw: rawCPSpec} + rawProviderConfig, err := json.Marshal(providerConfig) + if err != nil { + return nil, err + } + return &runtime.RawExtension{Raw: rawProviderConfig}, nil +} + +type RawConfig = CloudProviderSpec + +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/sdk/cloudprovider/hetzner/types.go b/sdk/cloudprovider/hetzner/types.go new file mode 100644 index 000000000..a9215c5df --- /dev/null +++ b/sdk/cloudprovider/hetzner/types.go @@ -0,0 +1,42 @@ +/* +Copyright 2019 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hetzner + +import ( + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" +) + +type RawConfig struct { + Token providerconfig.ConfigVarString `json:"token,omitempty"` + ServerType providerconfig.ConfigVarString `json:"serverType"` + Datacenter providerconfig.ConfigVarString `json:"datacenter"` + Image providerconfig.ConfigVarString `json:"image"` + Location providerconfig.ConfigVarString `json:"location"` + PlacementGroupPrefix providerconfig.ConfigVarString `json:"placementGroupPrefix"` + Networks []providerconfig.ConfigVarString `json:"networks"` + Firewalls []providerconfig.ConfigVarString `json:"firewalls"` + Labels map[string]string `json:"labels,omitempty"` + AssignPublicIPv4 providerconfig.ConfigVarBool `json:"assignPublicIPv4,omitempty"` + AssignPublicIPv6 providerconfig.ConfigVarBool `json:"assignPublicIPv6,omitempty"` +} + +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/sdk/cloudprovider/kubevirt/types.go b/sdk/cloudprovider/kubevirt/types.go new file mode 100644 index 000000000..cd90b07ba --- /dev/null +++ b/sdk/cloudprovider/kubevirt/types.go @@ -0,0 +1,167 @@ +/* +Copyright 2019 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubevirt + +import ( + kubevirtcorev1 "kubevirt.io/api/core/v1" + + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" + + corev1 "k8s.io/api/core/v1" +) + +var SupportedOS = map[providerconfig.OperatingSystem]*struct{}{ + providerconfig.OperatingSystemUbuntu: nil, + providerconfig.OperatingSystemRHEL: nil, + providerconfig.OperatingSystemFlatcar: nil, + providerconfig.OperatingSystemRockyLinux: nil, +} + +type RawConfig struct { + ClusterName providerconfig.ConfigVarString `json:"clusterName"` + Auth Auth `json:"auth,omitempty"` + VirtualMachine VirtualMachine `json:"virtualMachine,omitempty"` + Affinity Affinity `json:"affinity,omitempty"` + TopologySpreadConstraints []TopologySpreadConstraint `json:"topologySpreadConstraints"` +} + +// Auth. +type Auth struct { + Kubeconfig providerconfig.ConfigVarString `json:"kubeconfig,omitempty"` +} + +// VirtualMachine. +type VirtualMachine struct { + // Deprecated: use Instancetype/Preference instead. + Flavor Flavor `json:"flavor,omitempty"` + // Instancetype is optional. + Instancetype *kubevirtcorev1.InstancetypeMatcher `json:"instancetype,omitempty"` + // Preference is optional. + Preference *kubevirtcorev1.PreferenceMatcher `json:"preference,omitempty"` + Template Template `json:"template,omitempty"` + DNSPolicy providerconfig.ConfigVarString `json:"dnsPolicy,omitempty"` + DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` + Location *Location `json:"location,omitempty"` + ProviderNetwork *ProviderNetwork `json:"providerNetwork,omitempty"` + EnableNetworkMultiQueue providerconfig.ConfigVarBool `json:"enableNetworkMultiQueue,omitempty"` + EvictionStrategy string `json:"evictionStrategy,omitempty"` +} + +// Flavor. +type Flavor struct { + Name providerconfig.ConfigVarString `json:"name,omitempty"` + Profile providerconfig.ConfigVarString `json:"profile,omitempty"` +} + +// Template. +type Template struct { + CPUs providerconfig.ConfigVarString `json:"cpus,omitempty"` + Memory providerconfig.ConfigVarString `json:"memory,omitempty"` + PrimaryDisk PrimaryDisk `json:"primaryDisk,omitempty"` + SecondaryDisks []SecondaryDisks `json:"secondaryDisks,omitempty"` +} + +// PrimaryDisk. +type PrimaryDisk struct { + Disk + // DataVolumeSecretRef is the name of the secret that will be sent to the CDI data importer pod to read basic auth parameters. + DataVolumeSecretRef providerconfig.ConfigVarString `json:"dataVolumeSecretRef,omitempty"` + // ExtraHeaders is a list of strings containing extra headers to include with HTTP transfer requests + // +optional + ExtraHeaders []string `json:"extraHeaders,omitempty"` + // ExtraHeadersSecretRef is a secret that contains a list of strings containing extra headers to include with HTTP transfer requests + // +optional + ExtraHeadersSecretRef providerconfig.ConfigVarString `json:"extraHeadersSecretRef,omitempty"` + // StorageTarget describes which VirtualMachine storage target will be used in the DataVolumeTemplate. + StorageTarget providerconfig.ConfigVarString `json:"storageTarget,omitempty"` + // OsImage describes the OS that will be installed on the VirtualMachine. + OsImage providerconfig.ConfigVarString `json:"osImage,omitempty"` + // Source describes the VM Disk Image source. + Source providerconfig.ConfigVarString `json:"source,omitempty"` + // PullMethod describes the VM Disk Image source optional pull method for registry source. Defaults to 'node'. + PullMethod providerconfig.ConfigVarString `json:"pullMethod,omitempty"` +} + +// SecondaryDisks. +type SecondaryDisks struct { + Disk +} + +// Disk. +type Disk struct { + Size providerconfig.ConfigVarString `json:"size,omitempty"` + StorageClassName providerconfig.ConfigVarString `json:"storageClassName,omitempty"` + StorageAccessType providerconfig.ConfigVarString `json:"storageAccessType,omitempty"` +} + +// Affinity. +type Affinity struct { + // Deprecated: Use TopologySpreadConstraint instead. + PodAffinityPreset providerconfig.ConfigVarString `json:"podAffinityPreset,omitempty"` + // Deprecated: Use TopologySpreadConstraint instead. + PodAntiAffinityPreset providerconfig.ConfigVarString `json:"podAntiAffinityPreset,omitempty"` + NodeAffinityPreset NodeAffinityPreset `json:"nodeAffinityPreset,omitempty"` +} + +// NodeAffinityPreset. +type NodeAffinityPreset struct { + Type providerconfig.ConfigVarString `json:"type,omitempty"` + Key providerconfig.ConfigVarString `json:"key,omitempty"` + Values []providerconfig.ConfigVarString `json:"values,omitempty"` +} + +// TopologySpreadConstraint describes topology spread constraints for VMs. +type TopologySpreadConstraint struct { + // MaxSkew describes the degree to which VMs may be unevenly distributed. + MaxSkew providerconfig.ConfigVarString `json:"maxSkew,omitempty"` + // TopologyKey is the key of infra-node labels. + TopologyKey providerconfig.ConfigVarString `json:"topologyKey,omitempty"` + // WhenUnsatisfiable indicates how to deal with a VM if it doesn't satisfy + // the spread constraint. + WhenUnsatisfiable providerconfig.ConfigVarString `json:"whenUnsatisfiable,omitempty"` +} + +// Location describes the region and zone where the machines are created at and where the deployed resources will reside. +type Location struct { + Region string `json:"region,omitempty"` + Zone string `json:"zone,omitempty"` +} + +// ProviderNetwork describes the infra cluster network fabric that is being used. +type ProviderNetwork struct { + Name string `json:"name"` + VPC VPC `json:"vpc"` +} + +// VPC is a virtual network dedicated to a single tenant within a KubeVirt, where the resources in the VPC +// is isolated from any other resources within the KubeVirt infra cluster. +type VPC struct { + Name string `json:"name"` + Subnet *Subnet `json:"subnet,omitempty"` +} + +// Subnet a smaller, segmented portion of a larger network, like a Virtual Private Cloud (VPC). +type Subnet struct { + Name string `json:"name"` +} + +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/linode/types/types.go b/sdk/cloudprovider/linode/types.go similarity index 52% rename from pkg/cloudprovider/provider/linode/types/types.go rename to sdk/cloudprovider/linode/types.go index 955a8e3da..bde06e4a1 100644 --- a/pkg/cloudprovider/provider/linode/types/types.go +++ b/sdk/cloudprovider/linode/types.go @@ -14,23 +14,23 @@ See the License for the specific language governing permissions and limitations under the License. */ -package types +package linode import ( - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" ) type RawConfig struct { - Token providerconfigtypes.ConfigVarString `json:"token,omitempty"` - Region providerconfigtypes.ConfigVarString `json:"region"` - Type providerconfigtypes.ConfigVarString `json:"type"` - Backups providerconfigtypes.ConfigVarBool `json:"backups"` - PrivateNetworking providerconfigtypes.ConfigVarBool `json:"private_networking"` - Tags []providerconfigtypes.ConfigVarString `json:"tags,omitempty"` + Token providerconfig.ConfigVarString `json:"token,omitempty"` + Region providerconfig.ConfigVarString `json:"region"` + Type providerconfig.ConfigVarString `json:"type"` + Backups providerconfig.ConfigVarBool `json:"backups"` + PrivateNetworking providerconfig.ConfigVarBool `json:"private_networking"` + Tags []providerconfig.ConfigVarString `json:"tags,omitempty"` } -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { rawConfig := &RawConfig{} return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) diff --git a/pkg/cloudprovider/provider/nutanix/types/types.go b/sdk/cloudprovider/nutanix/types.go similarity index 61% rename from pkg/cloudprovider/provider/nutanix/types/types.go rename to sdk/cloudprovider/nutanix/types.go index e5fe32bde..e46c34c71 100644 --- a/pkg/cloudprovider/provider/nutanix/types/types.go +++ b/sdk/cloudprovider/nutanix/types.go @@ -14,11 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -package types +package nutanix import ( - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" ) const ( @@ -31,18 +31,18 @@ const ( ) type RawConfig struct { - Endpoint providerconfigtypes.ConfigVarString `json:"endpoint"` - Port providerconfigtypes.ConfigVarString `json:"port"` - Username providerconfigtypes.ConfigVarString `json:"username"` - Password providerconfigtypes.ConfigVarString `json:"password"` - AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` - ProxyURL providerconfigtypes.ConfigVarString `json:"proxyURL,omitempty"` + Endpoint providerconfig.ConfigVarString `json:"endpoint"` + Port providerconfig.ConfigVarString `json:"port"` + Username providerconfig.ConfigVarString `json:"username"` + Password providerconfig.ConfigVarString `json:"password"` + AllowInsecure providerconfig.ConfigVarBool `json:"allowInsecure"` + ProxyURL providerconfig.ConfigVarString `json:"proxyURL,omitempty"` - ClusterName providerconfigtypes.ConfigVarString `json:"clusterName"` - ProjectName *providerconfigtypes.ConfigVarString `json:"projectName,omitempty"` - SubnetName providerconfigtypes.ConfigVarString `json:"subnetName"` - AdditionalSubnetNames []string `json:"additionalSubnetNames,omitempty"` - ImageName providerconfigtypes.ConfigVarString `json:"imageName"` + ClusterName providerconfig.ConfigVarString `json:"clusterName"` + ProjectName *providerconfig.ConfigVarString `json:"projectName,omitempty"` + SubnetName providerconfig.ConfigVarString `json:"subnetName"` + AdditionalSubnetNames []string `json:"additionalSubnetNames,omitempty"` + ImageName providerconfig.ConfigVarString `json:"imageName"` // VM sizing configuration CPUs int64 `json:"cpus"` @@ -68,7 +68,7 @@ type ErrorResponseMsg struct { Reason string `json:"reason"` } -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { rawConfig := &RawConfig{} return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) diff --git a/sdk/cloudprovider/opennebula/types.go b/sdk/cloudprovider/opennebula/types.go new file mode 100644 index 000000000..017a95004 --- /dev/null +++ b/sdk/cloudprovider/opennebula/types.go @@ -0,0 +1,46 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package opennebula + +import ( + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" +) + +type RawConfig struct { + // Auth details + Username providerconfig.ConfigVarString `json:"username,omitempty"` + Password providerconfig.ConfigVarString `json:"password,omitempty"` + Endpoint providerconfig.ConfigVarString `json:"endpoint,omitempty"` + + // Machine details + CPU *float64 `json:"cpu"` + VCPU *int `json:"vcpu"` + Memory *int `json:"memory"` + Image providerconfig.ConfigVarString `json:"image"` + Datastore providerconfig.ConfigVarString `json:"datastore"` + DiskSize *int `json:"diskSize"` + Network providerconfig.ConfigVarString `json:"network"` + EnableVNC providerconfig.ConfigVarBool `json:"enableVNC"` + VMTemplateExtra map[string]string `json:"vmTemplateExtra,omitempty"` +} + +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/sdk/cloudprovider/openstack/types.go b/sdk/cloudprovider/openstack/types.go new file mode 100644 index 000000000..27028857b --- /dev/null +++ b/sdk/cloudprovider/openstack/types.go @@ -0,0 +1,64 @@ +/* +Copyright 2019 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package openstack + +import ( + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" +) + +type RawConfig struct { + // Auth details + IdentityEndpoint providerconfig.ConfigVarString `json:"identityEndpoint,omitempty"` + Username providerconfig.ConfigVarString `json:"username,omitempty"` + Password providerconfig.ConfigVarString `json:"password,omitempty"` + ApplicationCredentialID providerconfig.ConfigVarString `json:"applicationCredentialID,omitempty"` + ApplicationCredentialSecret providerconfig.ConfigVarString `json:"applicationCredentialSecret,omitempty"` + DomainName providerconfig.ConfigVarString `json:"domainName,omitempty"` + ProjectName providerconfig.ConfigVarString `json:"projectName,omitempty"` + ProjectID providerconfig.ConfigVarString `json:"projectID,omitempty"` + TenantName providerconfig.ConfigVarString `json:"tenantName,omitempty"` + TenantID providerconfig.ConfigVarString `json:"tenantID,omitempty"` + TokenID providerconfig.ConfigVarString `json:"tokenId,omitempty"` + Region providerconfig.ConfigVarString `json:"region,omitempty"` + InstanceReadyCheckPeriod providerconfig.ConfigVarString `json:"instanceReadyCheckPeriod,omitempty"` + InstanceReadyCheckTimeout providerconfig.ConfigVarString `json:"instanceReadyCheckTimeout,omitempty"` + ComputeAPIVersion providerconfig.ConfigVarString `json:"computeAPIVersion,omitempty"` + + // Machine details + Image providerconfig.ConfigVarString `json:"image"` + Flavor providerconfig.ConfigVarString `json:"flavor"` + SecurityGroups []providerconfig.ConfigVarString `json:"securityGroups,omitempty"` + Network providerconfig.ConfigVarString `json:"network,omitempty"` + Subnet providerconfig.ConfigVarString `json:"subnet,omitempty"` + FloatingIPPool providerconfig.ConfigVarString `json:"floatingIpPool,omitempty"` + AvailabilityZone providerconfig.ConfigVarString `json:"availabilityZone,omitempty"` + TrustDevicePath providerconfig.ConfigVarBool `json:"trustDevicePath"` + RootDiskSizeGB *int `json:"rootDiskSizeGB"` + RootDiskVolumeType providerconfig.ConfigVarString `json:"rootDiskVolumeType,omitempty"` + NodeVolumeAttachLimit *uint `json:"nodeVolumeAttachLimit"` + ServerGroup providerconfig.ConfigVarString `json:"serverGroup"` + ConfigDrive providerconfig.ConfigVarBool `json:"configDrive,omitempty"` + // This tag is related to server metadata, not compute server's tag + Tags map[string]string `json:"tags,omitempty"` +} + +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/provider/scaleway/types/types.go b/sdk/cloudprovider/scaleway/types.go similarity index 51% rename from pkg/cloudprovider/provider/scaleway/types/types.go rename to sdk/cloudprovider/scaleway/types.go index 718f27f11..22624f543 100644 --- a/pkg/cloudprovider/provider/scaleway/types/types.go +++ b/sdk/cloudprovider/scaleway/types.go @@ -14,24 +14,24 @@ See the License for the specific language governing permissions and limitations under the License. */ -package types +package scaleway import ( - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" ) type RawConfig struct { - AccessKey providerconfigtypes.ConfigVarString `json:"accessKey,omitempty"` - SecretKey providerconfigtypes.ConfigVarString `json:"secretKey,omitempty"` - ProjectID providerconfigtypes.ConfigVarString `json:"projectId,omitempty"` - Zone providerconfigtypes.ConfigVarString `json:"zone,omitempty"` - CommercialType providerconfigtypes.ConfigVarString `json:"commercialType"` - IPv6 providerconfigtypes.ConfigVarBool `json:"ipv6"` - Tags []string `json:"tags,omitempty"` + AccessKey providerconfig.ConfigVarString `json:"accessKey,omitempty"` + SecretKey providerconfig.ConfigVarString `json:"secretKey,omitempty"` + ProjectID providerconfig.ConfigVarString `json:"projectId,omitempty"` + Zone providerconfig.ConfigVarString `json:"zone,omitempty"` + CommercialType providerconfig.ConfigVarString `json:"commercialType"` + IPv6 providerconfig.ConfigVarBool `json:"ipv6"` + Tags []string `json:"tags,omitempty"` } -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { rawConfig := &RawConfig{} return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go b/sdk/cloudprovider/vmwareclouddirector/types.go similarity index 57% rename from pkg/cloudprovider/provider/vmwareclouddirector/types/types.go rename to sdk/cloudprovider/vmwareclouddirector/types.go index fe60277d3..f09afbec9 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/types/types.go +++ b/sdk/cloudprovider/vmwareclouddirector/types.go @@ -14,11 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -package types +package vmwareclouddirector import ( - "k8c.io/machine-controller/pkg/jsonutil" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" ) type IPAllocationMode string @@ -31,23 +31,23 @@ const ( // RawConfig represents VMware Cloud Director specific configuration. type RawConfig struct { // Provider configuration. - Username providerconfigtypes.ConfigVarString `json:"username"` - Password providerconfigtypes.ConfigVarString `json:"password"` - APIToken providerconfigtypes.ConfigVarString `json:"apiToken"` - Organization providerconfigtypes.ConfigVarString `json:"organization"` - URL providerconfigtypes.ConfigVarString `json:"url"` - VDC providerconfigtypes.ConfigVarString `json:"vdc"` - AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` + Username providerconfig.ConfigVarString `json:"username"` + Password providerconfig.ConfigVarString `json:"password"` + APIToken providerconfig.ConfigVarString `json:"apiToken"` + Organization providerconfig.ConfigVarString `json:"organization"` + URL providerconfig.ConfigVarString `json:"url"` + VDC providerconfig.ConfigVarString `json:"vdc"` + AllowInsecure providerconfig.ConfigVarBool `json:"allowInsecure"` // VM configuration. - VApp providerconfigtypes.ConfigVarString `json:"vapp"` - Template providerconfigtypes.ConfigVarString `json:"template"` - Catalog providerconfigtypes.ConfigVarString `json:"catalog"` - PlacementPolicy *string `json:"placementPolicy,omitempty"` + VApp providerconfig.ConfigVarString `json:"vapp"` + Template providerconfig.ConfigVarString `json:"template"` + Catalog providerconfig.ConfigVarString `json:"catalog"` + PlacementPolicy *string `json:"placementPolicy,omitempty"` // Network configuration. - Network providerconfigtypes.ConfigVarString `json:"network"` - IPAllocationMode IPAllocationMode `json:"ipAllocationMode,omitempty"` + Network providerconfig.ConfigVarString `json:"network"` + IPAllocationMode IPAllocationMode `json:"ipAllocationMode,omitempty"` // Compute configuration. CPUs int64 `json:"cpus"` @@ -65,7 +65,7 @@ type RawConfig struct { Metadata *map[string]string `json:"metadata,omitempty"` } -func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { rawConfig := &RawConfig{} return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) diff --git a/sdk/cloudprovider/vsphere/types.go b/sdk/cloudprovider/vsphere/types.go new file mode 100644 index 000000000..c7c1f5f0f --- /dev/null +++ b/sdk/cloudprovider/vsphere/types.go @@ -0,0 +1,69 @@ +/* +Copyright 2019 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vsphere + +import ( + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" +) + +// RawConfig represents vsphere specific configuration. +type RawConfig struct { + TemplateVMName providerconfig.ConfigVarString `json:"templateVMName"` + // Deprecated: use networks instead. + VMNetName providerconfig.ConfigVarString `json:"vmNetName"` + Networks []providerconfig.ConfigVarString `json:"networks"` + Username providerconfig.ConfigVarString `json:"username"` + Password providerconfig.ConfigVarString `json:"password"` + VSphereURL providerconfig.ConfigVarString `json:"vsphereURL"` + Datacenter providerconfig.ConfigVarString `json:"datacenter"` + + // Cluster defines the cluster to use in vcenter. + // Only needed for vm anti affinity. + Cluster providerconfig.ConfigVarString `json:"cluster"` + + Folder providerconfig.ConfigVarString `json:"folder"` + ResourcePool providerconfig.ConfigVarString `json:"resourcePool"` + + // Either Datastore or DatastoreCluster have to be provided. + DatastoreCluster providerconfig.ConfigVarString `json:"datastoreCluster"` + Datastore providerconfig.ConfigVarString `json:"datastore"` + + CPUs int32 `json:"cpus"` + MemoryMB int64 `json:"memoryMB"` + DiskSizeGB *int64 `json:"diskSizeGB,omitempty"` + Tags []Tag `json:"tags,omitempty"` + AllowInsecure providerconfig.ConfigVarBool `json:"allowInsecure"` + + // Placement rules + VMAntiAffinity providerconfig.ConfigVarBool `json:"vmAntiAffinity"` + VMGroup providerconfig.ConfigVarString `json:"vmGroup,omitempty"` +} + +// Tag represents vsphere tag. +type Tag struct { + Description string `json:"description,omitempty"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + CategoryID string `json:"categoryID"` +} + +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/sdk/cloudprovider/vultr/types.go b/sdk/cloudprovider/vultr/types.go new file mode 100644 index 000000000..b108d99b0 --- /dev/null +++ b/sdk/cloudprovider/vultr/types.go @@ -0,0 +1,42 @@ +/* +Copyright 2023 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vultr + +import ( + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/providerconfig" +) + +type RawConfig struct { + PhysicalMachine bool `json:"physicalMachine,omitempty"` + APIKey providerconfig.ConfigVarString `json:"apiKey,omitempty"` + Region providerconfig.ConfigVarString `json:"region"` + Plan providerconfig.ConfigVarString `json:"plan"` + OsID providerconfig.ConfigVarString `json:"osId"` + Tags []string `json:"tags,omitempty"` + VpcID []string `json:"vpcId,omitempty"` + Vpc2ID []string `json:"vpc2Id,omitempty"` + EnableVPC bool `json:"enableVPC,omitempty"` + EnableVPC2 bool `json:"enableVPC2,omitempty"` + EnableIPv6 bool `json:"enableIPv6,omitempty"` +} + +func GetConfig(pconfig providerconfig.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/sdk/go.mod b/sdk/go.mod new file mode 100644 index 000000000..bd8687cac --- /dev/null +++ b/sdk/go.mod @@ -0,0 +1,64 @@ +module k8c.io/machine-controller/sdk + +go 1.22.3 + +toolchain go1.23.1 + +require ( + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 + k8s.io/api v0.31.1 + k8s.io/apimachinery v0.31.1 + k8s.io/client-go v0.31.1 + k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 + kubevirt.io/api v1.3.1 + sigs.k8s.io/controller-runtime v0.19.0 + sigs.k8s.io/yaml v1.4.0 +) + +require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/imdario/mergo v0.3.16 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/onsi/ginkgo/v2 v2.20.1 // indirect + github.com/onsi/gomega v1.34.1 // indirect + github.com/openshift/custom-resource-status v1.1.2 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect + golang.org/x/time v0.6.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.31.1 // indirect + k8s.io/klog/v2 v2.130.1 // indirect + k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect + kubevirt.io/containerized-data-importer-api v1.60.3 // indirect + kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect +) diff --git a/sdk/go.sum b/sdk/go.sum new file mode 100644 index 000000000..bd0c6b0b2 --- /dev/null +++ b/sdk/go.sum @@ -0,0 +1,382 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= +github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.20.1 h1:YlVIbqct+ZmnEph770q9Q7NVAz4wwIiVNahee6JyUzo= +github.com/onsi/ginkgo/v2 v2.20.1/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= +github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= +golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= +golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= +golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= +gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= +k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU= +k8s.io/api v0.31.1/go.mod h1:sbN1g6eY6XVLeqNsZGLnI5FwVseTrZX7Fv3O26rhAaI= +k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/N6E40= +k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ= +k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= +k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= +k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0= +k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg= +k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 h1:1dWzkmJrrprYvjGwh9kEUxmcUV/CtNU8QM7h1FLWQOo= +k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= +k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +kubevirt.io/api v1.3.1 h1:MoTNo/zvDlZ44c2ocXLPln8XTaQOeUodiYbEKrTCqv4= +kubevirt.io/api v1.3.1/go.mod h1:tCn7VAZktEvymk490iPSMPCmKM9UjbbfH2OsFR/IOLU= +kubevirt.io/containerized-data-importer-api v1.60.3 h1:kQEXi7scpzUa0RPf3/3MKk1Kmem0ZlqqiuK3kDF5L2I= +kubevirt.io/containerized-data-importer-api v1.60.3/go.mod h1:8mwrkZIdy8j/LmCyKt2wFXbiMavLUIqDaegaIF67CZs= +kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 h1:fZYvD3/Vnitfkx6IJxjLAk8ugnZQ7CXVYcRfkSKmuZY= +kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= +sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= +sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/pkg/test/helper.go b/sdk/internal/test/helper.go similarity index 100% rename from pkg/test/helper.go rename to sdk/internal/test/helper.go diff --git a/pkg/jsonutil/strict.go b/sdk/jsonutil/strict.go similarity index 100% rename from pkg/jsonutil/strict.go rename to sdk/jsonutil/strict.go diff --git a/sdk/net/net.go b/sdk/net/net.go new file mode 100644 index 000000000..fc39ab1e1 --- /dev/null +++ b/sdk/net/net.go @@ -0,0 +1,55 @@ +/* +Copyright 2021 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package net + +import ( + gonet "net" +) + +const ( + ErrIPv6OnlyUnsupported = "IPv6 only network family not supported yet" + ErrUnknownNetworkFamily = "Unknown IP family %q only IPv4,IPv6,IPv4+IPv6 are valid values" +) + +// IPFamily IPv4 | IPv6 | IPv4+IPv6. +type IPFamily string + +const ( + IPFamilyUnspecified IPFamily = "" // interpreted as IPv4 + IPFamilyIPv4 IPFamily = "IPv4" // IPv4 only + IPFamilyIPv6 IPFamily = "IPv6" // IPv6 only + IPFamilyIPv4IPv6 IPFamily = "IPv4+IPv6" // dualstack with IPv4 as primary + IPFamilyIPv6IPv4 IPFamily = "IPv6+IPv4" // dualstack with IPv6 as primary +) + +func (f IPFamily) HasIPv6() bool { + return f == IPFamilyIPv6 || f == IPFamilyIPv4IPv6 || f == IPFamilyIPv6IPv4 +} + +func (f IPFamily) HasIPv4() bool { + return f == IPFamilyUnspecified || f == IPFamilyIPv4 || f == IPFamilyIPv4IPv6 || f == IPFamilyIPv6IPv4 +} + +func (f IPFamily) IsDualstack() bool { + return f == IPFamilyIPv4IPv6 || f == IPFamilyIPv6IPv4 +} + +// IsLinkLocal checks if given ip address is link local.. +func IsLinkLocal(ipAddr string) bool { + addr := gonet.ParseIP(ipAddr) + return addr.IsLinkLocalMulticast() || addr.IsLinkLocalUnicast() +} diff --git a/pkg/node/eviction/types/types.go b/sdk/node/eviction.go similarity index 97% rename from pkg/node/eviction/types/types.go rename to sdk/node/eviction.go index d69a4aa39..9de6e49cc 100644 --- a/pkg/node/eviction/types/types.go +++ b/sdk/node/eviction.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package types +package node const ( SkipEvictionAnnotationKey = "kubermatic.io/skip-eviction" diff --git a/pkg/providerconfig/types.go b/sdk/providerconfig/resolver.go similarity index 79% rename from pkg/providerconfig/types.go rename to sdk/providerconfig/resolver.go index c88f6df06..fe6805ff6 100644 --- a/pkg/providerconfig/types.go +++ b/sdk/providerconfig/resolver.go @@ -18,21 +18,12 @@ package providerconfig import ( "context" - "errors" "fmt" "os" "strconv" "time" - providerconfigtypes "k8c.io/machine-controller/pkg/providerconfig/types" - "k8c.io/machine-controller/pkg/userdata/amzn2" - "k8c.io/machine-controller/pkg/userdata/flatcar" - "k8c.io/machine-controller/pkg/userdata/rhel" - "k8c.io/machine-controller/pkg/userdata/rockylinux" - "k8c.io/machine-controller/pkg/userdata/ubuntu" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -42,7 +33,14 @@ type ConfigVarResolver struct { client ctrlruntimeclient.Client } -func (cvr *ConfigVarResolver) GetConfigVarDurationValue(configVar providerconfigtypes.ConfigVarString) (time.Duration, error) { +func NewConfigVarResolver(ctx context.Context, client ctrlruntimeclient.Client) *ConfigVarResolver { + return &ConfigVarResolver{ + ctx: ctx, + client: client, + } +} + +func (cvr *ConfigVarResolver) GetConfigVarDurationValue(configVar ConfigVarString) (time.Duration, error) { durStr, err := cvr.GetConfigVarStringValue(configVar) if err != nil { return 0, err @@ -51,7 +49,7 @@ func (cvr *ConfigVarResolver) GetConfigVarDurationValue(configVar providerconfig return time.ParseDuration(durStr) } -func (cvr *ConfigVarResolver) GetConfigVarDurationValueOrDefault(configVar providerconfigtypes.ConfigVarString, defaultDuration time.Duration) (time.Duration, error) { +func (cvr *ConfigVarResolver) GetConfigVarDurationValueOrDefault(configVar ConfigVarString, defaultDuration time.Duration) (time.Duration, error) { durStr, err := cvr.GetConfigVarStringValue(configVar) if err != nil { return 0, err @@ -73,7 +71,7 @@ func (cvr *ConfigVarResolver) GetConfigVarDurationValueOrDefault(configVar provi return duration, nil } -func (cvr *ConfigVarResolver) GetConfigVarStringValue(configVar providerconfigtypes.ConfigVarString) (string, error) { +func (cvr *ConfigVarResolver) GetConfigVarStringValue(configVar ConfigVarString) (string, error) { // We need all three of these to fetch and use a secret if configVar.SecretKeyRef.Name != "" && configVar.SecretKeyRef.Namespace != "" && configVar.SecretKeyRef.Key != "" { secret := &corev1.Secret{} @@ -105,7 +103,7 @@ func (cvr *ConfigVarResolver) GetConfigVarStringValue(configVar providerconfigty // GetConfigVarStringValueOrEnv tries to get the value from ConfigVarString, when it fails, it falls back to // getting the value from an environment variable specified by envVarName parameter. -func (cvr *ConfigVarResolver) GetConfigVarStringValueOrEnv(configVar providerconfigtypes.ConfigVarString, envVarName string) (string, error) { +func (cvr *ConfigVarResolver) GetConfigVarStringValueOrEnv(configVar ConfigVarString, envVarName string) (string, error) { cfgVar, err := cvr.GetConfigVarStringValue(configVar) if err == nil && len(cfgVar) > 0 { return cfgVar, err @@ -117,7 +115,7 @@ func (cvr *ConfigVarResolver) GetConfigVarStringValueOrEnv(configVar providercon // GetConfigVarBoolValue returns a boolean from a ConfigVarBool. If there is no valid source for the boolean, // the second bool returned will be false (to be able to differentiate between "false" and "unset"). -func (cvr *ConfigVarResolver) GetConfigVarBoolValue(configVar providerconfigtypes.ConfigVarBool) (bool, bool, error) { +func (cvr *ConfigVarResolver) GetConfigVarBoolValue(configVar ConfigVarBool) (bool, bool, error) { // We need all three of these to fetch and use a secret if configVar.SecretKeyRef.Name != "" && configVar.SecretKeyRef.Namespace != "" && configVar.SecretKeyRef.Key != "" { secret := &corev1.Secret{} @@ -153,7 +151,7 @@ func (cvr *ConfigVarResolver) GetConfigVarBoolValue(configVar providerconfigtype return configVar.Value != nil && *configVar.Value, true, nil } -func (cvr *ConfigVarResolver) GetConfigVarBoolValueOrEnv(configVar providerconfigtypes.ConfigVarBool, envVarName string) (bool, error) { +func (cvr *ConfigVarResolver) GetConfigVarBoolValueOrEnv(configVar ConfigVarBool, envVarName string) (bool, error) { boolVal, valid, err := cvr.GetConfigVarBoolValue(configVar) if valid && err == nil { return boolVal, nil @@ -170,30 +168,3 @@ func (cvr *ConfigVarResolver) GetConfigVarBoolValueOrEnv(configVar providerconfi return false, nil } - -func NewConfigVarResolver(ctx context.Context, client ctrlruntimeclient.Client) *ConfigVarResolver { - return &ConfigVarResolver{ - ctx: ctx, - client: client, - } -} - -func DefaultOperatingSystemSpec( - osys providerconfigtypes.OperatingSystem, - operatingSystemSpec runtime.RawExtension, -) (runtime.RawExtension, error) { - switch osys { - case providerconfigtypes.OperatingSystemAmazonLinux2: - return amzn2.DefaultConfig(operatingSystemSpec), nil - case providerconfigtypes.OperatingSystemFlatcar: - return flatcar.DefaultConfig(operatingSystemSpec), nil - case providerconfigtypes.OperatingSystemRHEL: - return rhel.DefaultConfig(operatingSystemSpec), nil - case providerconfigtypes.OperatingSystemUbuntu: - return ubuntu.DefaultConfig(operatingSystemSpec), nil - case providerconfigtypes.OperatingSystemRockyLinux: - return rockylinux.DefaultConfig(operatingSystemSpec), nil - } - - return operatingSystemSpec, errors.New("unknown OperatingSystem") -} diff --git a/pkg/providerconfig/types/types.go b/sdk/providerconfig/types.go similarity index 96% rename from pkg/providerconfig/types/types.go rename to sdk/providerconfig/types.go index b733f0476..888b4744a 100644 --- a/pkg/providerconfig/types/types.go +++ b/sdk/providerconfig/types.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package types +package providerconfig import ( "bytes" @@ -23,9 +23,9 @@ import ( "fmt" "strconv" - clusterv1alpha1 "k8c.io/machine-controller/pkg/apis/cluster/v1alpha1" - "k8c.io/machine-controller/pkg/cloudprovider/util" - "k8c.io/machine-controller/pkg/jsonutil" + clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" + "k8c.io/machine-controller/sdk/jsonutil" + "k8c.io/machine-controller/sdk/net" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" @@ -131,10 +131,10 @@ type DNSConfig struct { // NetworkConfig contains a machine's static network configuration. type NetworkConfig struct { - CIDR string `json:"cidr"` - Gateway string `json:"gateway"` - DNS DNSConfig `json:"dns"` - IPFamily util.IPFamily `json:"ipFamily,omitempty"` + CIDR string `json:"cidr"` + Gateway string `json:"gateway"` + DNS DNSConfig `json:"dns"` + IPFamily net.IPFamily `json:"ipFamily,omitempty"` } func (n *NetworkConfig) IsStaticIPConfig() bool { @@ -146,9 +146,9 @@ func (n *NetworkConfig) IsStaticIPConfig() bool { len(n.DNS.Servers) != 0 } -func (n *NetworkConfig) GetIPFamily() util.IPFamily { +func (n *NetworkConfig) GetIPFamily() net.IPFamily { if n == nil { - return util.IPFamilyUnspecified + return net.IPFamilyUnspecified } return n.IPFamily } diff --git a/pkg/providerconfig/types/types_test.go b/sdk/providerconfig/types_test.go similarity index 72% rename from pkg/providerconfig/types/types_test.go rename to sdk/providerconfig/types_test.go index 247b4e48a..52432ad2f 100644 --- a/pkg/providerconfig/types/types_test.go +++ b/sdk/providerconfig/types_test.go @@ -14,14 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -package types +package providerconfig import ( "encoding/json" "reflect" "testing" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" ) @@ -68,19 +68,19 @@ func TestConfigVarBoolUnmarshalling(t *testing.T) { }, { jsonString: `{"secretKeyRef":{"namespace":"ns","name":"name","key":"key"}}`, - expected: ConfigVarBool{Value: nil, SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, + expected: ConfigVarBool{Value: nil, SecretKeyRef: GlobalSecretKeySelector{ObjectReference: corev1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, }, { jsonString: `{"value": null, "secretKeyRef":{"namespace":"ns","name":"name","key":"key"}}`, - expected: ConfigVarBool{Value: nil, SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, + expected: ConfigVarBool{Value: nil, SecretKeyRef: GlobalSecretKeySelector{ObjectReference: corev1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, }, { jsonString: `{"value":false, "secretKeyRef":{"namespace":"ns","name":"name","key":"key"}}`, - expected: ConfigVarBool{Value: ptr.To(false), SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, + expected: ConfigVarBool{Value: ptr.To(false), SecretKeyRef: GlobalSecretKeySelector{ObjectReference: corev1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, }, { jsonString: `{"value":true, "secretKeyRef":{"namespace":"ns","name":"name","key":"key"}}`, - expected: ConfigVarBool{Value: ptr.To(true), SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, + expected: ConfigVarBool{Value: ptr.To(true), SecretKeyRef: GlobalSecretKeySelector{ObjectReference: corev1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, }, } @@ -103,7 +103,7 @@ func TestConfigVarStringMarshalling(t *testing.T) { expected: `"val"`, }, { - cvs: ConfigVarString{SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, + cvs: ConfigVarString{SecretKeyRef: GlobalSecretKeySelector{ObjectReference: corev1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, expected: `{"secretKeyRef":{"namespace":"ns","name":"name","key":"key"}}`, }, } @@ -133,15 +133,15 @@ func TestConfigVarBoolMarshalling(t *testing.T) { expected: `true`, }, { - cvb: ConfigVarBool{SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, + cvb: ConfigVarBool{SecretKeyRef: GlobalSecretKeySelector{ObjectReference: corev1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}}, expected: `{"secretKeyRef":{"namespace":"ns","name":"name","key":"key"}}`, }, { - cvb: ConfigVarBool{SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, Value: ptr.To(true)}, + cvb: ConfigVarBool{SecretKeyRef: GlobalSecretKeySelector{ObjectReference: corev1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, Value: ptr.To(true)}, expected: `{"secretKeyRef":{"namespace":"ns","name":"name","key":"key"},"value":true}`, }, { - cvb: ConfigVarBool{SecretKeyRef: GlobalSecretKeySelector{ObjectReference: v1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, Value: ptr.To(false)}, + cvb: ConfigVarBool{SecretKeyRef: GlobalSecretKeySelector{ObjectReference: corev1.ObjectReference{Namespace: "ns", Name: "name"}, Key: "key"}, Value: ptr.To(false)}, expected: `{"secretKeyRef":{"namespace":"ns","name":"name","key":"key"},"value":false}`, }, } @@ -161,22 +161,22 @@ func TestConfigVarStringMarshallingAndUnmarshalling(t *testing.T) { testCases := []ConfigVarString{ {Value: "val"}, {Value: "spe Date: Fri, 28 Feb 2025 19:29:11 +0100 Subject: [PATCH 459/489] Synchronize OWNERS_ALIASES file with Github teams (#1901) --- OWNERS_ALIASES | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 93b0fc0ac..6161372db 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -10,6 +10,7 @@ aliases: - kron4eg - moadqassem - moelsayed + - mohamed-rafraf - soer3n - xmudrii - xrstf From b5932c113352956608503846e659f5c1ab49ffc7 Mon Sep 17 00:00:00 2001 From: Kubermatic Bot <41968677+kubermatic-bot@users.noreply.github.com> Date: Mon, 3 Mar 2025 11:28:15 +0100 Subject: [PATCH 460/489] Synchronize OWNERS_ALIASES file with Github teams (#1904) --- OWNERS_ALIASES | 1 - 1 file changed, 1 deletion(-) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 6161372db..35764dfbc 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -14,4 +14,3 @@ aliases: - soer3n - xmudrii - xrstf - - yaa110 From 1427edda26a07e6069bd32f1ea46c65ec20f64f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 12:07:13 +0100 Subject: [PATCH 461/489] Bump golang.org/x/net from 0.29.0 to 0.33.0 (#1903) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.29.0 to 0.33.0. - [Commits](https://github.com/golang/net/compare/v0.29.0...v0.33.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 308b0b177..8ba164f6a 100644 --- a/go.mod +++ b/go.mod @@ -165,7 +165,7 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/net v0.29.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect diff --git a/go.sum b/go.sum index 63c5a8375..eb3142053 100644 --- a/go.sum +++ b/go.sum @@ -548,8 +548,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= From 99a4aa5532caaf58aa9528a484f51aa723c098d5 Mon Sep 17 00:00:00 2001 From: soer3n <43064202+soer3n@users.noreply.github.com> Date: Fri, 14 Mar 2025 16:03:30 +0100 Subject: [PATCH 462/489] Change KubeVirt VM CPU assignment to not overwrite cpu alloc ratio (#1906) * change kubevirt vm cpu assignment to not overwrite cpu alloc ratio Signed-off-by: soer3n * update kubevirt testdata Signed-off-by: soer3n * revert testdata change and add condition for vm cpu assignment Signed-off-by: soer3n * switch to providerSpec for enabling vcpu assignment Signed-off-by: soer3n * adapt kubevirt cpu struct * adapt kubevirt cpu struct for configuring vcpus for a virtual machine * modify logic in function for rendering resource requests and limits * modify validation accordingly to be a bit more specific regarding resources Signed-off-by: soer3n * revert unnessecarry changes to mocked kubevirt vm Signed-off-by: soer3n * changes after review Signed-off-by: soer3n --------- Signed-off-by: soer3n --- .../provider/kubevirt/provider.go | 54 +++++++++---- .../provider/kubevirt/provider_test.go | 11 +++ .../kubevirt/testdata/dedicated-vcpus.yaml | 80 +++++++++++++++++++ sdk/cloudprovider/kubevirt/types.go | 11 +++ 4 files changed, 142 insertions(+), 14 deletions(-) create mode 100644 pkg/cloudprovider/provider/kubevirt/testdata/dedicated-vcpus.yaml diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 8b8f1b616..78b14f034 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -98,6 +98,8 @@ type Config struct { DNSConfig *corev1.PodDNSConfig DNSPolicy corev1.DNSPolicy CPUs string + VCPUs *kubevirtcorev1.CPU + Resources *corev1.ResourceList Memory string Namespace string OSImageSource *cdicorev1beta1.DataVolumeSource @@ -273,14 +275,21 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf("failed to decode kubeconfig: %w", err) } - config.CPUs, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.CPUs) + cpus, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.CPUs) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "cpus" field: %w`, err) } - config.Memory, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.Memory) + + memory, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.Memory) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "memory" field: %w`, err) } + + config.Resources, config.VCPUs, err = parseResources(cpus, memory, rawConfig.VirtualMachine.Template.VCPUs) + if err != nil { + return nil, nil, fmt.Errorf(`failed to configure resource requests and limits and vcpus: %w`, err) + } + config.Namespace = getNamespace() if len(rawConfig.VirtualMachine.Template.PrimaryDisk.ExtraHeaders) > 0 { config.ExtraHeaders = rawConfig.VirtualMachine.Template.PrimaryDisk.ExtraHeaders @@ -616,8 +625,16 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus // If instancetype is specified, skip CPU and Memory validation. // Values will come from instancetype. if c.Instancetype == nil { - if _, err := parseResources(c.CPUs, c.Memory); err != nil { - return err + if c.Resources == nil { + return fmt.Errorf("no resource requests set for the virtual machine") + } + + if c.VCPUs == nil && c.Resources.Cpu() == nil { + return fmt.Errorf("no CPUs configured. Either vCPUs or CPUs have to be set.") + } + + if c.VCPUs != nil && c.Resources.Cpu() != nil { + return fmt.Errorf("vCPUs and CPUs cannot be configured at the same time.") } } @@ -753,12 +770,8 @@ func (p *provider) newVirtualMachine(c *Config, pc *providerconfig.Config, machi // if no instancetype, resources are from config. if c.Instancetype == nil { - requestsAndLimits, err := parseResources(c.CPUs, c.Memory) - if err != nil { - return nil, err - } - resourceRequirements.Requests = *requestsAndLimits - resourceRequirements.Limits = *requestsAndLimits + resourceRequirements.Requests = *c.Resources + resourceRequirements.Limits = *c.Resources } // Add cluster labels @@ -840,6 +853,13 @@ func (p *provider) newVirtualMachine(c *Config, pc *providerconfig.Config, machi DataVolumeTemplates: getDataVolumeTemplates(c, dataVolumeName, dvAnnotations), }, } + + if c.VCPUs != nil { + virtualMachine.Spec.Template.Spec.Domain.CPU = &kubevirtcorev1.CPU{ + Cores: c.VCPUs.Cores, + } + } + return virtualMachine, nil } @@ -867,19 +887,25 @@ func (p *provider) Cleanup(ctx context.Context, _ *zap.SugaredLogger, machine *c return false, sigClient.Delete(ctx, vm) } -func parseResources(cpus, memory string) (*corev1.ResourceList, error) { +func parseResources(cpus, memory string, vpcus kubevirttypes.VCPUs) (*corev1.ResourceList, *kubevirtcorev1.CPU, error) { memoryResource, err := resource.ParseQuantity(memory) if err != nil { - return nil, fmt.Errorf("failed to parse memory requests: %w", err) + return nil, nil, fmt.Errorf("failed to parse memory requests: %w", err) } + + if vpcus.Cores != 0 { + return &corev1.ResourceList{corev1.ResourceMemory: memoryResource}, &kubevirtcorev1.CPU{Cores: uint32(vpcus.Cores)}, nil + } + cpuResource, err := resource.ParseQuantity(cpus) if err != nil { - return nil, fmt.Errorf("failed to parse cpu request: %w", err) + return nil, nil, fmt.Errorf("failed to parse cpu requests: %w", err) } + return &corev1.ResourceList{ corev1.ResourceMemory: memoryResource, corev1.ResourceCPU: cpuResource, - }, nil + }, nil, nil } func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index 6647d84ff..956a88bcd 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -72,6 +72,7 @@ type kubevirtProviderSpecConf struct { ProviderNetwork *kubevirt.ProviderNetwork ExtraHeadersSet bool EvictStrategy string + VCPUs uint32 } func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { @@ -132,7 +133,13 @@ func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { }, {{- end }} "template": { + {{- if .VCPUs }} + "vcpus": { + "cores": {{ .VCPUs }} + }, + {{- else }} "cpus": "2", + {{- end }} "memory": "2Gi", {{- if .SecondaryDisks }} "secondaryDisks": [{ @@ -283,6 +290,10 @@ func TestNewVirtualMachine(t *testing.T) { name: "eviction-strategy-live-migrate", specConf: kubevirtProviderSpecConf{EvictStrategy: "LiveMigrate"}, }, + { + name: "dedicated-vcpus", + specConf: kubevirtProviderSpecConf{VCPUs: 2}, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/dedicated-vcpus.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/dedicated-vcpus.yaml new file mode 100644 index 000000000..daae69a66 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/dedicated-vcpus.yaml @@ -0,0 +1,80 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: dedicated-vcpus + md: md-name + name: dedicated-vcpus + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: dedicated-vcpus + spec: + storage: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + runStrategy: Once + template: + metadata: + creationTimestamp: null + annotations: + "kubevirt.io/allow-pod-bridge-network-live-migration": "true" + "ovn.kubernetes.io/allow_live_migration": "true" + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: dedicated-vcpus + md: md-name + spec: + affinity: {} + domain: + cpu: + cores: 2 + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - name: default + bridge: {} + networkInterfaceMultiqueue: true + resources: + limits: + memory: 2Gi + requests: + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: dedicated-vcpus + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External diff --git a/sdk/cloudprovider/kubevirt/types.go b/sdk/cloudprovider/kubevirt/types.go index cd90b07ba..7025990a0 100644 --- a/sdk/cloudprovider/kubevirt/types.go +++ b/sdk/cloudprovider/kubevirt/types.go @@ -70,12 +70,23 @@ type Flavor struct { // Template. type Template struct { + // VCPUs is to configure vcpus used by a the virtual machine + // when using kubevirts cpuAllocationRatio feature this leads to auto assignment of the + // calculated ratio as resource cpu requests for the pod which launches the virtual machine + VCPUs VCPUs `json:"vcpus,omitempty"` + // CPUs is to configure cpu requests and limits directly for the pod which launches the virtual machine + // and is related to the underlying hardware CPUs providerconfig.ConfigVarString `json:"cpus,omitempty"` Memory providerconfig.ConfigVarString `json:"memory,omitempty"` PrimaryDisk PrimaryDisk `json:"primaryDisk,omitempty"` SecondaryDisks []SecondaryDisks `json:"secondaryDisks,omitempty"` } +// VCPUs. +type VCPUs struct { + Cores int `json:"cores,omitempty"` +} + // PrimaryDisk. type PrimaryDisk struct { Disk From 7b6ae89532d1ef525806fdba37d80801b607d0d0 Mon Sep 17 00:00:00 2001 From: soer3n <43064202+soer3n@users.noreply.github.com> Date: Tue, 1 Apr 2025 15:04:05 +0200 Subject: [PATCH 463/489] fix kubevirt cpu check + update sdk version in go.mod (#1912) * fix kubevirt cpu check + update sdk version in go.mod Signed-off-by: soer3n * revert change regarding import of sdk submodule Signed-off-by: soer3n * kubevirt resources and vcpus should only be parsed wihtout specified instance type Signed-off-by: soer3n --------- Signed-off-by: soer3n --- pkg/cloudprovider/provider/kubevirt/provider.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 78b14f034..fd623ebb8 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -285,9 +285,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf(`failed to get value of "memory" field: %w`, err) } - config.Resources, config.VCPUs, err = parseResources(cpus, memory, rawConfig.VirtualMachine.Template.VCPUs) - if err != nil { - return nil, nil, fmt.Errorf(`failed to configure resource requests and limits and vcpus: %w`, err) + if rawConfig.VirtualMachine.Instancetype == nil { + config.Resources, config.VCPUs, err = parseResources(cpus, memory, rawConfig.VirtualMachine.Template.VCPUs) + if err != nil { + return nil, nil, fmt.Errorf(`failed to configure resource requests and limits and vcpus: %w`, err) + } } config.Namespace = getNamespace() @@ -629,11 +631,11 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus return fmt.Errorf("no resource requests set for the virtual machine") } - if c.VCPUs == nil && c.Resources.Cpu() == nil { + if c.VCPUs == nil && c.Resources.Cpu().IsZero() { return fmt.Errorf("no CPUs configured. Either vCPUs or CPUs have to be set.") } - if c.VCPUs != nil && c.Resources.Cpu() != nil { + if c.VCPUs != nil && !c.Resources.Cpu().IsZero() { return fmt.Errorf("vCPUs and CPUs cannot be configured at the same time.") } } From ed388a8e9fc51948d557bb6a996bd64829434773 Mon Sep 17 00:00:00 2001 From: Archana Sawant Date: Thu, 3 Apr 2025 13:31:06 +0530 Subject: [PATCH 464/489] Bump Go version to 1.23.7 (#1913) Signed-off-by: archups --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 12 ++++++------ .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 10 +++++----- .prow/verify.yaml | 16 ++++++++-------- Dockerfile | 2 +- Makefile | 2 +- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- 22 files changed, 45 insertions(+), 45 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index d958d6b2b..f9304adc7 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -35,7 +35,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -65,7 +65,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -95,7 +95,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -123,7 +123,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 6f772d882..3cb4f79b7 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -27,7 +27,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - /bin/bash - -c @@ -56,7 +56,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index 4139c0186..a309a8abe 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 6f02370e7..5e03de204 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -32,7 +32,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index d30e4de2d..b6db901d0 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -96,7 +96,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -130,7 +130,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -162,7 +162,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -194,7 +194,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 0c7d66053..3609bdc69 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -96,7 +96,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 9a135b35b..00bb8c572 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index d14dd0120..e127c6d88 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index ce0ea7650..a1e22dbf3 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 0b4f55dc6..39d0b0b8f 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 5fc091859..effd294f2 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index f9eba462b..df54861d5 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 15b9f2f66..1c5a9e83d 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 97102b6cd..4976b1a32 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -64,7 +64,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 960019489..870680c68 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 216993262..0c8cfccdc 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -32,7 +32,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index ee71eac20..96a4b5b86 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -95,7 +95,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -128,7 +128,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -161,7 +161,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.24-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 04c9cfcbb..6fcb09b28 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -22,7 +22,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-10 command: - make args: @@ -44,7 +44,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-10 command: - make args: @@ -66,7 +66,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-10 command: - make args: @@ -87,7 +87,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-10 command: - make args: @@ -107,7 +107,7 @@ presubmits: path_alias: k8c.io/machine-controller spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-10 command: - "/usr/local/bin/shfmt" args: @@ -136,7 +136,7 @@ presubmits: path_alias: k8c.io/machine-controller spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-10 command: - "./hack/verify-boilerplate.sh" resources: @@ -156,7 +156,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-10 command: - ./hack/verify-licenses.sh resources: @@ -173,7 +173,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-4 + - image: quay.io/kubermatic/build:go-1.23-node-20-10 command: - make args: diff --git a/Dockerfile b/Dockerfile index 0a1bdd85e..7186070b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.23.2 +ARG GO_VERSION=1.23.7 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/k8c.io/machine-controller COPY . . diff --git a/Makefile b/Makefile index 396c9f8e4..881a4013a 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.23.2 +GO_VERSION ?= 1.23.7 GOOS ?= $(shell go env GOOS) diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index b0fa24681..38b8782f6 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.23-node-20-4 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.23-node-20-10 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 778ba0bd8..66b1f5741 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.23-node-20-4 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.23-node-20-10 containerize ./hack/verify-licenses.sh go mod vendor From 30362bbcac03e5c11ebc26e40b632b93bae8a70b Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Fri, 4 Apr 2025 13:50:08 +0200 Subject: [PATCH 465/489] Improve SDK, add readme (#1915) * move configvarresolver into standalone package to lessen ctrl-runtime dependency further * add readme * fix anexia tests --- pkg/admission/machines.go | 5 +- pkg/cloudprovider/provider.go | 50 ++--- .../provider/alibaba/provider.go | 22 +-- pkg/cloudprovider/provider/anexia/provider.go | 4 +- .../provider/anexia/provider_test.go | 6 +- .../provider/anexia/resolve_config.go | 18 +- pkg/cloudprovider/provider/aws/provider.go | 38 ++-- pkg/cloudprovider/provider/azure/provider.go | 36 ++-- .../provider/baremetal/provider.go | 8 +- .../provider/digitalocean/provider.go | 20 +- pkg/cloudprovider/provider/edge/provider.go | 2 +- .../provider/equinixmetal/provider.go | 22 +-- .../provider/external/provider.go | 2 +- pkg/cloudprovider/provider/fake/provider.go | 2 +- pkg/cloudprovider/provider/gce/config.go | 36 ++-- pkg/cloudprovider/provider/gce/provider.go | 4 +- .../provider/gce/provider_test.go | 4 +- .../provider/hetzner/provider.go | 24 +-- .../provider/kubevirt/provider.go | 52 +++--- .../provider/kubevirt/provider_test.go | 4 +- pkg/cloudprovider/provider/linode/provider.go | 16 +- .../provider/nutanix/provider.go | 24 +-- .../provider/opennebula/provider.go | 18 +- .../provider/openstack/provider.go | 56 +++--- .../provider/openstack/provider_test.go | 6 +- .../provider/scaleway/provider.go | 16 +- .../provider/vmwareclouddirector/provider.go | 26 +-- .../provider/vsphere/provider.go | 34 ++-- .../provider/vsphere/provider_test.go | 4 +- pkg/cloudprovider/provider/vultr/provider.go | 12 +- pkg/controller/machine/controller.go | 5 +- pkg/controller/machine/metrics.go | 9 +- pkg/migrations/migrations.go | 5 +- sdk/README.md | 41 +++++ sdk/providerconfig/configvar/resolver.go | 173 ++++++++++++++++++ sdk/providerconfig/resolver.go | 154 +--------------- test/e2e/provisioning/migrateuidscenario.go | 5 +- 37 files changed, 522 insertions(+), 441 deletions(-) create mode 100644 sdk/README.md create mode 100644 sdk/providerconfig/configvar/resolver.go diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index 2975877db..62cf06f0b 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -28,6 +28,7 @@ import ( "k8c.io/machine-controller/sdk/apis/cluster/common" clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" "k8c.io/machine-controller/sdk/providerconfig" + "k8c.io/machine-controller/sdk/providerconfig/configvar" "k8c.io/machine-controller/sdk/userdata" admissionv1 "k8s.io/api/admission/v1" @@ -129,8 +130,8 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec } } - skg := providerconfig.NewConfigVarResolver(ctx, ad.workerClient) - prov, err := cloudprovider.ForProvider(providerConfig.CloudProvider, skg) + configResolver := configvar.NewResolver(ctx, ad.workerClient) + prov, err := cloudprovider.ForProvider(providerConfig.CloudProvider, configResolver) if err != nil { return fmt.Errorf("failed to get cloud provider %q: %w", providerConfig.CloudProvider, err) } diff --git a/pkg/cloudprovider/provider.go b/pkg/cloudprovider/provider.go index afa5eede4..c334b3a84 100644 --- a/pkg/cloudprovider/provider.go +++ b/pkg/cloudprovider/provider.go @@ -51,88 +51,88 @@ var ( // ErrProviderNotFound tells that the requested cloud provider was not found. ErrProviderNotFound = errors.New("cloudprovider not found") - providers = map[providerconfig.CloudProvider]func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider{ - providerconfig.CloudProviderDigitalocean: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providers = map[providerconfig.CloudProvider]func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider{ + providerconfig.CloudProviderDigitalocean: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return digitalocean.New(cvr) }, - providerconfig.CloudProviderAWS: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderAWS: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return aws.New(cvr) }, - providerconfig.CloudProviderOpenstack: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderOpenstack: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return openstack.New(cvr) }, - providerconfig.CloudProviderGoogle: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderGoogle: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return gce.New(cvr) }, - providerconfig.CloudProviderHetzner: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderHetzner: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return hetzner.New(cvr) }, - providerconfig.CloudProviderVsphere: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderVsphere: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return vsphere.New(cvr) }, - providerconfig.CloudProviderAzure: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderAzure: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return azure.New(cvr) }, - providerconfig.CloudProviderEquinixMetal: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderEquinixMetal: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return equinixmetal.New(cvr) }, // NB: This is explicitly left to allow old Packet machines to be deleted. // We can handle those machines in the same way as Equinix Metal machines // because there are no API changes. // TODO: Remove this after deprecation period. - providerconfig.CloudProviderPacket: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderPacket: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return equinixmetal.New(cvr) }, - providerconfig.CloudProviderFake: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderFake: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return fake.New(cvr) }, - providerconfig.CloudProviderEdge: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderEdge: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return edge.New(cvr) }, - providerconfig.CloudProviderKubeVirt: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderKubeVirt: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return kubevirt.New(cvr) }, - providerconfig.CloudProviderAlibaba: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderAlibaba: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return alibaba.New(cvr) }, - providerconfig.CloudProviderScaleway: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderScaleway: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return scaleway.New(cvr) }, - providerconfig.CloudProviderAnexia: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderAnexia: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return anexia.New(cvr) }, - providerconfig.CloudProviderBaremetal: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderBaremetal: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { // TODO(MQ): add a baremetal driver. return baremetal.New(cvr) }, - providerconfig.CloudProviderNutanix: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderNutanix: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return nutanix.New(cvr) }, - providerconfig.CloudProviderVMwareCloudDirector: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderVMwareCloudDirector: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return vcd.New(cvr) }, - providerconfig.CloudProviderExternal: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderExternal: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return external.New(cvr) }, } // communityProviders holds a map of cloud providers that have been implemented by community members and // contributed to machine-controller. They are not end-to-end tested by the machine-controller development team. - communityProviders = map[providerconfig.CloudProvider]func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider{ - providerconfig.CloudProviderLinode: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + communityProviders = map[providerconfig.CloudProvider]func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider{ + providerconfig.CloudProviderLinode: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return linode.New(cvr) }, - providerconfig.CloudProviderVultr: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderVultr: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return vultr.New(cvr) }, - providerconfig.CloudProviderOpenNebula: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + providerconfig.CloudProviderOpenNebula: func(cvr providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return opennebula.New(cvr) }, } ) // ForProvider returns a CloudProvider actuator for the requested provider. -func ForProvider(p providerconfig.CloudProvider, cvr *providerconfig.ConfigVarResolver) (cloudprovidertypes.Provider, error) { +func ForProvider(p providerconfig.CloudProvider, cvr providerconfig.ConfigVarResolver) (cloudprovidertypes.Provider, error) { if p, found := providers[p]; found { return NewValidationCacheWrappingCloudProvider(p(cvr)), nil } diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index 7c5029185..0b473b14f 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -56,7 +56,7 @@ const ( ) type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver } type Config struct { @@ -104,7 +104,7 @@ func (a *alibabaInstance) Status() instance.Status { } // New returns an Alibaba cloud provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -356,40 +356,40 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } c := Config{} - c.AccessKeyID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AccessKeyID, "ALIBABA_ACCESS_KEY_ID") + c.AccessKeyID, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.AccessKeyID, "ALIBABA_ACCESS_KEY_ID") if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"AccessKeyID\" field, error = %w", err) } - c.AccessKeySecret, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AccessKeySecret, "ALIBABA_ACCESS_KEY_SECRET") + c.AccessKeySecret, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.AccessKeySecret, "ALIBABA_ACCESS_KEY_SECRET") if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"AccessKeySecret\" field, error = %w", err) } - c.InstanceType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.InstanceType) + c.InstanceType, err = p.configVarResolver.GetStringValue(rawConfig.InstanceType) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"instanceType\" field, error = %w", err) } - c.RegionID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.RegionID) + c.RegionID, err = p.configVarResolver.GetStringValue(rawConfig.RegionID) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"regionID\" field, error = %w", err) } - c.VSwitchID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VSwitchID) + c.VSwitchID, err = p.configVarResolver.GetStringValue(rawConfig.VSwitchID) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"vSwitchID\" field, error = %w", err) } - c.ZoneID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ZoneID) + c.ZoneID, err = p.configVarResolver.GetStringValue(rawConfig.ZoneID) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"zoneID\" field, error = %w", err) } - c.InternetMaxBandwidthOut, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.InternetMaxBandwidthOut) + c.InternetMaxBandwidthOut, err = p.configVarResolver.GetStringValue(rawConfig.InternetMaxBandwidthOut) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"internetMaxBandwidthOut\" field, error = %w", err) } c.Labels = rawConfig.Labels - c.DiskType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.DiskType) + c.DiskType, err = p.configVarResolver.GetStringValue(rawConfig.DiskType) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"diskType\" field, error = %w", err) } - c.DiskSize, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.DiskSize) + c.DiskSize, err = p.configVarResolver.GetStringValue(rawConfig.DiskSize) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"diskSize\" field, error = %w", err) } diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 795add1e6..048653f92 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -55,7 +55,7 @@ const ( ) type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver } func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance instance.Instance, retErr error) { @@ -249,7 +249,7 @@ func (p *provider) getConfig(ctx context.Context, log *zap.SugaredLogger, provSp } // New returns an Anexia provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index 71ba0601a..30fd475c4 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -39,12 +39,14 @@ import ( "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/progress" "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" "go.uber.org/zap" + "sigs.k8s.io/controller-runtime/pkg/client/fake" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" anxtypes "k8c.io/machine-controller/sdk/cloudprovider/anexia" providerconfigtypes "k8c.io/machine-controller/sdk/providerconfig" + "k8c.io/machine-controller/sdk/providerconfig/configvar" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -239,7 +241,7 @@ func TestAnexiaProvider(t *testing.T) { }, } - provider := New(nil).(*provider) + provider := New(configvar.NewResolver(context.Background(), fake.NewClientBuilder().Build())).(*provider) for _, testCase := range testCases { templateID, err := provider.resolveTemplateID(context.Background(), a, testCase.config, "foo") if testCase.expectedError != "" { @@ -368,7 +370,7 @@ func TestValidate(t *testing.T) { }, ) - provider := New(nil) + provider := New(configvar.NewResolver(context.Background(), fake.NewClientBuilder().Build())) for _, testCase := range getSpecsForValidationTest(t, configCases) { err := provider.Validate(context.Background(), zap.NewNop().Sugar(), testCase.Spec) if testCase.ExpectedError != nil { diff --git a/pkg/cloudprovider/provider/anexia/resolve_config.go b/pkg/cloudprovider/provider/anexia/resolve_config.go index 620639715..96a009a39 100644 --- a/pkg/cloudprovider/provider/anexia/resolve_config.go +++ b/pkg/cloudprovider/provider/anexia/resolve_config.go @@ -60,12 +60,12 @@ type resolvedConfig struct { } func (p *provider) resolveTemplateID(ctx context.Context, a api.API, config anxtypes.RawConfig, locationID string) (string, error) { - templateName, err := p.configVarResolver.GetConfigVarStringValue(config.Template) + templateName, err := p.configVarResolver.GetStringValue(config.Template) if err != nil { return "", fmt.Errorf("failed to get 'template': %w", err) } - templateBuild, err := p.configVarResolver.GetConfigVarStringValue(config.TemplateBuild) + templateBuild, err := p.configVarResolver.GetStringValue(config.TemplateBuild) if err != nil { return "", fmt.Errorf("failed to get 'templateBuild': %w", err) } @@ -87,7 +87,7 @@ func (p *provider) resolveNetworkConfig(log *zap.SugaredLogger, config anxtypes. log.Info("Configuration uses the deprecated VlanID attribute, please migrate to the Networks array instead.") - vlanID, err := p.configVarResolver.GetConfigVarStringValue(config.VlanID) + vlanID, err := p.configVarResolver.GetStringValue(config.VlanID) if err != nil { return nil, fmt.Errorf("failed to get 'vlanID': %w", err) } @@ -102,14 +102,14 @@ func (p *provider) resolveNetworkConfig(log *zap.SugaredLogger, config anxtypes. ret := make([]resolvedNetwork, len(config.Networks)) for netIndex, net := range config.Networks { - vlanID, err := p.configVarResolver.GetConfigVarStringValue(net.VlanID) + vlanID, err := p.configVarResolver.GetStringValue(net.VlanID) if err != nil { return nil, fmt.Errorf("failed to get 'vlanID' for network %v: %w", netIndex, err) } prefixes := make([]string, len(net.PrefixIDs)) for prefixIndex, prefix := range net.PrefixIDs { - prefixID, err := p.configVarResolver.GetConfigVarStringValue(prefix) + prefixID, err := p.configVarResolver.GetStringValue(prefix) if err != nil { return nil, fmt.Errorf("failed to get 'prefixID' for network %v, prefix %v: %w", netIndex, prefixIndex, err) } @@ -145,7 +145,7 @@ func (p *provider) resolveDiskConfig(log *zap.SugaredLogger, config anxtypes.Raw ret := make([]resolvedDisk, len(config.Disks)) for idx, disk := range config.Disks { - performanceType, err := p.configVarResolver.GetConfigVarStringValue(disk.PerformanceType) + performanceType, err := p.configVarResolver.GetStringValue(disk.PerformanceType) if err != nil { return nil, fmt.Errorf("failed to get 'performanceType' of disk %v: %w", idx, err) } @@ -165,17 +165,17 @@ func (p *provider) resolveConfig(ctx context.Context, log *zap.SugaredLogger, co RawConfig: config, } - ret.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(config.Token, anxtypes.AnxTokenEnv) + ret.Token, err = p.configVarResolver.GetStringValueOrEnv(config.Token, anxtypes.AnxTokenEnv) if err != nil { return nil, fmt.Errorf("failed to get 'token': %w", err) } - ret.LocationID, err = p.configVarResolver.GetConfigVarStringValue(config.LocationID) + ret.LocationID, err = p.configVarResolver.GetStringValue(config.LocationID) if err != nil { return nil, fmt.Errorf("failed to get 'locationID': %w", err) } - ret.TemplateID, err = p.configVarResolver.GetConfigVarStringValue(config.TemplateID) + ret.TemplateID, err = p.configVarResolver.GetStringValue(config.TemplateID) if err != nil { return nil, fmt.Errorf("failed to get 'templateID': %w", err) } diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index fa9b494a7..e8d76480c 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -77,11 +77,11 @@ func init() { } type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver } // New returns a aws provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -352,55 +352,55 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } c := Config{} - c.AccessKeyID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AccessKeyID, "AWS_ACCESS_KEY_ID") + c.AccessKeyID, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.AccessKeyID, "AWS_ACCESS_KEY_ID") if err != nil { return nil, nil, nil, fmt.Errorf("failed to get the value of \"accessKeyId\" field, error = %w", err) } - c.SecretAccessKey, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.SecretAccessKey, "AWS_SECRET_ACCESS_KEY") + c.SecretAccessKey, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.SecretAccessKey, "AWS_SECRET_ACCESS_KEY") if err != nil { return nil, nil, nil, fmt.Errorf("failed to get the value of \"secretAccessKey\" field, error = %w", err) } - c.Region, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Region) + c.Region, err = p.configVarResolver.GetStringValue(rawConfig.Region) if err != nil { return nil, nil, nil, err } - c.VpcID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VpcID) + c.VpcID, err = p.configVarResolver.GetStringValue(rawConfig.VpcID) if err != nil { return nil, nil, nil, err } - c.SubnetID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.SubnetID) + c.SubnetID, err = p.configVarResolver.GetStringValue(rawConfig.SubnetID) if err != nil { return nil, nil, nil, err } - c.AvailabilityZone, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.AvailabilityZone) + c.AvailabilityZone, err = p.configVarResolver.GetStringValue(rawConfig.AvailabilityZone) if err != nil { return nil, nil, nil, err } for _, securityGroupIDRaw := range rawConfig.SecurityGroupIDs { - securityGroupID, err := p.configVarResolver.GetConfigVarStringValue(securityGroupIDRaw) + securityGroupID, err := p.configVarResolver.GetStringValue(securityGroupIDRaw) if err != nil { return nil, nil, nil, err } c.SecurityGroupIDs = append(c.SecurityGroupIDs, securityGroupID) } - c.InstanceProfile, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.InstanceProfile) + c.InstanceProfile, err = p.configVarResolver.GetStringValue(rawConfig.InstanceProfile) if err != nil { return nil, nil, nil, err } - instanceTypeStr, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.InstanceType) + instanceTypeStr, err := p.configVarResolver.GetStringValue(rawConfig.InstanceType) if err != nil { return nil, nil, nil, err } c.InstanceType = ec2types.InstanceType(instanceTypeStr) - c.AMI, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.AMI) + c.AMI, err = p.configVarResolver.GetStringValue(rawConfig.AMI) if err != nil { return nil, nil, nil, err } c.DiskSize = rawConfig.DiskSize - diskTypeStr, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.DiskType) + diskTypeStr, err := p.configVarResolver.GetStringValue(rawConfig.DiskType) if err != nil { return nil, nil, nil, err } @@ -428,7 +428,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c.DiskIops = rawConfig.DiskIops } - c.EBSVolumeEncrypted, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.EBSVolumeEncrypted) + c.EBSVolumeEncrypted, _, err = p.configVarResolver.GetBoolValue(rawConfig.EBSVolumeEncrypted) if err != nil { return nil, nil, nil, fmt.Errorf("failed to get ebsVolumeEncrypted value: %w", err) } @@ -436,30 +436,30 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c.AssignPublicIP = rawConfig.AssignPublicIP c.IsSpotInstance = rawConfig.IsSpotInstance if rawConfig.SpotInstanceConfig != nil && c.IsSpotInstance != nil && *c.IsSpotInstance { - maxPrice, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.SpotInstanceConfig.MaxPrice) + maxPrice, err := p.configVarResolver.GetStringValue(rawConfig.SpotInstanceConfig.MaxPrice) if err != nil { return nil, nil, nil, err } c.SpotMaxPrice = ptr.To(maxPrice) - persistentRequest, _, err := p.configVarResolver.GetConfigVarBoolValue(rawConfig.SpotInstanceConfig.PersistentRequest) + persistentRequest, _, err := p.configVarResolver.GetBoolValue(rawConfig.SpotInstanceConfig.PersistentRequest) if err != nil { return nil, nil, nil, err } c.SpotPersistentRequest = ptr.To(persistentRequest) - interruptionBehavior, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.SpotInstanceConfig.InterruptionBehavior) + interruptionBehavior, err := p.configVarResolver.GetStringValue(rawConfig.SpotInstanceConfig.InterruptionBehavior) if err != nil { return nil, nil, nil, err } c.SpotInterruptionBehavior = ptr.To(interruptionBehavior) } - assumeRoleARN, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AssumeRoleARN, "AWS_ASSUME_ROLE_ARN") + assumeRoleARN, err := p.configVarResolver.GetStringValueOrEnv(rawConfig.AssumeRoleARN, "AWS_ASSUME_ROLE_ARN") if err != nil { return nil, nil, nil, err } c.AssumeRoleARN = assumeRoleARN - assumeRoleExternalID, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AssumeRoleExternalID, "AWS_ASSUME_ROLE_EXTERNAL_ID") + assumeRoleExternalID, err := p.configVarResolver.GetStringValueOrEnv(rawConfig.AssumeRoleExternalID, "AWS_ASSUME_ROLE_EXTERNAL_ID") if err != nil { return nil, nil, nil, err } diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index f14cae897..57db40ec3 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -72,7 +72,7 @@ const ( ) type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver } type config struct { @@ -229,7 +229,7 @@ func getOSImageReference(c *config, os providerconfig.OperatingSystem) (*compute } // New returns a new azure provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -249,32 +249,32 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p } c := config{} - c.SubscriptionID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawCfg.SubscriptionID, envSubscriptionID) + c.SubscriptionID, err = p.configVarResolver.GetStringValueOrEnv(rawCfg.SubscriptionID, envSubscriptionID) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"subscriptionID\" field, error = %w", err) } - c.TenantID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawCfg.TenantID, envTenantID) + c.TenantID, err = p.configVarResolver.GetStringValueOrEnv(rawCfg.TenantID, envTenantID) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"tenantID\" field, error = %w", err) } - c.ClientID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawCfg.ClientID, envClientID) + c.ClientID, err = p.configVarResolver.GetStringValueOrEnv(rawCfg.ClientID, envClientID) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"clientID\" field, error = %w", err) } - c.ClientSecret, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawCfg.ClientSecret, envClientSecret) + c.ClientSecret, err = p.configVarResolver.GetStringValueOrEnv(rawCfg.ClientSecret, envClientSecret) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"clientSecret\" field, error = %w", err) } - c.ResourceGroup, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.ResourceGroup) + c.ResourceGroup, err = p.configVarResolver.GetStringValue(rawCfg.ResourceGroup) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"resourceGroup\" field, error = %w", err) } - c.VNetResourceGroup, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.VNetResourceGroup) + c.VNetResourceGroup, err = p.configVarResolver.GetStringValue(rawCfg.VNetResourceGroup) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"vnetResourceGroup\" field, error = %w", err) } @@ -283,37 +283,37 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p c.VNetResourceGroup = c.ResourceGroup } - c.Location, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.Location) + c.Location, err = p.configVarResolver.GetStringValue(rawCfg.Location) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"location\" field, error = %w", err) } - c.VMSize, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.VMSize) + c.VMSize, err = p.configVarResolver.GetStringValue(rawCfg.VMSize) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"vmSize\" field, error = %w", err) } - c.VNetName, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.VNetName) + c.VNetName, err = p.configVarResolver.GetStringValue(rawCfg.VNetName) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"vnetName\" field, error = %w", err) } - c.SubnetName, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.SubnetName) + c.SubnetName, err = p.configVarResolver.GetStringValue(rawCfg.SubnetName) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"subnetName\" field, error = %w", err) } - c.LoadBalancerSku, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.LoadBalancerSku) + c.LoadBalancerSku, err = p.configVarResolver.GetStringValue(rawCfg.LoadBalancerSku) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"loadBalancerSku\" field, error = %w", err) } - c.RouteTableName, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.RouteTableName) + c.RouteTableName, err = p.configVarResolver.GetStringValue(rawCfg.RouteTableName) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"routeTableName\" field, error = %w", err) } - c.AssignPublicIP, _, err = p.configVarResolver.GetConfigVarBoolValue(rawCfg.AssignPublicIP) + c.AssignPublicIP, _, err = p.configVarResolver.GetBoolValue(rawCfg.AssignPublicIP) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"assignPublicIP\" field, error = %w", err) } @@ -325,12 +325,12 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p c.AssignAvailabilitySet = rawCfg.AssignAvailabilitySet c.EnableAcceleratedNetworking = rawCfg.EnableAcceleratedNetworking - c.AvailabilitySet, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.AvailabilitySet) + c.AvailabilitySet, err = p.configVarResolver.GetStringValue(rawCfg.AvailabilitySet) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"availabilitySet\" field, error = %w", err) } - c.SecurityGroupName, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.SecurityGroupName) + c.SecurityGroupName, err = p.configVarResolver.GetStringValue(rawCfg.SecurityGroupName) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"securityGroupName\" field, error = %w", err) } @@ -365,7 +365,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p } } - c.ImageID, err = p.configVarResolver.GetConfigVarStringValue(rawCfg.ImageID) + c.ImageID, err = p.configVarResolver.GetStringValue(rawCfg.ImageID) if err != nil { return nil, nil, fmt.Errorf("failed to get image id: %w", err) } diff --git a/pkg/cloudprovider/provider/baremetal/provider.go b/pkg/cloudprovider/provider/baremetal/provider.go index 22c4ab50b..0815e374a 100644 --- a/pkg/cloudprovider/provider/baremetal/provider.go +++ b/pkg/cloudprovider/provider/baremetal/provider.go @@ -71,11 +71,11 @@ func (b bareMetalServer) Status() instance.Status { } type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver } // New returns a new BareMetal provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{ configVarResolver: configVarResolver, } @@ -108,7 +108,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c := Config{} - driverName, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.Driver) + driverName, err := p.configVarResolver.GetStringValue(rawConfig.Driver) if err != nil { return nil, nil, fmt.Errorf("failed to get baremetal provider's driver name: %w", err) } @@ -124,7 +124,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf("failed to unmarshal tinkerbell driver spec: %w", err) } - tinkConfig, err := tink.GetConfig(*driverConfig, p.configVarResolver.GetConfigVarStringValueOrEnv) + tinkConfig, err := tink.GetConfig(*driverConfig, p.configVarResolver.GetStringValueOrEnv) if err != nil { return nil, nil, err diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index 794a4a2a7..d2e99dfa2 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -45,11 +45,11 @@ import ( ) type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver } // New returns a digitalocean provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -116,36 +116,36 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } c := Config{} - c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, "DO_TOKEN") + c.Token, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Token, "DO_TOKEN") if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"token\" field, error = %w", err) } - c.Region, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Region) + c.Region, err = p.configVarResolver.GetStringValue(rawConfig.Region) if err != nil { return nil, nil, err } - c.Size, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Size) + c.Size, err = p.configVarResolver.GetStringValue(rawConfig.Size) if err != nil { return nil, nil, err } - c.Backups, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.Backups) + c.Backups, _, err = p.configVarResolver.GetBoolValue(rawConfig.Backups) if err != nil { return nil, nil, err } - c.IPv6, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.IPv6) + c.IPv6, _, err = p.configVarResolver.GetBoolValue(rawConfig.IPv6) if err != nil { return nil, nil, err } - c.PrivateNetworking, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.PrivateNetworking) + c.PrivateNetworking, _, err = p.configVarResolver.GetBoolValue(rawConfig.PrivateNetworking) if err != nil { return nil, nil, err } - c.Monitoring, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.Monitoring) + c.Monitoring, _, err = p.configVarResolver.GetBoolValue(rawConfig.Monitoring) if err != nil { return nil, nil, err } for _, tag := range rawConfig.Tags { - tagVal, err := p.configVarResolver.GetConfigVarStringValue(tag) + tagVal, err := p.configVarResolver.GetStringValue(tag) if err != nil { return nil, nil, err } diff --git a/pkg/cloudprovider/provider/edge/provider.go b/pkg/cloudprovider/provider/edge/provider.go index e9f1e3008..b311146b0 100644 --- a/pkg/cloudprovider/provider/edge/provider.go +++ b/pkg/cloudprovider/provider/edge/provider.go @@ -57,7 +57,7 @@ func (f CloudProviderInstance) Status() instance.Status { } // New returns a edge cloud provider. -func New(_ *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(_ providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{} } diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index b759fb5fe..871351b9c 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -48,7 +48,7 @@ const ( ) // New returns a Equinix Metal provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -77,7 +77,7 @@ func populateDefaults(c *equinixmetaltypes.RawConfig) { } type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver } func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *equinixmetaltypes.RawConfig, *providerconfig.Config, error) { @@ -96,49 +96,49 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *e } c := Config{} - c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, "METAL_AUTH_TOKEN") + c.Token, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Token, "METAL_AUTH_TOKEN") if err != nil || len(c.Token) == 0 { // This retry is temporary and is only required to facilitate migration from Packet to Equinix Metal // We look for env variable PACKET_API_KEY associated with Packet to ensure that nothing breaks during automated migration for the Machines // TODO(@ahmedwaleedmalik) Remove this after a release period - c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, "PACKET_API_KEY") + c.Token, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Token, "PACKET_API_KEY") if err != nil { return nil, nil, nil, fmt.Errorf("failed to get the value of \"apiKey\" field, error = %w", err) } } - c.ProjectID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProjectID, "METAL_PROJECT_ID") + c.ProjectID, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.ProjectID, "METAL_PROJECT_ID") if err != nil || len(c.ProjectID) == 0 { // This retry is temporary and is only required to facilitate migration from Packet to Equinix Metal // We look for env variable PACKET_PROJECT_ID associated with Packet to ensure that nothing breaks during automated migration for the Machines // TODO(@ahmedwaleedmalik) Remove this after a release period - c.ProjectID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProjectID, "PACKET_PROJECT_ID") + c.ProjectID, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.ProjectID, "PACKET_PROJECT_ID") if err != nil { return nil, nil, nil, fmt.Errorf("failed to get the value of \"apiKey\" field, error = %w", err) } } - c.InstanceType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.InstanceType) + c.InstanceType, err = p.configVarResolver.GetStringValue(rawConfig.InstanceType) if err != nil { return nil, nil, nil, fmt.Errorf("failed to get the value of \"instanceType\" field, error = %w", err) } - c.BillingCycle, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.BillingCycle) + c.BillingCycle, err = p.configVarResolver.GetStringValue(rawConfig.BillingCycle) if err != nil { return nil, nil, nil, fmt.Errorf("failed to get the value of \"billingCycle\" field, error = %w", err) } for i, tag := range rawConfig.Tags { - tagValue, err := p.configVarResolver.GetConfigVarStringValue(tag) + tagValue, err := p.configVarResolver.GetStringValue(tag) if err != nil { return nil, nil, nil, fmt.Errorf("failed to read the value for the Tag at index %d of the \"tags\" field, error = %w", i, err) } c.Tags = append(c.Tags, tagValue) } for i, facility := range rawConfig.Facilities { - facilityValue, err := p.configVarResolver.GetConfigVarStringValue(facility) + facilityValue, err := p.configVarResolver.GetStringValue(facility) if err != nil { return nil, nil, nil, fmt.Errorf("failed to read the value for the Tag at index %d of the \"facilities\" field, error = %w", i, err) } c.Facilities = append(c.Facilities, facilityValue) } - c.Metro, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Metro) + c.Metro, err = p.configVarResolver.GetStringValue(rawConfig.Metro) if err != nil { return nil, nil, nil, fmt.Errorf("failed to get the value of \"metro\" field, error = %w", err) } diff --git a/pkg/cloudprovider/provider/external/provider.go b/pkg/cloudprovider/provider/external/provider.go index a881360fc..9ce1e4cd8 100644 --- a/pkg/cloudprovider/provider/external/provider.go +++ b/pkg/cloudprovider/provider/external/provider.go @@ -58,7 +58,7 @@ func (f CloudProviderInstance) Status() instance.Status { } // New returns an external cloud provider. -func New(_ *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(_ providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{} } diff --git a/pkg/cloudprovider/provider/fake/provider.go b/pkg/cloudprovider/provider/fake/provider.go index 1ddbf68ae..b07091a13 100644 --- a/pkg/cloudprovider/provider/fake/provider.go +++ b/pkg/cloudprovider/provider/fake/provider.go @@ -61,7 +61,7 @@ func (f CloudProviderInstance) Status() instance.Status { } // New returns a fake cloud provider. -func New(_ *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(_ providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{} } diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index c8561d245..c78903569 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -122,7 +122,7 @@ type clientConfig struct { } // newConfig creates a Provider configuration out of the passed resolver and spec. -func newConfig(resolver *providerconfig.ConfigVarResolver, spec clusterv1alpha1.ProviderSpec) (*config, error) { +func newConfig(resolver providerconfig.ConfigVarResolver, spec clusterv1alpha1.ProviderSpec) (*config, error) { // Create cloud provider spec. cpSpec, providerConfig, err := newCloudProviderSpec(spec) if err != nil { @@ -138,12 +138,12 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec clusterv1alpha1. guestOSFeatures: cpSpec.GuestOSFeatures, } - cfg.serviceAccount, err = resolver.GetConfigVarStringValueOrEnv(cpSpec.ServiceAccount, envGoogleServiceAccount) + cfg.serviceAccount, err = resolver.GetStringValueOrEnv(cpSpec.ServiceAccount, envGoogleServiceAccount) if err != nil { return nil, fmt.Errorf("cannot retrieve service account: %w", err) } - cfg.projectID, err = resolver.GetConfigVarStringValue(cpSpec.ProjectID) + cfg.projectID, err = resolver.GetStringValue(cpSpec.ProjectID) if err != nil { return nil, fmt.Errorf("failed to retrieve project id: %w", err) } @@ -153,38 +153,38 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec clusterv1alpha1. return nil, fmt.Errorf("cannot prepare JWT: %w", err) } - cfg.zone, err = resolver.GetConfigVarStringValue(cpSpec.Zone) + cfg.zone, err = resolver.GetStringValue(cpSpec.Zone) if err != nil { return nil, fmt.Errorf("cannot retrieve zone: %w", err) } - cfg.machineType, err = resolver.GetConfigVarStringValue(cpSpec.MachineType) + cfg.machineType, err = resolver.GetStringValue(cpSpec.MachineType) if err != nil { return nil, fmt.Errorf("cannot retrieve machine type: %w", err) } - cfg.diskType, err = resolver.GetConfigVarStringValue(cpSpec.DiskType) + cfg.diskType, err = resolver.GetStringValue(cpSpec.DiskType) if err != nil { return nil, fmt.Errorf("cannot retrieve disk type: %w", err) } - cfg.network, err = resolver.GetConfigVarStringValue(cpSpec.Network) + cfg.network, err = resolver.GetStringValue(cpSpec.Network) if err != nil { return nil, fmt.Errorf("cannot retrieve network: %w", err) } - cfg.subnetwork, err = resolver.GetConfigVarStringValue(cpSpec.Subnetwork) + cfg.subnetwork, err = resolver.GetStringValue(cpSpec.Subnetwork) if err != nil { return nil, fmt.Errorf("cannot retrieve subnetwork: %w", err) } - cfg.preemptible, _, err = resolver.GetConfigVarBoolValue(cpSpec.Preemptible) + cfg.preemptible, _, err = resolver.GetBoolValue(cpSpec.Preemptible) if err != nil { return nil, fmt.Errorf("cannot retrieve preemptible: %w", err) } if cpSpec.AutomaticRestart != nil { - automaticRestart, _, err := resolver.GetConfigVarBoolValue(*cpSpec.AutomaticRestart) + automaticRestart, _, err := resolver.GetBoolValue(*cpSpec.AutomaticRestart) if err != nil { return nil, fmt.Errorf("cannot retrieve automaticRestart: %w", err) } @@ -196,7 +196,7 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec clusterv1alpha1. } if cpSpec.ProvisioningModel != nil { - provisioningModel, err := resolver.GetConfigVarStringValue(*cpSpec.ProvisioningModel) + provisioningModel, err := resolver.GetStringValue(*cpSpec.ProvisioningModel) if err != nil { return nil, fmt.Errorf("cannot retrieve provisioningModel: %w", err) } @@ -207,38 +207,38 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec clusterv1alpha1. cfg.assignPublicIPAddress = true if cpSpec.AssignPublicIPAddress != nil { - cfg.assignPublicIPAddress, _, err = resolver.GetConfigVarBoolValue(*cpSpec.AssignPublicIPAddress) + cfg.assignPublicIPAddress, _, err = resolver.GetBoolValue(*cpSpec.AssignPublicIPAddress) if err != nil { return nil, fmt.Errorf("failed to retrieve assignPublicIPAddress: %w", err) } } - cfg.multizone, _, err = resolver.GetConfigVarBoolValue(cpSpec.MultiZone) + cfg.multizone, _, err = resolver.GetBoolValue(cpSpec.MultiZone) if err != nil { return nil, fmt.Errorf("failed to retrieve multizone: %w", err) } - cfg.regional, _, err = resolver.GetConfigVarBoolValue(cpSpec.Regional) + cfg.regional, _, err = resolver.GetBoolValue(cpSpec.Regional) if err != nil { return nil, fmt.Errorf("failed to retrieve regional: %w", err) } - cfg.customImage, err = resolver.GetConfigVarStringValue(cpSpec.CustomImage) + cfg.customImage, err = resolver.GetStringValue(cpSpec.CustomImage) if err != nil { return nil, fmt.Errorf("failed to retrieve gce custom image: %w", err) } - cfg.disableMachineServiceAccount, _, err = resolver.GetConfigVarBoolValue(cpSpec.DisableMachineServiceAccount) + cfg.disableMachineServiceAccount, _, err = resolver.GetBoolValue(cpSpec.DisableMachineServiceAccount) if err != nil { return nil, fmt.Errorf("failed to retrieve disable machine service account: %w", err) } - cfg.enableNestedVirtualization, _, err = resolver.GetConfigVarBoolValue(cpSpec.EnableNestedVirtualization) + cfg.enableNestedVirtualization, _, err = resolver.GetBoolValue(cpSpec.EnableNestedVirtualization) if err != nil { return nil, fmt.Errorf("failed to retrieve enable nested virtualization: %w", err) } - cfg.minCPUPlatform, err = resolver.GetConfigVarStringValue(cpSpec.MinCPUPlatform) + cfg.minCPUPlatform, err = resolver.GetStringValue(cpSpec.MinCPUPlatform) if err != nil { return nil, fmt.Errorf("failed to retrieve min cpu platform: %w", err) } diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index dc115822a..93eabd389 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -72,11 +72,11 @@ var _ cloudprovidertypes.Provider = New(nil) // Provider implements the cloud.Provider interface for the Google Cloud Platform. type Provider struct { - resolver *providerconfig.ConfigVarResolver + resolver providerconfig.ConfigVarResolver } // New creates a cloud provider instance for the Google Cloud Platform. -func New(configVarResolver *providerconfig.ConfigVarResolver) *Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) *Provider { return &Provider{ resolver: configVarResolver, } diff --git a/pkg/cloudprovider/provider/gce/provider_test.go b/pkg/cloudprovider/provider/gce/provider_test.go index 945776c9c..47a6807d8 100644 --- a/pkg/cloudprovider/provider/gce/provider_test.go +++ b/pkg/cloudprovider/provider/gce/provider_test.go @@ -27,7 +27,7 @@ import ( "go.uber.org/zap" clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" - "k8c.io/machine-controller/sdk/providerconfig" + "k8c.io/machine-controller/sdk/providerconfig/configvar" "k8s.io/apimachinery/pkg/runtime" fake2 "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -123,7 +123,7 @@ func TestValidate(t *testing.T) { return data } - p := New(providerconfig.NewConfigVarResolver(context.Background(), fake2.NewClientBuilder().Build())) + p := New(configvar.NewResolver(context.Background(), fake2.NewClientBuilder().Build())) tests := []struct { name string mspec clusterv1alpha1.MachineSpec diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 3e9635a53..f5ea57937 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -46,11 +46,11 @@ const ( ) type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver } // New returns a Hetzner provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -98,38 +98,38 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } c := Config{} - c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, "HZ_TOKEN") + c.Token, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Token, "HZ_TOKEN") if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"token\" field, error = %w", err) } - c.ServerType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ServerType) + c.ServerType, err = p.configVarResolver.GetStringValue(rawConfig.ServerType) if err != nil { return nil, nil, err } - c.Datacenter, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Datacenter) + c.Datacenter, err = p.configVarResolver.GetStringValue(rawConfig.Datacenter) if err != nil { return nil, nil, err } - c.Image, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Image) + c.Image, err = p.configVarResolver.GetStringValue(rawConfig.Image) if err != nil { return nil, nil, err } - c.Location, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Location) + c.Location, err = p.configVarResolver.GetStringValue(rawConfig.Location) if err != nil { return nil, nil, err } - c.PlacementGroupPrefix, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.PlacementGroupPrefix) + c.PlacementGroupPrefix, err = p.configVarResolver.GetStringValue(rawConfig.PlacementGroupPrefix) if err != nil { return nil, nil, err } for _, network := range rawConfig.Networks { - networkValue, err := p.configVarResolver.GetConfigVarStringValue(network) + networkValue, err := p.configVarResolver.GetStringValue(network) if err != nil { return nil, nil, err } @@ -137,7 +137,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } for _, firewall := range rawConfig.Firewalls { - firewallValue, err := p.configVarResolver.GetConfigVarStringValue(firewall) + firewallValue, err := p.configVarResolver.GetStringValue(firewall) if err != nil { return nil, nil, err } @@ -609,12 +609,12 @@ func hzErrorToTerminalError(err error, msg string) error { } func (p *provider) publicIPsAssignment(rawConfig *hetznertypes.RawConfig) (bool, bool, error) { - assignIPv4, ipv4Set, err := p.configVarResolver.GetConfigVarBoolValue(rawConfig.AssignPublicIPv4) + assignIPv4, ipv4Set, err := p.configVarResolver.GetBoolValue(rawConfig.AssignPublicIPv4) if err != nil { return false, false, err } - assignIPv6, ipv6Set, err := p.configVarResolver.GetConfigVarBoolValue(rawConfig.AssignPublicIPv6) + assignIPv6, ipv6Set, err := p.configVarResolver.GetBoolValue(rawConfig.AssignPublicIPv6) if err != nil { return false, false, err } diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index fd623ebb8..9e0cca504 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -83,11 +83,11 @@ const ( ) type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver } // New returns a Kubevirt provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -146,7 +146,7 @@ const ( ) func (p *provider) affinityType(affinityType providerconfig.ConfigVarString) (AffinityType, error) { - podAffinityPresetString, err := p.configVarResolver.GetConfigVarStringValue(affinityType) + podAffinityPresetString, err := p.configVarResolver.GetStringValue(affinityType) if err != nil { return "", fmt.Errorf(`failed to parse "podAffinityPreset" field: %w`, err) } @@ -243,7 +243,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } else { // Environment variable or secret reference was used for providing the value of kubeconfig // We have to be lenient in this case and allow unencoded values as well. - config.Kubeconfig, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Auth.Kubeconfig, "KUBEVIRT_KUBECONFIG") + config.Kubeconfig, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Auth.Kubeconfig, "KUBEVIRT_KUBECONFIG") if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "kubeconfig" field: %w`, err) } @@ -256,7 +256,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } var enableNetworkMultiQueueSet bool - config.EnableNetworkMultiQueue, enableNetworkMultiQueueSet, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.VirtualMachine.EnableNetworkMultiQueue) + config.EnableNetworkMultiQueue, enableNetworkMultiQueueSet, err = p.configVarResolver.GetBoolValue(rawConfig.VirtualMachine.EnableNetworkMultiQueue) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "enableNetworkMultiQueue" field: %w`, err) } @@ -265,7 +265,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p config.EnableNetworkMultiQueue = true } - config.ClusterName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ClusterName) + config.ClusterName, err = p.configVarResolver.GetStringValue(rawConfig.ClusterName) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "clusterName" field: %w`, err) } @@ -275,12 +275,12 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf("failed to decode kubeconfig: %w", err) } - cpus, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.CPUs) + cpus, err := p.configVarResolver.GetStringValue(rawConfig.VirtualMachine.Template.CPUs) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "cpus" field: %w`, err) } - memory, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.Memory) + memory, err := p.configVarResolver.GetStringValue(rawConfig.VirtualMachine.Template.Memory) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "memory" field: %w`, err) } @@ -296,12 +296,12 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if len(rawConfig.VirtualMachine.Template.PrimaryDisk.ExtraHeaders) > 0 { config.ExtraHeaders = rawConfig.VirtualMachine.Template.PrimaryDisk.ExtraHeaders } - dataVolumeSecretRef, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.DataVolumeSecretRef) + dataVolumeSecretRef, err := p.configVarResolver.GetStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.DataVolumeSecretRef) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "dataVolumeSecretRef" field: %w`, err) } config.DataVolumeSecretRef = dataVolumeSecretRef - extraHeadersSecretRef, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.ExtraHeadersSecretRef) + extraHeadersSecretRef, err := p.configVarResolver.GetStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.ExtraHeadersSecretRef) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "extraHeadersSecretRef" field: %w`, err) } @@ -314,20 +314,20 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf(`failed to get value of "osImageSource" field: %w`, err) } - storageTarget, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.StorageTarget) + storageTarget, err := p.configVarResolver.GetStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.StorageTarget) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "storageTarget" field: %w`, err) } config.StorageTarget = StorageTarget(storageTarget) - pvcSize, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.Size) + pvcSize, err := p.configVarResolver.GetStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.Size) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "pvcSize" field: %w`, err) } if config.PVCSize, err = resource.ParseQuantity(pvcSize); err != nil { return nil, nil, fmt.Errorf(`failed to parse value of "pvcSize" field: %w`, err) } - config.StorageClassName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.StorageClassName) + config.StorageClassName, err = p.configVarResolver.GetStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.StorageClassName) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "storageClassName" field: %w`, err) } @@ -335,7 +335,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p config.Instancetype = rawConfig.VirtualMachine.Instancetype config.Preference = rawConfig.VirtualMachine.Preference - dnsPolicyString, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.DNSPolicy) + dnsPolicyString, err := p.configVarResolver.GetStringValue(rawConfig.VirtualMachine.DNSPolicy) if err != nil { return nil, nil, fmt.Errorf(`failed to parse "dnsPolicy" field: %w`, err) } @@ -386,7 +386,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p func (p *provider) getStorageAccessType(ctx context.Context, accessType providerconfig.ConfigVarString, infraClient ctrlruntimeclient.Client, storageClassName string) (corev1.PersistentVolumeAccessMode, error) { - at, _ := p.configVarResolver.GetConfigVarStringValue(accessType) + at, _ := p.configVarResolver.GetStringValue(accessType) if at == "" { sp := &cdicorev1beta1.StorageProfile{} if err := infraClient.Get(ctx, types.NamespacedName{Name: storageClassName}, sp); err != nil { @@ -416,13 +416,13 @@ func (p *provider) parseNodeAffinityPreset(nodeAffinityPreset kubevirttypes.Node if err != nil { return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.type" field: %w`, err) } - nodeAffinity.Key, err = p.configVarResolver.GetConfigVarStringValue(nodeAffinityPreset.Key) + nodeAffinity.Key, err = p.configVarResolver.GetStringValue(nodeAffinityPreset.Key) if err != nil { return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.key" field: %w`, err) } nodeAffinity.Values = make([]string, 0, len(nodeAffinityPreset.Values)) for _, v := range nodeAffinityPreset.Values { - valueString, err := p.configVarResolver.GetConfigVarStringValue(v) + valueString, err := p.configVarResolver.GetStringValue(v) if err != nil { return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.value" field: %w`, err) } @@ -434,7 +434,7 @@ func (p *provider) parseNodeAffinityPreset(nodeAffinityPreset kubevirttypes.Node func (p *provider) parseTopologySpreadConstraint(topologyConstraints []kubevirttypes.TopologySpreadConstraint) ([]corev1.TopologySpreadConstraint, error) { parsedTopologyConstraints := make([]corev1.TopologySpreadConstraint, 0, len(topologyConstraints)) for _, constraint := range topologyConstraints { - maxSkewString, err := p.configVarResolver.GetConfigVarStringValue(constraint.MaxSkew) + maxSkewString, err := p.configVarResolver.GetStringValue(constraint.MaxSkew) if err != nil { return nil, fmt.Errorf(`failed to parse "topologySpreadConstraint.maxSkew" field: %w`, err) } @@ -442,11 +442,11 @@ func (p *provider) parseTopologySpreadConstraint(topologyConstraints []kubevirtt if err != nil { return nil, fmt.Errorf(`failed to parse "topologySpreadConstraint.maxSkew" field: %w`, err) } - topologyKey, err := p.configVarResolver.GetConfigVarStringValue(constraint.TopologyKey) + topologyKey, err := p.configVarResolver.GetStringValue(constraint.TopologyKey) if err != nil { return nil, fmt.Errorf(`failed to parse "topologySpreadConstraint.topologyKey" field: %w`, err) } - whenUnsatisfiable, err := p.configVarResolver.GetConfigVarStringValue(constraint.WhenUnsatisfiable) + whenUnsatisfiable, err := p.configVarResolver.GetStringValue(constraint.WhenUnsatisfiable) if err != nil { return nil, fmt.Errorf(`failed to parse "topologySpreadConstraint.whenUnsatisfiable" field: %w`, err) } @@ -460,11 +460,11 @@ func (p *provider) parseTopologySpreadConstraint(topologyConstraints []kubevirtt } func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, config *Config) (*cdicorev1beta1.DataVolumeSource, error) { - osImage, err := p.configVarResolver.GetConfigVarStringValue(primaryDisk.OsImage) + osImage, err := p.configVarResolver.GetStringValue(primaryDisk.OsImage) if err != nil { return nil, fmt.Errorf(`failed to get value of "primaryDisk.osImage" field: %w`, err) } - osImageSource, err := p.configVarResolver.GetConfigVarStringValue(primaryDisk.Source) + osImageSource, err := p.configVarResolver.GetStringValue(primaryDisk.Source) if err != nil { return nil, fmt.Errorf(`failed to get value of "primaryDisk.source" field: %w`, err) } @@ -548,7 +548,7 @@ func getNamespace() string { } func (p *provider) getPullMethod(pullMethod providerconfig.ConfigVarString) (cdicorev1beta1.RegistryPullMethod, error) { - resolvedPM, err := p.configVarResolver.GetConfigVarStringValue(pullMethod) + resolvedPM, err := p.configVarResolver.GetStringValue(pullMethod) if err != nil { return "", err } @@ -1184,7 +1184,7 @@ func setOVNAnnotations(c *Config, annotations map[string]string) error { func (p *provider) configureStorage(infraClient ctrlruntimeclient.Client, template kubevirttypes.Template) (corev1.PersistentVolumeAccessMode, []SecondaryDisks, error) { secondaryDisks := make([]SecondaryDisks, 0, len(template.SecondaryDisks)) for i, sd := range template.SecondaryDisks { - sdSizeString, err := p.configVarResolver.GetConfigVarStringValue(sd.Size) + sdSizeString, err := p.configVarResolver.GetStringValue(sd.Size) if err != nil { return "", nil, fmt.Errorf(`failed to parse "secondaryDisks.size" field: %w`, err) } @@ -1193,7 +1193,7 @@ func (p *provider) configureStorage(infraClient ctrlruntimeclient.Client, templa return "", nil, fmt.Errorf(`failed to parse value of "secondaryDisks.size" field: %w`, err) } - scString, err := p.configVarResolver.GetConfigVarStringValue(sd.StorageClassName) + scString, err := p.configVarResolver.GetStringValue(sd.StorageClassName) if err != nil { return "", nil, fmt.Errorf(`failed to parse value of "secondaryDisks.storageClass" field: %w`, err) } @@ -1208,7 +1208,7 @@ func (p *provider) configureStorage(infraClient ctrlruntimeclient.Client, templa StorageAccessType: storageAccessMode, }) } - scString, err := p.configVarResolver.GetConfigVarStringValue(template.PrimaryDisk.StorageClassName) + scString, err := p.configVarResolver.GetStringValue(template.PrimaryDisk.StorageClassName) if err != nil { return "", nil, fmt.Errorf(`failed to parse value of "primaryDisk.storageClass" field: %w`, err) } diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index 956a88bcd..dd925cff7 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -30,7 +30,7 @@ import ( cloudprovidertesting "k8c.io/machine-controller/pkg/cloudprovider/testing" "k8c.io/machine-controller/sdk/cloudprovider/kubevirt" - "k8c.io/machine-controller/sdk/providerconfig" + "k8c.io/machine-controller/sdk/providerconfig/configvar" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -299,7 +299,7 @@ func TestNewVirtualMachine(t *testing.T) { t.Run(tt.name, func(t *testing.T) { p := &provider{ // Note that configVarResolver is not used in this test as the getConfigFunc is mocked. - configVarResolver: providerconfig.NewConfigVarResolver(context.Background(), fakeclient), + configVarResolver: configvar.NewResolver(context.Background(), fakeclient), } machine := cloudprovidertesting.Creator{ diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index dad86a02b..7eb2c2957 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -47,11 +47,11 @@ import ( ) type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver } // New returns a linode provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -117,29 +117,29 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } c := Config{} - c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, "LINODE_TOKEN") + c.Token, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Token, "LINODE_TOKEN") if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"token\" field, error = %w", err) } - c.Region, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Region) + c.Region, err = p.configVarResolver.GetStringValue(rawConfig.Region) if err != nil { return nil, nil, err } - c.Type, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Type) + c.Type, err = p.configVarResolver.GetStringValue(rawConfig.Type) if err != nil { return nil, nil, err } - c.Backups, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.Backups) + c.Backups, _, err = p.configVarResolver.GetBoolValue(rawConfig.Backups) if err != nil { return nil, nil, err } - c.PrivateNetworking, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.PrivateNetworking) + c.PrivateNetworking, _, err = p.configVarResolver.GetBoolValue(rawConfig.PrivateNetworking) if err != nil { return nil, nil, err } for _, tag := range rawConfig.Tags { - tagVal, err := p.configVarResolver.GetConfigVarStringValue(tag) + tagVal, err := p.configVarResolver.GetStringValue(tag) if err != nil { return nil, nil, err } diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index ec7eb257a..e84cb5c6c 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -62,7 +62,7 @@ type Config struct { } type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver } // Server holds Nutanix server information. @@ -103,7 +103,7 @@ func (nutanixServer Server) Status() instance.Status { } // New returns a nutanix provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { provider := &provider{configVarResolver: configVarResolver} return provider } @@ -125,12 +125,12 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c := Config{} - c.Endpoint, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Endpoint, "NUTANIX_ENDPOINT") + c.Endpoint, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Endpoint, "NUTANIX_ENDPOINT") if err != nil { return nil, nil, nil, err } - port, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Port, "NUTANIX_PORT") + port, err := p.configVarResolver.GetStringValueOrEnv(rawConfig.Port, "NUTANIX_PORT") if err != nil { return nil, nil, nil, err } @@ -144,46 +144,46 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c.Port = ptr.To(portInt) } - c.Username, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Username, "NUTANIX_USERNAME") + c.Username, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Username, "NUTANIX_USERNAME") if err != nil { return nil, nil, nil, err } - c.Password, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Password, "NUTANIX_PASSWORD") + c.Password, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Password, "NUTANIX_PASSWORD") if err != nil { return nil, nil, nil, err } - c.AllowInsecure, err = p.configVarResolver.GetConfigVarBoolValueOrEnv(rawConfig.AllowInsecure, "NUTANIX_INSECURE") + c.AllowInsecure, err = p.configVarResolver.GetBoolValueOrEnv(rawConfig.AllowInsecure, "NUTANIX_INSECURE") if err != nil { return nil, nil, nil, err } - c.ProxyURL, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProxyURL, "NUTANIX_PROXY_URL") + c.ProxyURL, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.ProxyURL, "NUTANIX_PROXY_URL") if err != nil { return nil, nil, nil, err } - c.ClusterName, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ClusterName, "NUTANIX_CLUSTER_NAME") + c.ClusterName, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.ClusterName, "NUTANIX_CLUSTER_NAME") if err != nil { return nil, nil, nil, err } if rawConfig.ProjectName != nil { - c.ProjectName, err = p.configVarResolver.GetConfigVarStringValue(*rawConfig.ProjectName) + c.ProjectName, err = p.configVarResolver.GetStringValue(*rawConfig.ProjectName) if err != nil { return nil, nil, nil, err } } - c.SubnetName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.SubnetName) + c.SubnetName, err = p.configVarResolver.GetStringValue(rawConfig.SubnetName) if err != nil { return nil, nil, nil, err } c.AdditionalSubnetNames = append(c.AdditionalSubnetNames, rawConfig.AdditionalSubnetNames...) - c.ImageName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ImageName) + c.ImageName, err = p.configVarResolver.GetStringValue(rawConfig.ImageName) if err != nil { return nil, nil, nil, err } diff --git a/pkg/cloudprovider/provider/opennebula/provider.go b/pkg/cloudprovider/provider/opennebula/provider.go index 8495758b0..fc5a731b3 100644 --- a/pkg/cloudprovider/provider/opennebula/provider.go +++ b/pkg/cloudprovider/provider/opennebula/provider.go @@ -44,7 +44,7 @@ import ( ) type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver } type CloudProviderSpec struct { @@ -56,7 +56,7 @@ const ( ) // New returns a OpenNebula provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -94,17 +94,17 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } c := Config{} - c.Username, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Username, "ONE_USERNAME") + c.Username, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Username, "ONE_USERNAME") if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"username\" field, error = %w", err) } - c.Password, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Password, "ONE_PASSWORD") + c.Password, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Password, "ONE_PASSWORD") if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"password\" field, error = %w", err) } - c.Endpoint, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Endpoint, "ONE_ENDPOINT") + c.Endpoint, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Endpoint, "ONE_ENDPOINT") if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"endpoint\" field, error = %w", err) } @@ -115,24 +115,24 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c.Memory = rawConfig.Memory - c.Image, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Image) + c.Image, err = p.configVarResolver.GetStringValue(rawConfig.Image) if err != nil { return nil, nil, err } - c.Datastore, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Datastore) + c.Datastore, err = p.configVarResolver.GetStringValue(rawConfig.Datastore) if err != nil { return nil, nil, err } c.DiskSize = rawConfig.DiskSize - c.Network, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Network) + c.Network, err = p.configVarResolver.GetStringValue(rawConfig.Network) if err != nil { return nil, nil, err } - c.EnableVNC, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.EnableVNC) + c.EnableVNC, _, err = p.configVarResolver.GetBoolValue(rawConfig.EnableVNC) if err != nil { return nil, nil, err } diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 44ae50e1f..b7748f75f 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -66,13 +66,13 @@ type clientGetterFunc func(c *Config) (*gophercloud.ProviderClient, error) type portReadinessWaiterFunc func(ctx context.Context, instanceLog *zap.SugaredLogger, netClient *gophercloud.ServiceClient, serverID string, networkID string, instanceReadyCheckPeriod time.Duration, instanceReadyCheckTimeout time.Duration) error type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver clientGetter clientGetterFunc portReadinessWaiter portReadinessWaiterFunc } // New returns a openstack provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{ configVarResolver: configVarResolver, clientGetter: getClient, @@ -125,44 +125,44 @@ var floatingIPAssignLock = &sync.Mutex{} // Get the Project name from config or env var. If not defined fallback to tenant name. func (p *provider) getProjectNameOrTenantName(rawConfig *openstacktypes.RawConfig) (string, error) { - projectName, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProjectName, "OS_PROJECT_NAME") + projectName, err := p.configVarResolver.GetStringValueOrEnv(rawConfig.ProjectName, "OS_PROJECT_NAME") if err == nil && len(projectName) > 0 { return projectName, nil } //fallback to tenantName. - return p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.TenantName, "OS_TENANT_NAME") + return p.configVarResolver.GetStringValueOrEnv(rawConfig.TenantName, "OS_TENANT_NAME") } // Get the Project id from config or env var. If not defined fallback to tenant id. func (p *provider) getProjectIDOrTenantID(rawConfig *openstacktypes.RawConfig) (string, error) { - projectID, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ProjectID, "OS_PROJECT_ID") + projectID, err := p.configVarResolver.GetStringValueOrEnv(rawConfig.ProjectID, "OS_PROJECT_ID") if err == nil && len(projectID) > 0 { return projectID, nil } //fallback to tenantName. - return p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.TenantID, "OS_TENANT_ID") + return p.configVarResolver.GetStringValueOrEnv(rawConfig.TenantID, "OS_TENANT_ID") } func (p *provider) getConfigAuth(c *Config, rawConfig *openstacktypes.RawConfig) error { var err error - c.ApplicationCredentialID, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ApplicationCredentialID, "OS_APPLICATION_CREDENTIAL_ID") + c.ApplicationCredentialID, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.ApplicationCredentialID, "OS_APPLICATION_CREDENTIAL_ID") if err != nil { return fmt.Errorf("failed to get the value of \"applicationCredentialID\" field, error = %w", err) } if c.ApplicationCredentialID != "" { - c.ApplicationCredentialSecret, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.ApplicationCredentialSecret, "OS_APPLICATION_CREDENTIAL_SECRET") + c.ApplicationCredentialSecret, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.ApplicationCredentialSecret, "OS_APPLICATION_CREDENTIAL_SECRET") if err != nil { return fmt.Errorf("failed to get the value of \"applicationCredentialSecret\" field, error = %w", err) } return nil } - c.Username, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Username, "OS_USER_NAME") + c.Username, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Username, "OS_USER_NAME") if err != nil { return fmt.Errorf("failed to get the value of \"username\" field, error = %w", err) } - c.Password, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Password, "OS_PASSWORD") + c.Password, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Password, "OS_PASSWORD") if err != nil { return fmt.Errorf("failed to get the value of \"password\" field, error = %w", err) } @@ -193,7 +193,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } cfg := Config{} - cfg.IdentityEndpoint, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.IdentityEndpoint, "OS_AUTH_URL") + cfg.IdentityEndpoint, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.IdentityEndpoint, "OS_AUTH_URL") if err != nil { return nil, nil, nil, fmt.Errorf("failed to get the value of \"identityEndpoint\" field, error = %w", err) } @@ -205,80 +205,80 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } // Ignore Region not found as Region might not be found and we can default it later. - cfg.Region, _ = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Region, "OS_REGION_NAME") + cfg.Region, _ = p.configVarResolver.GetStringValueOrEnv(rawConfig.Region, "OS_REGION_NAME") - cfg.InstanceReadyCheckPeriod, err = p.configVarResolver.GetConfigVarDurationValueOrDefault(rawConfig.InstanceReadyCheckPeriod, 5*time.Second) + cfg.InstanceReadyCheckPeriod, err = p.configVarResolver.GetDurationValueOrDefault(rawConfig.InstanceReadyCheckPeriod, 5*time.Second) if err != nil { return nil, nil, nil, fmt.Errorf(`failed to get the value of "InstanceReadyCheckPeriod" field, error = %w`, err) } - cfg.InstanceReadyCheckTimeout, err = p.configVarResolver.GetConfigVarDurationValueOrDefault(rawConfig.InstanceReadyCheckTimeout, 10*time.Second) + cfg.InstanceReadyCheckTimeout, err = p.configVarResolver.GetDurationValueOrDefault(rawConfig.InstanceReadyCheckTimeout, 10*time.Second) if err != nil { return nil, nil, nil, fmt.Errorf(`failed to get the value of "InstanceReadyCheckTimeout" field, error = %w`, err) } // We ignore errors here because the OS domain is only required when using Identity API V3. - cfg.DomainName, _ = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.DomainName, "OS_DOMAIN_NAME") - cfg.TokenID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.TokenID) + cfg.DomainName, _ = p.configVarResolver.GetStringValueOrEnv(rawConfig.DomainName, "OS_DOMAIN_NAME") + cfg.TokenID, err = p.configVarResolver.GetStringValue(rawConfig.TokenID) if err != nil { return nil, nil, nil, err } - cfg.Image, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Image) + cfg.Image, err = p.configVarResolver.GetStringValue(rawConfig.Image) if err != nil { return nil, nil, nil, err } - cfg.Flavor, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Flavor) + cfg.Flavor, err = p.configVarResolver.GetStringValue(rawConfig.Flavor) if err != nil { return nil, nil, nil, err } for _, securityGroup := range rawConfig.SecurityGroups { - securityGroupValue, err := p.configVarResolver.GetConfigVarStringValue(securityGroup) + securityGroupValue, err := p.configVarResolver.GetStringValue(securityGroup) if err != nil { return nil, nil, nil, err } cfg.SecurityGroups = append(cfg.SecurityGroups, securityGroupValue) } - cfg.Network, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Network) + cfg.Network, err = p.configVarResolver.GetStringValue(rawConfig.Network) if err != nil { return nil, nil, nil, err } - cfg.Subnet, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Subnet) + cfg.Subnet, err = p.configVarResolver.GetStringValue(rawConfig.Subnet) if err != nil { return nil, nil, nil, err } - cfg.FloatingIPPool, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.FloatingIPPool) + cfg.FloatingIPPool, err = p.configVarResolver.GetStringValue(rawConfig.FloatingIPPool) if err != nil { return nil, nil, nil, err } - cfg.AvailabilityZone, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.AvailabilityZone) + cfg.AvailabilityZone, err = p.configVarResolver.GetStringValue(rawConfig.AvailabilityZone) if err != nil { return nil, nil, nil, err } - cfg.TrustDevicePath, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.TrustDevicePath) + cfg.TrustDevicePath, _, err = p.configVarResolver.GetBoolValue(rawConfig.TrustDevicePath) if err != nil { return nil, nil, nil, err } - cfg.ConfigDrive, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.ConfigDrive) + cfg.ConfigDrive, _, err = p.configVarResolver.GetBoolValue(rawConfig.ConfigDrive) if err != nil { return nil, nil, nil, err } - cfg.ComputeAPIVersion, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ComputeAPIVersion) + cfg.ComputeAPIVersion, err = p.configVarResolver.GetStringValue(rawConfig.ComputeAPIVersion) if err != nil { return nil, nil, nil, err } cfg.RootDiskSizeGB = rawConfig.RootDiskSizeGB - cfg.RootDiskVolumeType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.RootDiskVolumeType) + cfg.RootDiskVolumeType, err = p.configVarResolver.GetStringValue(rawConfig.RootDiskVolumeType) if err != nil { return nil, nil, nil, err } @@ -289,7 +289,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p cfg.Tags = map[string]string{} } - cfg.ServerGroup, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ServerGroup) + cfg.ServerGroup, err = p.configVarResolver.GetStringValue(rawConfig.ServerGroup) if err != nil { return nil, nil, nil, err } diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index 25f26e132..4491f77a6 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -35,7 +35,7 @@ import ( cloudprovidertesting "k8c.io/machine-controller/pkg/cloudprovider/testing" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" - "k8c.io/machine-controller/sdk/providerconfig" + "k8c.io/machine-controller/sdk/providerconfig/configvar" "k8s.io/apimachinery/pkg/runtime" "k8s.io/utils/ptr" @@ -275,7 +275,7 @@ func TestCreateServer(t *testing.T) { ExpectServerCreated(t, tt.wantServerReq) p := &provider{ // Note that configVarResolver is not used in this test as the getConfigFunc is mocked. - configVarResolver: providerconfig.NewConfigVarResolver(context.Background(), fakectrlruntimeclient.NewClientBuilder().Build()), + configVarResolver: configvar.NewResolver(context.Background(), fakectrlruntimeclient.NewClientBuilder().Build()), // mock client config getter clientGetter: func(*Config) (*gophercloud.ProviderClient, error) { pc := client.ServiceClient() @@ -339,7 +339,7 @@ func TestProjectAuthVarsAreCorrectlyLoaded(t *testing.T) { t.Run(tt.name, func(t *testing.T) { p := &provider{ // Note that configVarResolver is not used in this test as the getConfigFunc is mocked. - configVarResolver: providerconfig.NewConfigVarResolver(context.Background(), fakectrlruntimeclient. + configVarResolver: configvar.NewResolver(context.Background(), fakectrlruntimeclient. NewClientBuilder(). Build()), } diff --git a/pkg/cloudprovider/provider/scaleway/provider.go b/pkg/cloudprovider/provider/scaleway/provider.go index 8cff1e5ac..ed8f83843 100644 --- a/pkg/cloudprovider/provider/scaleway/provider.go +++ b/pkg/cloudprovider/provider/scaleway/provider.go @@ -42,11 +42,11 @@ import ( ) type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver } // New returns a Scaleway provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -100,27 +100,27 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } c := Config{} - c.AccessKey, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AccessKey, scw.ScwAccessKeyEnv) + c.AccessKey, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.AccessKey, scw.ScwAccessKeyEnv) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"access_key\" field, error = %w", err) } - c.SecretKey, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.SecretKey, scw.ScwSecretKeyEnv) + c.SecretKey, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.SecretKey, scw.ScwSecretKeyEnv) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"secret_key\" field, error = %w", err) } - c.ProjectID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ProjectID) + c.ProjectID, err = p.configVarResolver.GetStringValue(rawConfig.ProjectID) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"project_id\" field, error = %w", err) } - c.Zone, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Zone) + c.Zone, err = p.configVarResolver.GetStringValue(rawConfig.Zone) if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"zone\" field, error = %w", err) } - c.CommercialType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.CommercialType) + c.CommercialType, err = p.configVarResolver.GetStringValue(rawConfig.CommercialType) if err != nil { return nil, nil, err } - c.IPv6, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.IPv6) + c.IPv6, _, err = p.configVarResolver.GetBoolValue(rawConfig.IPv6) if err != nil { return nil, nil, err } diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go index 351408b51..9d649d65e 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -55,7 +55,7 @@ const ( ) type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver } type Auth struct { @@ -98,7 +98,7 @@ type Config struct { } // New returns a VMware Cloud Director provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -325,57 +325,57 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c := Config{} - c.APIToken, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.APIToken, "VCD_API_TOKEN") + c.APIToken, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.APIToken, "VCD_API_TOKEN") if err != nil { return nil, nil, nil, fmt.Errorf(`failed to get the value of "apiToken" field, error = %w`, err) } - c.Username, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Username, "VCD_USER") + c.Username, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Username, "VCD_USER") if err != nil { return nil, nil, nil, fmt.Errorf(`failed to get the value of "username" field, error = %w`, err) } - c.Password, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Password, "VCD_PASSWORD") + c.Password, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Password, "VCD_PASSWORD") if err != nil { return nil, nil, nil, fmt.Errorf(`failed to get the value of "password" field, error = %w`, err) } - c.Organization, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Organization, "VCD_ORG") + c.Organization, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Organization, "VCD_ORG") if err != nil { return nil, nil, nil, fmt.Errorf(`failed to get the value of "organization" field, error = %w`, err) } - c.URL, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.URL, "VCD_URL") + c.URL, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.URL, "VCD_URL") if err != nil { return nil, nil, nil, fmt.Errorf(`failed to get the value of "url" field, error = %w`, err) } - c.VDC, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.VDC, "VCD_VDC") + c.VDC, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.VDC, "VCD_VDC") if err != nil { return nil, nil, nil, fmt.Errorf(`failed to get the value of "vdc" field, error = %w`, err) } - c.AllowInsecure, err = p.configVarResolver.GetConfigVarBoolValueOrEnv(rawConfig.AllowInsecure, "VCD_ALLOW_UNVERIFIED_SSL") + c.AllowInsecure, err = p.configVarResolver.GetBoolValueOrEnv(rawConfig.AllowInsecure, "VCD_ALLOW_UNVERIFIED_SSL") if err != nil { return nil, nil, nil, fmt.Errorf(`failed to get the value of "allowInsecure" field, error = %w`, err) } - c.VApp, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VApp) + c.VApp, err = p.configVarResolver.GetStringValue(rawConfig.VApp) if err != nil { return nil, nil, nil, err } - c.Template, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Template) + c.Template, err = p.configVarResolver.GetStringValue(rawConfig.Template) if err != nil { return nil, nil, nil, err } - c.Catalog, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Catalog) + c.Catalog, err = p.configVarResolver.GetStringValue(rawConfig.Catalog) if err != nil { return nil, nil, nil, err } - c.Network, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Network) + c.Network, err = p.configVarResolver.GetStringValue(rawConfig.Network) if err != nil { return nil, nil, nil, err } diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 7dc1aff5a..dd7fbc84e 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -45,11 +45,11 @@ import ( ) type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver } // New returns a VSphere provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { provider := &provider{configVarResolver: configVarResolver} return provider } @@ -135,82 +135,82 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } c := Config{} - c.TemplateVMName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.TemplateVMName) + c.TemplateVMName, err = p.configVarResolver.GetStringValue(rawConfig.TemplateVMName) if err != nil { return nil, nil, nil, err } //nolint:staticcheck //lint:ignore SA1019: rawConfig.VMNetName is deprecated: use networks instead. - c.VMNetName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VMNetName) + c.VMNetName, err = p.configVarResolver.GetStringValue(rawConfig.VMNetName) if err != nil { return nil, nil, nil, err } for _, network := range rawConfig.Networks { - networkValue, err := p.configVarResolver.GetConfigVarStringValue(network) + networkValue, err := p.configVarResolver.GetStringValue(network) if err != nil { return nil, nil, rawConfig, err } c.Networks = append(c.Networks, networkValue) } - c.Username, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Username, "VSPHERE_USERNAME") + c.Username, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Username, "VSPHERE_USERNAME") if err != nil { return nil, nil, nil, err } - c.Password, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Password, "VSPHERE_PASSWORD") + c.Password, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.Password, "VSPHERE_PASSWORD") if err != nil { return nil, nil, nil, err } - c.VSphereURL, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.VSphereURL, "VSPHERE_ADDRESS") + c.VSphereURL, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.VSphereURL, "VSPHERE_ADDRESS") if err != nil { return nil, nil, nil, err } - c.Datacenter, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Datacenter) + c.Datacenter, err = p.configVarResolver.GetStringValue(rawConfig.Datacenter) if err != nil { return nil, nil, nil, err } - c.Cluster, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Cluster) + c.Cluster, err = p.configVarResolver.GetStringValue(rawConfig.Cluster) if err != nil { return nil, nil, nil, err } - c.Folder, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Folder) + c.Folder, err = p.configVarResolver.GetStringValue(rawConfig.Folder) if err != nil { return nil, nil, nil, err } - c.ResourcePool, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ResourcePool) + c.ResourcePool, err = p.configVarResolver.GetStringValue(rawConfig.ResourcePool) if err != nil { return nil, nil, nil, err } - c.Datastore, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Datastore) + c.Datastore, err = p.configVarResolver.GetStringValue(rawConfig.Datastore) if err != nil { return nil, nil, nil, err } - c.DatastoreCluster, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.DatastoreCluster) + c.DatastoreCluster, err = p.configVarResolver.GetStringValue(rawConfig.DatastoreCluster) if err != nil { return nil, nil, nil, err } - c.AllowInsecure, err = p.configVarResolver.GetConfigVarBoolValueOrEnv(rawConfig.AllowInsecure, "VSPHERE_ALLOW_INSECURE") + c.AllowInsecure, err = p.configVarResolver.GetBoolValueOrEnv(rawConfig.AllowInsecure, "VSPHERE_ALLOW_INSECURE") if err != nil { return nil, nil, nil, err } - c.VMAntiAffinity, _, err = p.configVarResolver.GetConfigVarBoolValue(rawConfig.VMAntiAffinity) + c.VMAntiAffinity, _, err = p.configVarResolver.GetBoolValue(rawConfig.VMAntiAffinity) if err != nil { return nil, nil, nil, err } - c.VMGroup, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VMGroup) + c.VMGroup, err = p.configVarResolver.GetStringValue(rawConfig.VMGroup) if err != nil { return nil, nil, nil, err } diff --git a/pkg/cloudprovider/provider/vsphere/provider_test.go b/pkg/cloudprovider/provider/vsphere/provider_test.go index 33fa89e52..3bcdad6ed 100644 --- a/pkg/cloudprovider/provider/vsphere/provider_test.go +++ b/pkg/cloudprovider/provider/vsphere/provider_test.go @@ -28,7 +28,7 @@ import ( "go.uber.org/zap" cloudprovidertesting "k8c.io/machine-controller/pkg/cloudprovider/testing" - "k8c.io/machine-controller/sdk/providerconfig" + "k8c.io/machine-controller/sdk/providerconfig/configvar" "k8s.io/utils/ptr" fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -174,7 +174,7 @@ func TestValidate(t *testing.T) { password, _ := simulator.DefaultLogin.Password() p := &provider{ // Note that configVarResolver is not used in this test as the getConfigFunc is mocked. - configVarResolver: providerconfig.NewConfigVarResolver(context.Background(), fakectrlruntimeclient. + configVarResolver: configvar.NewResolver(context.Background(), fakectrlruntimeclient. NewClientBuilder(). Build()), } diff --git a/pkg/cloudprovider/provider/vultr/provider.go b/pkg/cloudprovider/provider/vultr/provider.go index f308943e1..3a4c77e37 100644 --- a/pkg/cloudprovider/provider/vultr/provider.go +++ b/pkg/cloudprovider/provider/vultr/provider.go @@ -55,11 +55,11 @@ type ValidVPC struct { } type provider struct { - configVarResolver *providerconfig.ConfigVarResolver + configVarResolver providerconfig.ConfigVarResolver } // New returns a new vultr provider. -func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { +func New(configVarResolver providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return &provider{configVarResolver: configVarResolver} } @@ -111,22 +111,22 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p c := Config{} - c.APIKey, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.APIKey, "VULTR_API_KEY") + c.APIKey, err = p.configVarResolver.GetStringValueOrEnv(rawConfig.APIKey, "VULTR_API_KEY") if err != nil { return nil, nil, fmt.Errorf("failed to get the value of \"apiKey\" field, error = %w", err) } - c.Plan, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Plan) + c.Plan, err = p.configVarResolver.GetStringValue(rawConfig.Plan) if err != nil { return nil, nil, err } - c.Region, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Region) + c.Region, err = p.configVarResolver.GetStringValue(rawConfig.Region) if err != nil { return nil, nil, err } - c.OsID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.OsID) + c.OsID, err = p.configVarResolver.GetStringValue(rawConfig.OsID) if err != nil { return nil, nil, err } diff --git a/pkg/controller/machine/controller.go b/pkg/controller/machine/controller.go index 00f57f68b..c7749980c 100644 --- a/pkg/controller/machine/controller.go +++ b/pkg/controller/machine/controller.go @@ -45,6 +45,7 @@ import ( clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" "k8c.io/machine-controller/sdk/bootstrap" "k8c.io/machine-controller/sdk/providerconfig" + "k8c.io/machine-controller/sdk/providerconfig/configvar" "k8c.io/machine-controller/sdk/userdata/rhel" corev1 "k8s.io/api/core/v1" @@ -409,8 +410,8 @@ func (r *Reconciler) reconcile(ctx context.Context, log *zap.SugaredLogger, mach if err != nil { return nil, fmt.Errorf("failed to get provider config: %w", err) } - skg := providerconfig.NewConfigVarResolver(ctx, r.client) - prov, err := cloudprovider.ForProvider(providerConfig.CloudProvider, skg) + configResolver := configvar.NewResolver(ctx, r.client) + prov, err := cloudprovider.ForProvider(providerConfig.CloudProvider, configResolver) if err != nil { return nil, fmt.Errorf("failed to get cloud provider %q: %w", providerConfig.CloudProvider, err) } diff --git a/pkg/controller/machine/metrics.go b/pkg/controller/machine/metrics.go index 64399c6dd..ff09db81d 100644 --- a/pkg/controller/machine/metrics.go +++ b/pkg/controller/machine/metrics.go @@ -26,6 +26,7 @@ import ( "k8c.io/machine-controller/pkg/cloudprovider" clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" "k8c.io/machine-controller/sdk/providerconfig" + "k8c.io/machine-controller/sdk/providerconfig/configvar" "k8s.io/apimachinery/pkg/api/equality" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -122,7 +123,7 @@ func (l *machineMetricLabels) Counter(value uint) prometheus.Counter { func NewMachineCollector(ctx context.Context, client ctrlruntimeclient.Client) *MachineCollector { // Start periodically calling the providers SetMetricsForMachines in a dedicated go routine - skg := providerconfig.NewConfigVarResolver(ctx, client) + configResolver := configvar.NewResolver(ctx, client) go func() { metricGatheringExecutor := func() { machines := &clusterv1alpha1.MachineList{} @@ -152,7 +153,7 @@ func NewMachineCollector(ctx context.Context, client ctrlruntimeclient.Client) * } for provider, providerMachineList := range providerMachineMap { - prov, err := cloudprovider.ForProvider(provider, skg) + prov, err := cloudprovider.ForProvider(provider, configResolver) if err != nil { utilruntime.HandleError(fmt.Errorf("failed to get cloud provider for SetMetricsForMachines:: %q: %w", provider, err)) continue @@ -205,7 +206,7 @@ func (mc MachineCollector) Collect(ch chan<- prometheus.Metric) { return } - cvr := providerconfig.NewConfigVarResolver(mc.ctx, mc.client) + configResolver := configvar.NewResolver(mc.ctx, mc.client) machineCountByLabels := make(map[*machineMetricLabels]uint) for _, machine := range machines.Items { @@ -231,7 +232,7 @@ func (mc MachineCollector) Collect(ch chan<- prometheus.Metric) { continue } - provider, err := cloudprovider.ForProvider(providerConfig.CloudProvider, cvr) + provider, err := cloudprovider.ForProvider(providerConfig.CloudProvider, configResolver) if err != nil { utilruntime.HandleError(fmt.Errorf("failed to determine provider provider: %w", err)) continue diff --git a/pkg/migrations/migrations.go b/pkg/migrations/migrations.go index bfebff658..85065d570 100644 --- a/pkg/migrations/migrations.go +++ b/pkg/migrations/migrations.go @@ -33,6 +33,7 @@ import ( "k8c.io/machine-controller/sdk/apis/machines" machinesv1alpha1 "k8c.io/machine-controller/sdk/apis/machines/v1alpha1" "k8c.io/machine-controller/sdk/providerconfig" + "k8c.io/machine-controller/sdk/providerconfig/configvar" corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" @@ -234,8 +235,8 @@ func migrateMachines(ctx context.Context, log *zap.SugaredLogger, client ctrlrun if err != nil { return fmt.Errorf("failed to get provider config: %w", err) } - skg := providerconfig.NewConfigVarResolver(ctx, client) - prov, err := cloudprovider.ForProvider(providerConfig.CloudProvider, skg) + configResolver := configvar.NewResolver(ctx, client) + prov, err := cloudprovider.ForProvider(providerConfig.CloudProvider, configResolver) if err != nil { return fmt.Errorf("failed to get cloud provider %q: %w", providerConfig.CloudProvider, err) } diff --git a/sdk/README.md b/sdk/README.md new file mode 100644 index 000000000..19febcf65 --- /dev/null +++ b/sdk/README.md @@ -0,0 +1,41 @@ +# machine-controller SDK + +This directory contains the `k8c.io/machine-controller/sdk` Go module. If you're +looking at integrating the machine controller (MC) into your application, this +is where you should start. + +## Usage + +Simply `go get` the SDK to use it in your application: + +```shell +go get k8c.io/machine-controller/sdk +``` + +If necessary, you can also import the main MC module, but this comes with heavy +dependencies that might be too costly to maintain for you: + +```shell +go get k8c.io/machine-controller +go get k8c.io/machine-controller/sdk +``` + +In this case it's recommended to always keep both dependencies on the exact same +version. + +## Development + +There are two main design criteria for the SDK: + +1. The SDK should contain a minimal set of dependencies, in a perfect world it + would be only Kube dependencies. The idea behind the SDK is to make importing + KKP cheap and easy and to not force dependencies onto consumers. + +1. The SDK should not contain as few functions as possible. Functions always + represent application logic and usually that logic should not be hardcoded into + client apps. Every function in the SDK is therefore to be considered "eternal". + +1. The SDK should truly follow the Go Modules idea of declaring the _minimum_ + compatible versions of every dependency and even of Go. The main machine + controller module can and should have the _latest_ dependencies, but the SDK + should not force consumers to be on the most recent Kube version, for example. diff --git a/sdk/providerconfig/configvar/resolver.go b/sdk/providerconfig/configvar/resolver.go new file mode 100644 index 000000000..b0b657ca8 --- /dev/null +++ b/sdk/providerconfig/configvar/resolver.go @@ -0,0 +1,173 @@ +/* +Copyright 2019 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package configvar + +import ( + "context" + "fmt" + "os" + "strconv" + "time" + + "k8c.io/machine-controller/sdk/providerconfig" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Resolver struct { + ctx context.Context + client ctrlruntimeclient.Client +} + +func NewResolver(ctx context.Context, client ctrlruntimeclient.Client) *Resolver { + return &Resolver{ + ctx: ctx, + client: client, + } +} + +var _ providerconfig.ConfigVarResolver = &Resolver{} + +func (r *Resolver) GetDurationValue(configVar providerconfig.ConfigVarString) (time.Duration, error) { + durStr, err := r.GetStringValue(configVar) + if err != nil { + return 0, err + } + + return time.ParseDuration(durStr) +} + +func (r *Resolver) GetDurationValueOrDefault(configVar providerconfig.ConfigVarString, defaultDuration time.Duration) (time.Duration, error) { + durStr, err := r.GetStringValue(configVar) + if err != nil { + return 0, err + } + + if durStr == "" { + return defaultDuration, nil + } + + duration, err := time.ParseDuration(durStr) + if err != nil { + return 0, err + } + + if duration <= 0 { + return defaultDuration, nil + } + + return duration, nil +} + +func (r *Resolver) GetStringValue(configVar providerconfig.ConfigVarString) (string, error) { + // We need all three of these to fetch and use a secret + if configVar.SecretKeyRef.Name != "" && configVar.SecretKeyRef.Namespace != "" && configVar.SecretKeyRef.Key != "" { + secret := &corev1.Secret{} + name := types.NamespacedName{Namespace: configVar.SecretKeyRef.Namespace, Name: configVar.SecretKeyRef.Name} + if err := r.client.Get(r.ctx, name, secret); err != nil { + return "", fmt.Errorf("error retrieving secret '%s' from namespace '%s': '%w'", configVar.SecretKeyRef.Name, configVar.SecretKeyRef.Namespace, err) + } + if val, ok := secret.Data[configVar.SecretKeyRef.Key]; ok { + return string(val), nil + } + return "", fmt.Errorf("secret '%s' in namespace '%s' has no key '%s'", configVar.SecretKeyRef.Name, configVar.SecretKeyRef.Namespace, configVar.SecretKeyRef.Key) + } + + // We need all three of these to fetch and use a configmap + if configVar.ConfigMapKeyRef.Name != "" && configVar.ConfigMapKeyRef.Namespace != "" && configVar.ConfigMapKeyRef.Key != "" { + configMap := &corev1.ConfigMap{} + name := types.NamespacedName{Namespace: configVar.ConfigMapKeyRef.Namespace, Name: configVar.ConfigMapKeyRef.Name} + if err := r.client.Get(r.ctx, name, configMap); err != nil { + return "", fmt.Errorf("error retrieving configmap '%s' from namespace '%s': '%w'", configVar.ConfigMapKeyRef.Name, configVar.ConfigMapKeyRef.Namespace, err) + } + if val, ok := configMap.Data[configVar.ConfigMapKeyRef.Key]; ok { + return val, nil + } + return "", fmt.Errorf("configmap '%s' in namespace '%s' has no key '%s'", configVar.ConfigMapKeyRef.Name, configVar.ConfigMapKeyRef.Namespace, configVar.ConfigMapKeyRef.Key) + } + + return configVar.Value, nil +} + +// GetStringValueOrEnv tries to get the value from ConfigVarString, when it fails, it falls back to +// getting the value from an environment variable specified by envVarName parameter. +func (r *Resolver) GetStringValueOrEnv(configVar providerconfig.ConfigVarString, envVarName string) (string, error) { + cfgVar, err := r.GetStringValue(configVar) + if err == nil && len(cfgVar) > 0 { + return cfgVar, err + } + + envVal, _ := os.LookupEnv(envVarName) + return envVal, nil +} + +// GetBoolValue returns a boolean from a ConfigVarBool. If there is no valid source for the boolean, +// the second bool returned will be false (to be able to differentiate between "false" and "unset"). +func (r *Resolver) GetBoolValue(configVar providerconfig.ConfigVarBool) (bool, bool, error) { + // We need all three of these to fetch and use a secret + if configVar.SecretKeyRef.Name != "" && configVar.SecretKeyRef.Namespace != "" && configVar.SecretKeyRef.Key != "" { + secret := &corev1.Secret{} + name := types.NamespacedName{Namespace: configVar.SecretKeyRef.Namespace, Name: configVar.SecretKeyRef.Name} + if err := r.client.Get(r.ctx, name, secret); err != nil { + return false, false, fmt.Errorf("error retrieving secret '%s' from namespace '%s': '%w'", configVar.SecretKeyRef.Name, configVar.SecretKeyRef.Namespace, err) + } + if val, ok := secret.Data[configVar.SecretKeyRef.Key]; ok { + boolVal, err := strconv.ParseBool(string(val)) + return boolVal, (err == nil), err + } + return false, false, fmt.Errorf("secret '%s' in namespace '%s' has no key '%s'", configVar.SecretKeyRef.Name, configVar.SecretKeyRef.Namespace, configVar.SecretKeyRef.Key) + } + + // We need all three of these to fetch and use a configmap + if configVar.ConfigMapKeyRef.Name != "" && configVar.ConfigMapKeyRef.Namespace != "" && configVar.ConfigMapKeyRef.Key != "" { + configMap := &corev1.ConfigMap{} + name := types.NamespacedName{Namespace: configVar.ConfigMapKeyRef.Namespace, Name: configVar.ConfigMapKeyRef.Name} + if err := r.client.Get(r.ctx, name, configMap); err != nil { + return false, false, fmt.Errorf("error retrieving configmap '%s' from namespace '%s': '%w'", configVar.ConfigMapKeyRef.Name, configVar.ConfigMapKeyRef.Namespace, err) + } + if val, ok := configMap.Data[configVar.ConfigMapKeyRef.Key]; ok { + boolVal, err := strconv.ParseBool(val) + return boolVal, (err == nil), err + } + return false, false, fmt.Errorf("configmap '%s' in namespace '%s' has no key '%s'", configVar.ConfigMapKeyRef.Name, configVar.ConfigMapKeyRef.Namespace, configVar.ConfigMapKeyRef.Key) + } + + if configVar.Value == nil { + return false, false, nil + } + + return configVar.Value != nil && *configVar.Value, true, nil +} + +func (r *Resolver) GetBoolValueOrEnv(configVar providerconfig.ConfigVarBool, envVarName string) (bool, error) { + boolVal, valid, err := r.GetBoolValue(configVar) + if valid && err == nil { + return boolVal, nil + } + + envVal, envValFound := os.LookupEnv(envVarName) + if envValFound { + envValBool, err := strconv.ParseBool(envVal) + if err != nil { + return false, err + } + return envValBool, nil + } + + return false, nil +} diff --git a/sdk/providerconfig/resolver.go b/sdk/providerconfig/resolver.go index fe6805ff6..7f07cb5c9 100644 --- a/sdk/providerconfig/resolver.go +++ b/sdk/providerconfig/resolver.go @@ -17,154 +17,14 @@ limitations under the License. package providerconfig import ( - "context" - "fmt" - "os" - "strconv" "time" - - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/types" - ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) -type ConfigVarResolver struct { - ctx context.Context - client ctrlruntimeclient.Client -} - -func NewConfigVarResolver(ctx context.Context, client ctrlruntimeclient.Client) *ConfigVarResolver { - return &ConfigVarResolver{ - ctx: ctx, - client: client, - } -} - -func (cvr *ConfigVarResolver) GetConfigVarDurationValue(configVar ConfigVarString) (time.Duration, error) { - durStr, err := cvr.GetConfigVarStringValue(configVar) - if err != nil { - return 0, err - } - - return time.ParseDuration(durStr) -} - -func (cvr *ConfigVarResolver) GetConfigVarDurationValueOrDefault(configVar ConfigVarString, defaultDuration time.Duration) (time.Duration, error) { - durStr, err := cvr.GetConfigVarStringValue(configVar) - if err != nil { - return 0, err - } - - if durStr == "" { - return defaultDuration, nil - } - - duration, err := time.ParseDuration(durStr) - if err != nil { - return 0, err - } - - if duration <= 0 { - return defaultDuration, nil - } - - return duration, nil -} - -func (cvr *ConfigVarResolver) GetConfigVarStringValue(configVar ConfigVarString) (string, error) { - // We need all three of these to fetch and use a secret - if configVar.SecretKeyRef.Name != "" && configVar.SecretKeyRef.Namespace != "" && configVar.SecretKeyRef.Key != "" { - secret := &corev1.Secret{} - name := types.NamespacedName{Namespace: configVar.SecretKeyRef.Namespace, Name: configVar.SecretKeyRef.Name} - if err := cvr.client.Get(cvr.ctx, name, secret); err != nil { - return "", fmt.Errorf("error retrieving secret '%s' from namespace '%s': '%w'", configVar.SecretKeyRef.Name, configVar.SecretKeyRef.Namespace, err) - } - if val, ok := secret.Data[configVar.SecretKeyRef.Key]; ok { - return string(val), nil - } - return "", fmt.Errorf("secret '%s' in namespace '%s' has no key '%s'", configVar.SecretKeyRef.Name, configVar.SecretKeyRef.Namespace, configVar.SecretKeyRef.Key) - } - - // We need all three of these to fetch and use a configmap - if configVar.ConfigMapKeyRef.Name != "" && configVar.ConfigMapKeyRef.Namespace != "" && configVar.ConfigMapKeyRef.Key != "" { - configMap := &corev1.ConfigMap{} - name := types.NamespacedName{Namespace: configVar.ConfigMapKeyRef.Namespace, Name: configVar.ConfigMapKeyRef.Name} - if err := cvr.client.Get(cvr.ctx, name, configMap); err != nil { - return "", fmt.Errorf("error retrieving configmap '%s' from namespace '%s': '%w'", configVar.ConfigMapKeyRef.Name, configVar.ConfigMapKeyRef.Namespace, err) - } - if val, ok := configMap.Data[configVar.ConfigMapKeyRef.Key]; ok { - return val, nil - } - return "", fmt.Errorf("configmap '%s' in namespace '%s' has no key '%s'", configVar.ConfigMapKeyRef.Name, configVar.ConfigMapKeyRef.Namespace, configVar.ConfigMapKeyRef.Key) - } - - return configVar.Value, nil -} - -// GetConfigVarStringValueOrEnv tries to get the value from ConfigVarString, when it fails, it falls back to -// getting the value from an environment variable specified by envVarName parameter. -func (cvr *ConfigVarResolver) GetConfigVarStringValueOrEnv(configVar ConfigVarString, envVarName string) (string, error) { - cfgVar, err := cvr.GetConfigVarStringValue(configVar) - if err == nil && len(cfgVar) > 0 { - return cfgVar, err - } - - envVal, _ := os.LookupEnv(envVarName) - return envVal, nil -} - -// GetConfigVarBoolValue returns a boolean from a ConfigVarBool. If there is no valid source for the boolean, -// the second bool returned will be false (to be able to differentiate between "false" and "unset"). -func (cvr *ConfigVarResolver) GetConfigVarBoolValue(configVar ConfigVarBool) (bool, bool, error) { - // We need all three of these to fetch and use a secret - if configVar.SecretKeyRef.Name != "" && configVar.SecretKeyRef.Namespace != "" && configVar.SecretKeyRef.Key != "" { - secret := &corev1.Secret{} - name := types.NamespacedName{Namespace: configVar.SecretKeyRef.Namespace, Name: configVar.SecretKeyRef.Name} - if err := cvr.client.Get(cvr.ctx, name, secret); err != nil { - return false, false, fmt.Errorf("error retrieving secret '%s' from namespace '%s': '%w'", configVar.SecretKeyRef.Name, configVar.SecretKeyRef.Namespace, err) - } - if val, ok := secret.Data[configVar.SecretKeyRef.Key]; ok { - boolVal, err := strconv.ParseBool(string(val)) - return boolVal, (err == nil), err - } - return false, false, fmt.Errorf("secret '%s' in namespace '%s' has no key '%s'", configVar.SecretKeyRef.Name, configVar.SecretKeyRef.Namespace, configVar.SecretKeyRef.Key) - } - - // We need all three of these to fetch and use a configmap - if configVar.ConfigMapKeyRef.Name != "" && configVar.ConfigMapKeyRef.Namespace != "" && configVar.ConfigMapKeyRef.Key != "" { - configMap := &corev1.ConfigMap{} - name := types.NamespacedName{Namespace: configVar.ConfigMapKeyRef.Namespace, Name: configVar.ConfigMapKeyRef.Name} - if err := cvr.client.Get(cvr.ctx, name, configMap); err != nil { - return false, false, fmt.Errorf("error retrieving configmap '%s' from namespace '%s': '%w'", configVar.ConfigMapKeyRef.Name, configVar.ConfigMapKeyRef.Namespace, err) - } - if val, ok := configMap.Data[configVar.ConfigMapKeyRef.Key]; ok { - boolVal, err := strconv.ParseBool(val) - return boolVal, (err == nil), err - } - return false, false, fmt.Errorf("configmap '%s' in namespace '%s' has no key '%s'", configVar.ConfigMapKeyRef.Name, configVar.ConfigMapKeyRef.Namespace, configVar.ConfigMapKeyRef.Key) - } - - if configVar.Value == nil { - return false, false, nil - } - - return configVar.Value != nil && *configVar.Value, true, nil -} - -func (cvr *ConfigVarResolver) GetConfigVarBoolValueOrEnv(configVar ConfigVarBool, envVarName string) (bool, error) { - boolVal, valid, err := cvr.GetConfigVarBoolValue(configVar) - if valid && err == nil { - return boolVal, nil - } - - envVal, envValFound := os.LookupEnv(envVarName) - if envValFound { - envValBool, err := strconv.ParseBool(envVal) - if err != nil { - return false, err - } - return envValBool, nil - } - - return false, nil +type ConfigVarResolver interface { + GetDurationValue(configVar ConfigVarString) (time.Duration, error) + GetDurationValueOrDefault(configVar ConfigVarString, defaultDuration time.Duration) (time.Duration, error) + GetStringValue(configVar ConfigVarString) (string, error) + GetStringValueOrEnv(configVar ConfigVarString, envVarName string) (string, error) + GetBoolValue(configVar ConfigVarBool) (bool, bool, error) + GetBoolValueOrEnv(configVar ConfigVarBool, envVarName string) (bool, error) } diff --git a/test/e2e/provisioning/migrateuidscenario.go b/test/e2e/provisioning/migrateuidscenario.go index ffb092431..556ca65d6 100644 --- a/test/e2e/provisioning/migrateuidscenario.go +++ b/test/e2e/provisioning/migrateuidscenario.go @@ -29,7 +29,8 @@ import ( cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" - providerconfig "k8c.io/machine-controller/sdk/providerconfig" + "k8c.io/machine-controller/sdk/providerconfig" + "k8c.io/machine-controller/sdk/providerconfig/configvar" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -79,7 +80,7 @@ func verifyMigrateUID(ctx context.Context, _, manifestPath string, parameters [] if err != nil { return fmt.Errorf("failed to get provideSpec: %w", err) } - skg := providerconfig.NewConfigVarResolver(ctx, fakeClient) + skg := configvar.NewResolver(ctx, fakeClient) prov, err := cloudprovider.ForProvider(providerSpec.CloudProvider, skg) if err != nil { return fmt.Errorf("failed to get cloud provider %q: %w", providerSpec.CloudProvider, err) From 744bbba4cc7bb45a95fd421c5186f63e58d1093f Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Fri, 4 Apr 2025 15:52:07 +0200 Subject: [PATCH 466/489] Update to Go 1.24.2, golangci-lint 2.x (#1914) * migrate golangci-lint config * adjust code to new linter settings * bump to Go 1.24.2 * treat the SDK the same way --- .gimps.yaml | 2 +- .golangci.yml | 146 ++++++++++-------- .prow/e2e-features.yaml | 8 +- .prow/postsubmits.yaml | 4 +- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 12 +- .prow/provider-azure.yaml | 6 +- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 +- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 10 +- .prow/verify.yaml | 16 +- Dockerfile | 2 +- Makefile | 2 +- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- pkg/admission/admission.go | 2 +- pkg/admission/machines.go | 5 +- pkg/cloudprovider/provider/anexia/provider.go | 2 +- pkg/cloudprovider/provider/aws/provider.go | 6 +- .../provider/azure/create_delete_resources.go | 6 +- pkg/cloudprovider/provider/azure/provider.go | 4 +- .../provider/hetzner/provider.go | 4 +- .../provider/kubevirt/provider.go | 8 +- .../provider/kubevirt/provider_test.go | 2 +- pkg/cloudprovider/provider/linode/provider.go | 2 +- .../provider/openstack/provider.go | 2 +- .../provider/openstack/provider_test.go | 2 +- .../provider/vmwareclouddirector/client.go | 2 +- .../provider/vmwareclouddirector/helper.go | 2 +- pkg/cloudprovider/provider/vsphere/helper.go | 13 +- .../provider/vsphere/provider.go | 8 +- pkg/cloudprovider/provider/vsphere/rule.go | 2 +- pkg/cloudprovider/provider/vsphere/vmgroup.go | 2 +- pkg/clusterinfo/configmap.go | 6 +- .../machinedeployment/controller.go | 14 +- pkg/controller/machinedeployment/sync.go | 2 +- pkg/controller/machineset/controller.go | 28 ++-- pkg/controller/machineset/delete_policy.go | 10 +- pkg/controller/machineset/machine.go | 2 +- pkg/controller/machineset/status.go | 2 +- pkg/controller/nodecsrapprover/controller.go | 2 +- pkg/migrations/migrations.go | 2 +- pkg/node/eviction/eviction.go | 2 +- sdk/.golangci.yml | 42 +++-- .../v1alpha1/conversions/conversions.go | 2 +- sdk/net/net.go | 4 +- sdk/providerconfig/types.go | 30 ++-- test/e2e/provisioning/helper.go | 4 +- test/e2e/provisioning/verify.go | 6 +- 58 files changed, 251 insertions(+), 219 deletions(-) diff --git a/.gimps.yaml b/.gimps.yaml index 7856ca9c5..fd0b820ba 100644 --- a/.gimps.yaml +++ b/.gimps.yaml @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This is the configuration for https://github.com/xrstf/gimps. +# This is the configuration for https://codeberg.org/xrstf/gimps. importOrder: [std, external, kubermatic, kubernetes] sets: diff --git a/.golangci.yml b/.golangci.yml index 8c9c3df8c..3f32b1828 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,8 +1,9 @@ +version: "2" run: - timeout: 20m build-tags: - e2e linters: + default: none enable: - asciicheck - bidichk @@ -15,8 +16,6 @@ linters: - goconst - gocyclo - godot - - gofmt - - gosimple - govet - importas - ineffassign @@ -27,73 +26,90 @@ linters: - nosprintfhostport - predeclared - promlinter - - revive - staticcheck - - tenv - unconvert - unused - wastedassign - whitespace - disable-all: true - -linters-settings: - depguard: - rules: - main: - deny: - - { pkg: io/ioutil, desc: https://go.dev/doc/go1.16#ioutil } - - { pkg: github.com/ghodss/yaml, desc: use sigs.k8s.io/yaml instead } - - revive: + settings: + depguard: + rules: + main: + deny: + - pkg: io/ioutil + desc: https://go.dev/doc/go1.16#ioutil + - pkg: github.com/ghodss/yaml + desc: use sigs.k8s.io/yaml instead + govet: + enable: + - nilness # find tautologies / impossible conditions + importas: + alias: + # Machine Controller + - pkg: k8c.io/machine-controller/sdk/apis/(\w+)/(v[\w\d]+) + alias: $1$2 + # Kubernetes + - pkg: k8s.io/api/(\w+)/(v[\w\d]+) + alias: $1$2 + - pkg: k8s.io/apimachinery/pkg/apis/meta/v1 + alias: metav1 + - pkg: k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 + alias: apiextensionsv1 + - pkg: k8s.io/apimachinery/pkg/api/errors + alias: apierrors + - pkg: k8s.io/apimachinery/pkg/util/errors + alias: kerrors + # Controller Runtime + - pkg: sigs.k8s.io/controller-runtime/pkg/client + alias: ctrlruntimeclient + # Other Kube APIs + - pkg: go.anx.io/go-anxcloud/pkg/apis/(\w+)/(v[\w\d]+) + alias: anx$1$2 + - pkg: github.com/tinkerbell/tink/api/(v[\w\d]+) + alias: tink$1 + - pkg: kubevirt.io/api/(\w+)/(v[\w\d]+) + alias: kubevirt$1$2 + - pkg: kubevirt.io/containerized-data-importer-api/pkg/apis/(\w+)/(v[\w\d]+) + alias: cdi$1$2 + no-unaliased: true + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling rules: - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#add-constant - - name: duplicated-imports - severity: warning - - govet: - enable: - - nilness # find tautologies / impossible conditions - - importas: - no-unaliased: true - alias: - # Machine Controller - - pkg: k8c.io/machine-controller/sdk/apis/(\w+)/(v[\w\d]+) - alias: $1$2 - # Kubernetes - - pkg: k8s.io/api/(\w+)/(v[\w\d]+) - alias: $1$2 - - pkg: k8s.io/apimachinery/pkg/apis/meta/v1 - alias: metav1 - - pkg: k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 - alias: apiextensionsv1 - - pkg: k8s.io/apimachinery/pkg/api/errors - alias: apierrors - - pkg: k8s.io/apimachinery/pkg/util/errors - alias: kerrors - # Controller Runtime - - pkg: sigs.k8s.io/controller-runtime/pkg/client - alias: ctrlruntimeclient - # Other Kube APIs - - pkg: go.anx.io/go-anxcloud/pkg/apis/(\w+)/(v[\w\d]+) - alias: anx$1$2 - - pkg: github.com/tinkerbell/tink/api/(v[\w\d]+) - alias: tink$1 - - pkg: kubevirt.io/api/(\w+)/(v[\w\d]+) - alias: kubevirt$1$2 - - pkg: kubevirt.io/containerized-data-importer-api/pkg/apis/(\w+)/(v[\w\d]+) - alias: cdi$1$2 - + - path: (.+)\.go$ + text: func Convert_MachinesV1alpha1Machine_To_ClusterV1alpha1Machine should be ConvertMachinesV1alpha1MachineToClusterV1alpha1Machine + - path: (.+)\.go$ + text: func Convert_MachineDeployment_ProviderConfig_To_ProviderSpec should be ConvertMachineDeploymentProviderConfigToProviderSpec + - path: (.+)\.go$ + text: func Convert_MachineSet_ProviderConfig_To_ProviderSpec should be ConvertMachineSetProviderConfigToProviderSpec + - path: (.+)\.go$ + text: func Convert_Machine_ProviderConfig_To_ProviderSpec should be ConvertMachineProviderConfigToProviderSpec + - path: (.+)\.go$ + text: cyclomatic complexity [0-9]+ of func `\(\*provider\)\.Create` is high + - path: (.+)\.go$ + text: cyclomatic complexity [0-9]+ of func `\(\*provider\)\.Validate` is high + - path: (.+)\.go$ + text: cyclomatic complexity [0-9]+ of func `\(\*provider\)\.getConfig` is high + - path: (.+)\.go$ + text: 'SA1019: s.server.IPv6 is deprecated' + paths: + - apis/machines + - third_party$ + - builtin$ + - examples$ issues: max-same-issues: 0 - exclude: - - func Convert_MachinesV1alpha1Machine_To_ClusterV1alpha1Machine should be ConvertMachinesV1alpha1MachineToClusterV1alpha1Machine - - func Convert_MachineDeployment_ProviderConfig_To_ProviderSpec should be ConvertMachineDeploymentProviderConfigToProviderSpec - - func Convert_MachineSet_ProviderConfig_To_ProviderSpec should be ConvertMachineSetProviderConfigToProviderSpec - - func Convert_Machine_ProviderConfig_To_ProviderSpec should be ConvertMachineProviderConfigToProviderSpec - - 'cyclomatic complexity [0-9]+ of func `\(\*provider\)\.Create` is high' - - 'cyclomatic complexity [0-9]+ of func `\(\*provider\)\.Validate` is high' - - 'cyclomatic complexity [0-9]+ of func `\(\*provider\)\.getConfig` is high' - - "SA1019: s.server.IPv6 is deprecated" - exclude-dirs: - - apis/machines +formatters: + enable: + - gofmt + exclusions: + generated: lax + paths: + - apis/machines + - third_party$ + - builtin$ + - examples$ diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index f9304adc7..0dcaf5103 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -35,7 +35,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -65,7 +65,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -95,7 +95,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -123,7 +123,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 3cb4f79b7..8ff4a990b 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -27,7 +27,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - /bin/bash - -c @@ -56,7 +56,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index a309a8abe..a1461b83c 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 5e03de204..aff05124a 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -32,7 +32,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index b6db901d0..68eaa6b41 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -96,7 +96,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -130,7 +130,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -162,7 +162,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -194,7 +194,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 3609bdc69..551e1ba2f 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -96,7 +96,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 00bb8c572..6da6e33b8 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index e127c6d88..fff89c2e1 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index a1e22dbf3..322d38ad4 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 39d0b0b8f..62501fc53 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index effd294f2..bec18fafc 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index df54861d5..2e615e29f 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 1c5a9e83d..8e90f0a72 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 4976b1a32..e02483e16 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -64,7 +64,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 870680c68..ec467e703 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 0c8cfccdc..ffaaed035 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -32,7 +32,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 96a4b5b86..096d986b1 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -95,7 +95,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -128,7 +128,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -161,7 +161,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-kind-0.26-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 6fcb09b28..20ada4993 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -22,7 +22,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-3 command: - make args: @@ -44,7 +44,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-3 command: - make args: @@ -66,7 +66,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-3 command: - make args: @@ -87,7 +87,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-3 command: - make args: @@ -107,7 +107,7 @@ presubmits: path_alias: k8c.io/machine-controller spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-3 command: - "/usr/local/bin/shfmt" args: @@ -136,7 +136,7 @@ presubmits: path_alias: k8c.io/machine-controller spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-3 command: - "./hack/verify-boilerplate.sh" resources: @@ -156,7 +156,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-3 command: - ./hack/verify-licenses.sh resources: @@ -173,7 +173,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.23-node-20-10 + - image: quay.io/kubermatic/build:go-1.24-node-20-3 command: - make args: diff --git a/Dockerfile b/Dockerfile index 7186070b5..85ec085a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.23.7 +ARG GO_VERSION=1.24.2 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/k8c.io/machine-controller COPY . . diff --git a/Makefile b/Makefile index 881a4013a..f27111938 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.23.7 +GO_VERSION ?= 1.24.2 GOOS ?= $(shell go env GOOS) diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 38b8782f6..667d517ec 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.23-node-20-10 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.24-node-20-3 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 66b1f5741..6ad5f712b 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.23-node-20-10 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.24-node-20-3 containerize ./hack/verify-licenses.sh go mod vendor diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go index ddb8b486b..a8ed057ab 100644 --- a/pkg/admission/admission.go +++ b/pkg/admission/admission.go @@ -169,7 +169,7 @@ func handleFuncFactory(log *zap.SugaredLogger, mutate mutator) http.HandlerFunc // proper AdmissionReview responses require metadata that is not available // in broken requests, so we return a basic failure response w.WriteHeader(http.StatusBadRequest) - if _, err := w.Write([]byte(fmt.Sprintf("invalid request: %v", err))); err != nil { + if _, err := fmt.Fprintf(w, "invalid request: %v", err); err != nil { log.Errorw("Failed to write badRequest", zap.Error(err)) } return diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index 62cf06f0b..b371529d2 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -19,6 +19,7 @@ package admission import ( "context" "encoding/json" + "errors" "fmt" "github.com/Masterminds/semver/v3" @@ -143,7 +144,7 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec // Check kubelet version if spec.Versions.Kubelet == "" { - return fmt.Errorf("Kubelet version must be set") + return errors.New("kubelet version must be set") } kubeletVer, err := semver.NewVersion(spec.Versions.Kubelet) @@ -162,7 +163,7 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec // Validate SSH keys if err := validatePublicKeys(providerConfig.SSHPublicKeys); err != nil { - return fmt.Errorf("Invalid public keys specified: %w", err) + return fmt.Errorf("invalid public keys specified: %w", err) } defaultedOperatingSystemSpec, err := userdata.DefaultOperatingSystemSpec( diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 048653f92..e0a5070ba 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -412,7 +412,7 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine var respErr *anxclient.ResponseError // Only error if the error was not "not found" - if !(errors.As(err, &respErr) && respErr.ErrorData.Code == http.StatusNotFound) { + if !errors.As(err, &respErr) || respErr.ErrorData.Code != http.StatusNotFound { return false, newError(common.DeleteMachineError, "failed to delete machine: %v", err) } diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index e8d76480c..cee0d668c 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -408,12 +408,12 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if c.DiskType == ec2types.VolumeTypeIo1 { if rawConfig.DiskIops == nil { - return nil, nil, nil, errors.New("Missing required field `diskIops`") + return nil, nil, nil, errors.New("missing required field `diskIops`") } iops := *rawConfig.DiskIops if iops < 100 || iops > 64000 { - return nil, nil, nil, errors.New("Invalid value for `diskIops` (min: 100, max: 64000)") + return nil, nil, nil, errors.New("invalid value for `diskIops` (min: 100, max: 64000)") } c.DiskIops = rawConfig.DiskIops @@ -422,7 +422,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p iops := *rawConfig.DiskIops if iops < 3000 || iops > 64000 { - return nil, nil, nil, errors.New("Invalid value for `diskIops` (min: 3000, max: 64000)") + return nil, nil, nil, errors.New("invalid value for `diskIops` (min: 3000, max: 64000)") } c.DiskIops = rawConfig.DiskIops diff --git a/pkg/cloudprovider/provider/azure/create_delete_resources.go b/pkg/cloudprovider/provider/azure/create_delete_resources.go index 34742a0ae..4e76f90b2 100644 --- a/pkg/cloudprovider/provider/azure/create_delete_resources.go +++ b/pkg/cloudprovider/provider/azure/create_delete_resources.go @@ -339,7 +339,7 @@ func createOrUpdateNetworkInterface(ctx context.Context, log *zap.SugaredLogger, Tags: map[string]*string{machineUIDTag: to.StringPtr(string(machineUID))}, } - *ifSpec.InterfacePropertiesFormat.IPConfigurations = append(*ifSpec.InterfacePropertiesFormat.IPConfigurations, network.InterfaceIPConfiguration{ + *ifSpec.IPConfigurations = append(*ifSpec.IPConfigurations, network.InterfaceIPConfiguration{ Name: to.StringPtr("ip-config-1"), InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ Subnet: &subnet, @@ -350,7 +350,7 @@ func createOrUpdateNetworkInterface(ctx context.Context, log *zap.SugaredLogger, }) if ipFamily.IsDualstack() { - *ifSpec.InterfacePropertiesFormat.IPConfigurations = append(*ifSpec.InterfacePropertiesFormat.IPConfigurations, network.InterfaceIPConfiguration{ + *ifSpec.IPConfigurations = append(*ifSpec.IPConfigurations, network.InterfaceIPConfiguration{ Name: to.StringPtr("ip-config-2"), InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ PrivateIPAllocationMethod: network.Dynamic, @@ -362,7 +362,7 @@ func createOrUpdateNetworkInterface(ctx context.Context, log *zap.SugaredLogger, }) } - ifSpec.InterfacePropertiesFormat.EnableAcceleratedNetworking = enableAcceleratedNetworking + ifSpec.EnableAcceleratedNetworking = enableAcceleratedNetworking if config.SecurityGroupName != "" { authorizer, err := auth.NewClientCredentialsConfig(config.ClientID, config.ClientSecret, config.TenantID).Authorizer() diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 57db40ec3..d4971ee16 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -387,7 +387,7 @@ func getVMIPAddresses(ctx context.Context, log *zap.SugaredLogger, c *config, vm return nil, fmt.Errorf("machine is missing properties") } - if vm.VirtualMachineProperties.NetworkProfile == nil { + if vm.NetworkProfile == nil { return nil, fmt.Errorf("machine has no network profile") } @@ -689,7 +689,7 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * config.AssignAvailabilitySet != nil && *config.AssignAvailabilitySet && config.AvailabilitySet != "" { // Azure expects the full path to the resource asURI := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/availabilitySets/%s", config.SubscriptionID, config.ResourceGroup, config.AvailabilitySet) - vmSpec.VirtualMachineProperties.AvailabilitySet = &compute.SubResource{ID: to.StringPtr(asURI)} + vmSpec.AvailabilitySet = &compute.SubResource{ID: to.StringPtr(asURI)} } if config.EnableBootDiagnostics { diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index f5ea57937..797a8ab90 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -512,8 +512,8 @@ func (p *provider) MigrateUID(ctx context.Context, log *zap.SugaredLogger, machi if err != nil { return fmt.Errorf("failed to update UID label: %w", err) } - if response.Response.StatusCode != http.StatusOK { - return fmt.Errorf("got unexpected response code %v, expected %v", response.Response.Status, http.StatusOK) + if response.StatusCode != http.StatusOK { + return fmt.Errorf("got unexpected response code %v, expected %v", response.Status, http.StatusOK) } // This succeeds, but does not result in a label on the server, seems to be a bug // on Hetzner side diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 9e0cca504..f59e369e9 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -628,15 +628,15 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus // Values will come from instancetype. if c.Instancetype == nil { if c.Resources == nil { - return fmt.Errorf("no resource requests set for the virtual machine") + return errors.New("no resource requests set for the virtual machine") } if c.VCPUs == nil && c.Resources.Cpu().IsZero() { - return fmt.Errorf("no CPUs configured. Either vCPUs or CPUs have to be set.") + return errors.New("no CPUs configured. Either vCPUs or CPUs have to be set") } if c.VCPUs != nil && !c.Resources.Cpu().IsZero() { - return fmt.Errorf("vCPUs and CPUs cannot be configured at the same time.") + return errors.New("vCPUs and CPUs cannot be configured at the same time") } } @@ -649,7 +649,7 @@ func (p *provider) Validate(ctx context.Context, _ *zap.SugaredLogger, spec clus } if c.DNSPolicy == corev1.DNSNone { if c.DNSConfig == nil || len(c.DNSConfig.Nameservers) == 0 { - return fmt.Errorf("dns config must be specified when dns policy is None") + return errors.New("dns config must be specified when dns policy is None") } } // Check if we can reach the API of the target cluster. diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index dd925cff7..f2d5518e2 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -318,7 +318,7 @@ func TestNewVirtualMachine(t *testing.T) { // Check the created VirtualMachine vm, _ := p.newVirtualMachine(c, pc, machine, labels, "udsn", userdata, fakeMachineDeploymentNameAndRevisionForMachineGetter()) - vm.TypeMeta.APIVersion, vm.TypeMeta.Kind = kubevirtcorev1.VirtualMachineGroupVersionKind.ToAPIVersionAndKind() + vm.APIVersion, vm.Kind = kubevirtcorev1.VirtualMachineGroupVersionKind.ToAPIVersionAndKind() if !equality.Semantic.DeepEqual(vm, expectedVms[tt.name]) { t.Errorf("Diff %v", diff.ObjectGoPrintDiff(expectedVms[tt.name], vm)) diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index 7eb2c2957..dc0a9e39f 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -195,7 +195,7 @@ func createRandomPassword() (string, error) { rawRootPass := make([]byte, 50) _, err := rand.Read(rawRootPass) if err != nil { - return "", fmt.Errorf("Failed to generate random password") + return "", fmt.Errorf("failed to generate random password: %w", err) } rootPass := base64.StdEncoding.EncodeToString(rawRootPass) return rootPass, nil diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index b7748f75f..7fcb69ded 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -603,7 +603,7 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * log.Infow("Creating security group for worker nodes", "group", securityGroupName) err = ensureKubernetesSecurityGroupExist(log, client, cfg.Region, securityGroupName) if err != nil { - return nil, fmt.Errorf("Error occurred creating security groups: %w", err) + return nil, fmt.Errorf("error occurred creating security groups: %w", err) } securityGroups = append(securityGroups, securityGroupName) } diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index 4491f77a6..c85ec0136 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -280,7 +280,7 @@ func TestCreateServer(t *testing.T) { clientGetter: func(*Config) (*gophercloud.ProviderClient, error) { pc := client.ServiceClient() // endpoint locator used to redirect to local test endpoint - pc.ProviderClient.EndpointLocator = func(_ gophercloud.EndpointOpts) (string, error) { + pc.EndpointLocator = func(_ gophercloud.EndpointOpts) (string, error) { return pc.Endpoint, nil } return pc.ProviderClient, nil diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/client.go b/pkg/cloudprovider/provider/vmwareclouddirector/client.go index a628c660f..256ea07de 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/client.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/client.go @@ -120,7 +120,7 @@ func (c *Client) GetOrganization() (*govcd.Org, error) { func (c *Client) GetVDCForOrg(org govcd.Org) (*govcd.Vdc, error) { if c.Auth.VDC == "" { - return nil, errors.New("Organization VDC must be configured") + return nil, errors.New("organization VDC must be configured") } vcd, err := org.GetVDCByNameOrId(c.Auth.VDC, false) if err != nil { diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go index 1078e13d7..9883e1ca4 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go @@ -164,7 +164,7 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * apiEndpoint, err := url.Parse(vapp.VApp.HREF) if err != nil { - return fmt.Errorf("error getting vapp href '%s': %w", c.Auth.URL, err) + return fmt.Errorf("error getting vApp href '%s': %w", c.URL, err) } apiEndpoint.Path = path.Join(apiEndpoint.Path, "action/recomposeVApp") diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go index f6b6000ab..e1abcf4d1 100644 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ b/pkg/cloudprovider/provider/vsphere/helper.go @@ -22,7 +22,6 @@ import ( "encoding/base64" "errors" "fmt" - "math" "os" "os/exec" "text/template" @@ -41,6 +40,8 @@ const ( localTempDir = "/tmp" metaDataTemplate = `instance-id: {{ .InstanceID}} local-hostname: {{ .Hostname }}` + + gigaByte = (1024 * 1024 * 1024) ) func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, config *Config, session *Session, containerLinuxUserdata string) (*object.VirtualMachine, error) { @@ -181,7 +182,7 @@ func createClonedVM(ctx context.Context, log *zap.SugaredLogger, vmName string, log.Debugw("Increasing disk size", "targetgb", *config.DiskSizeGB) disk := disks[0] - disk.CapacityInBytes = *config.DiskSizeGB * int64(math.Pow(1024, 3)) + disk.CapacityInBytes = *config.DiskSizeGB * gigaByte diskspec := &types.VirtualDeviceConfigSpec{Operation: types.VirtualDeviceConfigSpecOperationEdit, Device: disk} deviceSpecs = append(deviceSpecs, diskspec) } @@ -418,9 +419,9 @@ func validateDiskResizing(disks []*types.VirtualDisk, requestedSize int64) error if diskLen := len(disks); diskLen != 1 { return fmt.Errorf("expected vm to have exactly one disk, got %d", diskLen) } - requestedCapacityInBytes := requestedSize * int64(math.Pow(1024, 3)) + requestedCapacityInBytes := requestedSize * gigaByte if requestedCapacityInBytes < disks[0].CapacityInBytes { - attachedDiskSizeInGiB := disks[0].CapacityInBytes / int64(math.Pow(1024, 3)) + attachedDiskSizeInGiB := disks[0].CapacityInBytes / gigaByte return fmt.Errorf("requested diskSizeGB %d is smaller than size of attached disk(%dGiB)", requestedSize, attachedDiskSizeInGiB) } return nil @@ -431,12 +432,12 @@ func getDatastoreFromVM(ctx context.Context, session *Session, vmRef *object.Vir var props mo.VirtualMachine // Obtain VM properties if err := vmRef.Properties(ctx, vmRef.Reference(), nil, &props); err != nil { - return nil, fmt.Errorf("error getting VM properties: %w", err) + return nil, fmt.Errorf("failed to get VM properties: %w", err) } datastorePathObj := new(object.DatastorePath) isSuccess := datastorePathObj.FromString(props.Summary.Config.VmPathName) if !isSuccess { - return nil, fmt.Errorf("Failed to parse volPath: %s", props.Summary.Config.VmPathName) + return nil, fmt.Errorf("failed to parse volPath: %s", props.Summary.Config.VmPathName) } return session.Finder.Datastore(ctx, datastorePathObj.Datastore) } diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index dd7fbc84e..31707cc63 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -509,14 +509,14 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine datastore, err := getDatastoreFromVM(ctx, session, virtualMachine) if err != nil { - return false, fmt.Errorf("Error getting datastore from VM %s: %w", virtualMachine.Name(), err) + return false, fmt.Errorf("failed to get datastore from VM %s: %w", virtualMachine.Name(), err) } destroyTask, err := virtualMachine.Destroy(ctx) if err != nil { - return false, fmt.Errorf("failed to destroy vm %s: %w", virtualMachine.Name(), err) + return false, fmt.Errorf("failed to destroy VM %s: %w", virtualMachine.Name(), err) } if err := destroyTask.WaitEx(ctx); err != nil { - return false, fmt.Errorf("failed to destroy vm %s: %w", virtualMachine.Name(), err) + return false, fmt.Errorf("failed to destroy VM %s: %w", virtualMachine.Name(), err) } if pc.OperatingSystem != providerconfig.OperatingSystemFlatcar { @@ -530,7 +530,7 @@ func (p *provider) Cleanup(ctx context.Context, log *zap.SugaredLogger, machine } } - log.Infow("Successfully destroyed vm", "vm", virtualMachine.Name()) + log.Infow("Successfully destroyed VM", "vm", virtualMachine.Name()) return true, nil } diff --git a/pkg/cloudprovider/provider/vsphere/rule.go b/pkg/cloudprovider/provider/vsphere/rule.go index fdb1f6d67..9140c9807 100644 --- a/pkg/cloudprovider/provider/vsphere/rule.go +++ b/pkg/cloudprovider/provider/vsphere/rule.go @@ -59,7 +59,7 @@ func (p *provider) createOrUpdateVMAntiAffinityRule(ctx context.Context, log *za var ruleVMRef []types.ManagedObjectReference for _, vm := range vmsInFolder { // Only add VMs with the same machineSetName to the rule and exclude the machine itself if it is being deleted - if strings.HasPrefix(vm.Name(), machineSetName) && !(vm.Name() == machine.Name && machine.DeletionTimestamp != nil) { + if strings.HasPrefix(vm.Name(), machineSetName) && (vm.Name() != machine.Name || machine.DeletionTimestamp == nil) { ruleVMRef = append(ruleVMRef, vm.Reference()) } } diff --git a/pkg/cloudprovider/provider/vsphere/vmgroup.go b/pkg/cloudprovider/provider/vsphere/vmgroup.go index 5cea8f01c..ad23e14cc 100644 --- a/pkg/cloudprovider/provider/vsphere/vmgroup.go +++ b/pkg/cloudprovider/provider/vsphere/vmgroup.go @@ -48,7 +48,7 @@ func (p *provider) addToVMGroup(ctx context.Context, log *zap.SugaredLogger, ses var vmRefs []types.ManagedObjectReference for _, vm := range vmsInFolder { // Only add VMs with the same machineSetName to the rule and exclude the machine itself if it is being deleted - if strings.HasPrefix(vm.Name(), machineSetName) && !(vm.Name() == machine.Name && machine.DeletionTimestamp != nil) { + if strings.HasPrefix(vm.Name(), machineSetName) && (vm.Name() != machine.Name || machine.DeletionTimestamp == nil) { vmRefs = append(vmRefs, vm.Reference()) } } diff --git a/pkg/clusterinfo/configmap.go b/pkg/clusterinfo/configmap.go index 116e39416..daae2d348 100644 --- a/pkg/clusterinfo/configmap.go +++ b/pkg/clusterinfo/configmap.go @@ -130,11 +130,11 @@ func getSecurePort(endpointSubset corev1.EndpointSubset) *corev1.EndpointPort { } func getCAData(config *rest.Config) ([]byte, error) { - if len(config.TLSClientConfig.CAData) > 0 { - return config.TLSClientConfig.CAData, nil + if len(config.CAData) > 0 { + return config.CAData, nil } - return os.ReadFile(config.TLSClientConfig.CAFile) + return os.ReadFile(config.CAFile) } func (p *KubeconfigProvider) GetBearerToken() string { diff --git a/pkg/controller/machinedeployment/controller.go b/pkg/controller/machinedeployment/controller.go index 5c623860e..615043f00 100644 --- a/pkg/controller/machinedeployment/controller.go +++ b/pkg/controller/machinedeployment/controller.go @@ -160,8 +160,8 @@ func (r *ReconcileMachineDeployment) reconcile(ctx context.Context, log *zap.Sug } if !contains(d.Finalizers, metav1.FinalizerDeleteDependents) { - d.Finalizers = append(d.ObjectMeta.Finalizers, metav1.FinalizerDeleteDependents) - if err := r.Client.Update(ctx, d); err != nil { + d.Finalizers = append(d.Finalizers, metav1.FinalizerDeleteDependents) + if err := r.Update(ctx, d); err != nil { return reconcile.Result{}, err } @@ -195,7 +195,7 @@ func (r *ReconcileMachineDeployment) getMachineSetsForDeployment(ctx context.Con // List all MachineSets to find those we own but that no longer match our selector. machineSets := &clusterv1alpha1.MachineSetList{} listOptions := &ctrlruntimeclient.ListOptions{Namespace: d.Namespace} - if err := r.Client.List(ctx, machineSets, listOptions); err != nil { + if err := r.List(ctx, machineSets, listOptions); err != nil { return nil, err } @@ -243,7 +243,7 @@ func (r *ReconcileMachineDeployment) getMachineSetsForDeployment(ctx context.Con func (r *ReconcileMachineDeployment) adoptOrphan(ctx context.Context, deployment *clusterv1alpha1.MachineDeployment, machineSet *clusterv1alpha1.MachineSet) error { newRef := *metav1.NewControllerRef(deployment, controllerKind) machineSet.OwnerReferences = append(machineSet.OwnerReferences, newRef) - return r.Client.Update(ctx, machineSet) + return r.Update(ctx, machineSet) } // getMachineDeploymentsForMachineSet returns a list of MachineDeployments that could potentially match a MachineSet. @@ -255,7 +255,7 @@ func (r *ReconcileMachineDeployment) getMachineDeploymentsForMachineSet(ctx cont dList := &clusterv1alpha1.MachineDeploymentList{} listOptions := &ctrlruntimeclient.ListOptions{Namespace: ms.Namespace} - if err := r.Client.List(ctx, dList, listOptions); err != nil { + if err := r.List(ctx, dList, listOptions); err != nil { log.Errorw("Failed to list MachineDeployments", zap.Error(err)) return nil } @@ -286,7 +286,7 @@ func (r *ReconcileMachineDeployment) MachineSetToDeployments() handler.MapFunc { ms := &clusterv1alpha1.MachineSet{} key := ctrlruntimeclient.ObjectKey{Namespace: o.GetNamespace(), Name: o.GetName()} - if err := r.Client.Get(ctx, key, ms); err != nil { + if err := r.Get(ctx, key, ms); err != nil { if !apierrors.IsNotFound(err) { r.log.Errorw("Failed to retrieve MachineSet for possible MachineDeployment adoption", "machineset", key, zap.Error(err)) } @@ -295,7 +295,7 @@ func (r *ReconcileMachineDeployment) MachineSetToDeployments() handler.MapFunc { // Check if the controller reference is already set and // return an empty result when one is found. - for _, ref := range ms.ObjectMeta.OwnerReferences { + for _, ref := range ms.OwnerReferences { if ref.Controller != nil && *ref.Controller { return result } diff --git a/pkg/controller/machinedeployment/sync.go b/pkg/controller/machinedeployment/sync.go index 8dbf04765..73fe95359 100644 --- a/pkg/controller/machinedeployment/sync.go +++ b/pkg/controller/machinedeployment/sync.go @@ -405,7 +405,7 @@ func (r *ReconcileMachineDeployment) cleanupDeployment(ctx context.Context, log // Avoid deleting machine set with deletion timestamp set aliveFilter := func(ms *clusterv1alpha1.MachineSet) bool { - return ms != nil && ms.ObjectMeta.DeletionTimestamp == nil + return ms != nil && ms.DeletionTimestamp == nil } cleanableMSes := dutil.FilterMachineSets(oldMSs, aliveFilter) diff --git a/pkg/controller/machineset/controller.go b/pkg/controller/machineset/controller.go index cf1a18ca7..b7e20e197 100644 --- a/pkg/controller/machineset/controller.go +++ b/pkg/controller/machineset/controller.go @@ -149,7 +149,7 @@ func (r *ReconcileMachineSet) reconcile(ctx context.Context, log *zap.SugaredLog log.Debug("Reconcile MachineSet") allMachines := &clusterv1alpha1.MachineList{} - if err := r.Client.List(ctx, allMachines, ctrlruntimeclient.InNamespace(machineSet.Namespace)); err != nil { + if err := r.List(ctx, allMachines, ctrlruntimeclient.InNamespace(machineSet.Namespace)); err != nil { return reconcile.Result{}, errors.Wrap(err, "failed to list machines") } @@ -166,9 +166,9 @@ func (r *ReconcileMachineSet) reconcile(ctx context.Context, log *zap.SugaredLog // Add foregroundDeletion finalizer if !contains(machineSet.Finalizers, metav1.FinalizerDeleteDependents) { - machineSet.Finalizers = append(machineSet.ObjectMeta.Finalizers, metav1.FinalizerDeleteDependents) + machineSet.Finalizers = append(machineSet.Finalizers, metav1.FinalizerDeleteDependents) - if err := r.Client.Update(ctx, machineSet); err != nil { + if err := r.Update(ctx, machineSet); err != nil { return reconcile.Result{}, err } @@ -177,7 +177,7 @@ func (r *ReconcileMachineSet) reconcile(ctx context.Context, log *zap.SugaredLog } // Return early if the MachineSet is deleted. - if !machineSet.ObjectMeta.DeletionTimestamp.IsZero() { + if !machineSet.DeletionTimestamp.IsZero() { return reconcile.Result{}, nil } @@ -259,7 +259,7 @@ func (r *ReconcileMachineSet) syncReplicas(ctx context.Context, log *zap.Sugared replicasLog.Infow("Creating new machine", "index", i+1) machine := r.createMachine(ms) - if err := r.Client.Create(ctx, machine); err != nil { + if err := r.Create(ctx, machine); err != nil { log.Errorw("Failed to create Machine", "machine", ctrlruntimeclient.ObjectKeyFromObject(machine), zap.Error(err)) errstrings = append(errstrings, err.Error()) continue @@ -291,7 +291,7 @@ func (r *ReconcileMachineSet) syncReplicas(ctx context.Context, log *zap.Sugared for _, machine := range machinesToDelete { go func(targetMachine *clusterv1alpha1.Machine) { defer wg.Done() - err := r.Client.Delete(ctx, targetMachine) + err := r.Delete(ctx, targetMachine) if err != nil { log.Errorw("Failed to delete Machine", "machine", ctrlruntimeclient.ObjectKeyFromObject(targetMachine), zap.Error(err)) errCh <- err @@ -327,8 +327,8 @@ func (r *ReconcileMachineSet) createMachine(machineSet *clusterv1alpha1.MachineS ObjectMeta: machineSet.Spec.Template.ObjectMeta, Spec: machineSet.Spec.Template.Spec, } - machine.ObjectMeta.GenerateName = fmt.Sprintf("%s-", machineSet.Name) - machine.ObjectMeta.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(machineSet, controllerKind)} + machine.GenerateName = fmt.Sprintf("%s-", machineSet.Name) + machine.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(machineSet, controllerKind)} machine.Namespace = machineSet.Namespace return machine } @@ -341,7 +341,7 @@ func shouldExcludeMachine(machineLog *zap.SugaredLogger, machineSet *clusterv1al return true } - if machine.ObjectMeta.DeletionTimestamp != nil { + if machine.DeletionTimestamp != nil { return true } @@ -356,7 +356,7 @@ func shouldExcludeMachine(machineLog *zap.SugaredLogger, machineSet *clusterv1al func (r *ReconcileMachineSet) adoptOrphan(ctx context.Context, machineSet *clusterv1alpha1.MachineSet, machine *clusterv1alpha1.Machine) error { newRef := *metav1.NewControllerRef(machineSet, controllerKind) machine.OwnerReferences = append(machine.OwnerReferences, newRef) - return r.Client.Update(ctx, machine) + return r.Update(ctx, machine) } func (r *ReconcileMachineSet) waitForMachineCreation(ctx context.Context, log *zap.SugaredLogger, machineList []*clusterv1alpha1.Machine) error { @@ -364,7 +364,7 @@ func (r *ReconcileMachineSet) waitForMachineCreation(ctx context.Context, log *z pollErr := wait.PollUntilContextTimeout(ctx, stateConfirmationInterval, stateConfirmationTimeout, false, func(ctx context.Context) (bool, error) { key := ctrlruntimeclient.ObjectKey{Namespace: machine.Namespace, Name: machine.Name} - if err := r.Client.Get(ctx, key, &clusterv1alpha1.Machine{}); err != nil { + if err := r.Get(ctx, key, &clusterv1alpha1.Machine{}); err != nil { if apierrors.IsNotFound(err) { return false, nil } @@ -389,7 +389,7 @@ func (r *ReconcileMachineSet) waitForMachineDeletion(ctx context.Context, machin m := &clusterv1alpha1.Machine{} key := ctrlruntimeclient.ObjectKey{Namespace: machine.Namespace, Name: machine.Name} - err := r.Client.Get(ctx, key, m) + err := r.Get(ctx, key, m) if apierrors.IsNotFound(err) || !m.DeletionTimestamp.IsZero() { return true, nil } @@ -414,7 +414,7 @@ func (r *ReconcileMachineSet) MachineToMachineSets() handler.MapFunc { key := ctrlruntimeclient.ObjectKey{Namespace: o.GetNamespace(), Name: o.GetName()} machineLog := r.log.With("machine", key) - if err := r.Client.Get(ctx, key, m); err != nil { + if err := r.Get(ctx, key, m); err != nil { if !apierrors.IsNotFound(err) { machineLog.Errorw("Failed to retrieve Machine for possible MachineSet adoption", zap.Error(err)) } @@ -423,7 +423,7 @@ func (r *ReconcileMachineSet) MachineToMachineSets() handler.MapFunc { // Check if the controller reference is already set and // return an empty result when one is found. - for _, ref := range m.ObjectMeta.OwnerReferences { + for _, ref := range m.OwnerReferences { if ref.Controller != nil && *ref.Controller { return result } diff --git a/pkg/controller/machineset/delete_policy.go b/pkg/controller/machineset/delete_policy.go index 73f87f990..77ed4e6d8 100644 --- a/pkg/controller/machineset/delete_policy.go +++ b/pkg/controller/machineset/delete_policy.go @@ -51,16 +51,16 @@ func oldestDeletePriority(machine *clusterv1alpha1.Machine) deletePriority { if machine.DeletionTimestamp != nil && !machine.DeletionTimestamp.IsZero() { return mustDelete } - if machine.ObjectMeta.Annotations != nil && machine.ObjectMeta.Annotations[DeleteNodeAnnotation] != "" { + if machine.Annotations != nil && machine.Annotations[DeleteNodeAnnotation] != "" { return mustDelete } if machine.Status.ErrorReason != nil || machine.Status.ErrorMessage != nil { return mustDelete } - if machine.ObjectMeta.CreationTimestamp.Time.IsZero() { + if machine.CreationTimestamp.Time.IsZero() { return mustNotDelete } - d := metav1.Now().Sub(machine.ObjectMeta.CreationTimestamp.Time) + d := metav1.Now().Sub(machine.CreationTimestamp.Time) if d.Seconds() < 0 { return mustNotDelete } @@ -71,7 +71,7 @@ func newestDeletePriority(machine *clusterv1alpha1.Machine) deletePriority { if machine.DeletionTimestamp != nil && !machine.DeletionTimestamp.IsZero() { return mustDelete } - if machine.ObjectMeta.Annotations != nil && machine.ObjectMeta.Annotations[DeleteNodeAnnotation] != "" { + if machine.Annotations != nil && machine.Annotations[DeleteNodeAnnotation] != "" { return mustDelete } if machine.Status.ErrorReason != nil || machine.Status.ErrorMessage != nil { @@ -84,7 +84,7 @@ func randomDeletePolicy(machine *clusterv1alpha1.Machine) deletePriority { if machine.DeletionTimestamp != nil && !machine.DeletionTimestamp.IsZero() { return mustDelete } - if machine.ObjectMeta.Annotations != nil && machine.ObjectMeta.Annotations[DeleteNodeAnnotation] != "" { + if machine.Annotations != nil && machine.Annotations[DeleteNodeAnnotation] != "" { return betterDelete } if machine.Status.ErrorReason != nil || machine.Status.ErrorMessage != nil { diff --git a/pkg/controller/machineset/machine.go b/pkg/controller/machineset/machine.go index 501b83652..0215a273f 100644 --- a/pkg/controller/machineset/machine.go +++ b/pkg/controller/machineset/machine.go @@ -39,7 +39,7 @@ func (c *ReconcileMachineSet) getMachineSetsForMachine(ctx context.Context, mach Namespace: m.Namespace, } - err := c.Client.List(ctx, msList, listOptions) + err := c.List(ctx, msList, listOptions) if err != nil { machineLog.Errorw("Failed to list MachineSets", zap.Error(err)) return nil diff --git a/pkg/controller/machineset/status.go b/pkg/controller/machineset/status.go index d4162e817..981cf5875 100644 --- a/pkg/controller/machineset/status.go +++ b/pkg/controller/machineset/status.go @@ -138,7 +138,7 @@ func (c *ReconcileMachineSet) getMachineNode(ctx context.Context, machine *clust } node := &corev1.Node{} - err := c.Client.Get(ctx, ctrlruntimeclient.ObjectKey{Name: nodeRef.Name}, node) + err := c.Get(ctx, ctrlruntimeclient.ObjectKey{Name: nodeRef.Name}, node) return node, err } diff --git a/pkg/controller/nodecsrapprover/controller.go b/pkg/controller/nodecsrapprover/controller.go index f8713bed9..0ec752af5 100644 --- a/pkg/controller/nodecsrapprover/controller.go +++ b/pkg/controller/nodecsrapprover/controller.go @@ -254,7 +254,7 @@ func (r *reconciler) validateX509CSR(csr *certificatesv1.CertificateSigningReque func (r *reconciler) getMachineForNode(ctx context.Context, nodeName string) (clusterv1alpha1.Machine, bool, error) { // List all Machines in all namespaces. machines := &clusterv1alpha1.MachineList{} - if err := r.Client.List(ctx, machines); err != nil { + if err := r.List(ctx, machines); err != nil { return clusterv1alpha1.Machine{}, false, fmt.Errorf("failed to list all machine objects: %w", err) } diff --git a/pkg/migrations/migrations.go b/pkg/migrations/migrations.go index 85065d570..de790f7d9 100644 --- a/pkg/migrations/migrations.go +++ b/pkg/migrations/migrations.go @@ -351,7 +351,7 @@ func ensureClusterV1Alpha1NodeOwnership(ctx context.Context, machineLog *zap.Sug machineLog.Info("No node for machines found") continue } - return fmt.Errorf("Failed to get node %s for machine %s: %w", + return fmt.Errorf("failed to get node %s for machine %s: %w", machine.Spec.Name, machine.Name, err) } diff --git a/pkg/node/eviction/eviction.go b/pkg/node/eviction/eviction.go index f41c27007..2e513e49c 100644 --- a/pkg/node/eviction/eviction.go +++ b/pkg/node/eviction/eviction.go @@ -108,7 +108,7 @@ func (ne *NodeEviction) getFilteredPods(ctx context.Context) ([]corev1.Pod, erro if controllerRef := metav1.GetControllerOf(&candidatePod); controllerRef != nil && controllerRef.Kind == "DaemonSet" { continue } - if _, found := candidatePod.ObjectMeta.Annotations[corev1.MirrorPodAnnotationKey]; found { + if _, found := candidatePod.Annotations[corev1.MirrorPodAnnotationKey]; found { continue } filteredPods = append(filteredPods, candidatePod) diff --git a/sdk/.golangci.yml b/sdk/.golangci.yml index 10a3f40e2..c678a6c2d 100644 --- a/sdk/.golangci.yml +++ b/sdk/.golangci.yml @@ -18,22 +18,36 @@ # root's .golangci.yml and once with the SDK's config file. # +version: "2" run: - timeout: 10m modules-download-mode: readonly - linters: + default: none enable: - depguard - disable-all: true - -linters-settings: - depguard: - rules: - noreverse: - deny: - - { pkg: k8c.io/machine-controller/pkg, desc: SDK must not depend on the main module } - -issues: - exclude-files: - - zz_generated.*.go + settings: + depguard: + rules: + noreverse: + deny: + - pkg: k8c.io/machine-controller/pkg + desc: SDK must not depend on the main module + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + paths: + - zz_generated.*.go + - third_party$ + - builtin$ + - examples$ +formatters: + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/sdk/apis/cluster/v1alpha1/conversions/conversions.go b/sdk/apis/cluster/v1alpha1/conversions/conversions.go index 88a649681..7a0f7a83b 100644 --- a/sdk/apis/cluster/v1alpha1/conversions/conversions.go +++ b/sdk/apis/cluster/v1alpha1/conversions/conversions.go @@ -34,7 +34,7 @@ func Convert_MachinesV1alpha1Machine_To_ClusterV1alpha1Machine(in *machinesv1alp out.ResourceVersion = "" out.Generation = 0 out.CreationTimestamp = metav1.Time{} - out.ObjectMeta.Namespace = metav1.NamespaceSystem + out.Namespace = metav1.NamespaceSystem // k8c.io/machine-controller/sdk/apis/cluster/v1alpha1.MachineStatus and // sdk/apis/machines/v1alpha1.MachineStatus are semantically identical, the former diff --git a/sdk/net/net.go b/sdk/net/net.go index fc39ab1e1..de70a41c5 100644 --- a/sdk/net/net.go +++ b/sdk/net/net.go @@ -21,8 +21,8 @@ import ( ) const ( - ErrIPv6OnlyUnsupported = "IPv6 only network family not supported yet" - ErrUnknownNetworkFamily = "Unknown IP family %q only IPv4,IPv6,IPv4+IPv6 are valid values" + ErrIPv6OnlyUnsupported = "IPv6-only network family not supported yet" + ErrUnknownNetworkFamily = "unknown IP family %q, only IPv4,IPv6,IPv4+IPv6 are valid values" ) // IPFamily IPv4 | IPv6 | IPv4+IPv6. diff --git a/sdk/providerconfig/types.go b/sdk/providerconfig/types.go index 888b4744a..b986a8a8b 100644 --- a/sdk/providerconfig/types.go +++ b/sdk/providerconfig/types.go @@ -197,14 +197,14 @@ type configVarStringWithoutUnmarshaller ConfigVarString // https://github.com/golang/go/issues/11939. func (configVarString ConfigVarString) MarshalJSON() ([]byte, error) { var secretKeyRefEmpty, configMapKeyRefEmpty bool - if configVarString.SecretKeyRef.ObjectReference.Namespace == "" && - configVarString.SecretKeyRef.ObjectReference.Name == "" && + if configVarString.SecretKeyRef.Namespace == "" && + configVarString.SecretKeyRef.Name == "" && configVarString.SecretKeyRef.Key == "" { secretKeyRefEmpty = true } - if configVarString.ConfigMapKeyRef.ObjectReference.Namespace == "" && - configVarString.ConfigMapKeyRef.ObjectReference.Name == "" && + if configVarString.ConfigMapKeyRef.Namespace == "" && + configVarString.ConfigMapKeyRef.Name == "" && configVarString.ConfigMapKeyRef.Key == "" { configMapKeyRefEmpty = true } @@ -219,7 +219,7 @@ func (configVarString ConfigVarString) MarshalJSON() ([]byte, error) { if err != nil { return nil, err } - buffer.WriteString(fmt.Sprintf(`"secretKeyRef":%s`, string(jsonVal))) + fmt.Fprintf(buffer, `"secretKeyRef":%s`, string(jsonVal)) } if !configMapKeyRefEmpty { @@ -231,11 +231,11 @@ func (configVarString ConfigVarString) MarshalJSON() ([]byte, error) { if err != nil { return nil, err } - buffer.WriteString(fmt.Sprintf(`%s"configMapKeyRef":%s`, leadingComma, jsonVal)) + fmt.Fprintf(buffer, `%s"configMapKeyRef":%s`, leadingComma, jsonVal) } if configVarString.Value != "" { - buffer.WriteString(fmt.Sprintf(`,"value":"%s"`, configVarString.Value)) + fmt.Fprintf(buffer, `,"value":"%s"`, configVarString.Value) } buffer.WriteString("}") @@ -285,14 +285,14 @@ type configVarBoolWithoutUnmarshaller ConfigVarBool // https://github.com/golang/go/issues/11939 func (configVarBool ConfigVarBool) MarshalJSON() ([]byte, error) { var secretKeyRefEmpty, configMapKeyRefEmpty bool - if configVarBool.SecretKeyRef.ObjectReference.Namespace == "" && - configVarBool.SecretKeyRef.ObjectReference.Name == "" && + if configVarBool.SecretKeyRef.Namespace == "" && + configVarBool.SecretKeyRef.Name == "" && configVarBool.SecretKeyRef.Key == "" { secretKeyRefEmpty = true } - if configVarBool.ConfigMapKeyRef.ObjectReference.Namespace == "" && - configVarBool.ConfigMapKeyRef.ObjectReference.Name == "" && + if configVarBool.ConfigMapKeyRef.Namespace == "" && + configVarBool.ConfigMapKeyRef.Name == "" && configVarBool.ConfigMapKeyRef.Key == "" { configMapKeyRefEmpty = true } @@ -311,7 +311,7 @@ func (configVarBool ConfigVarBool) MarshalJSON() ([]byte, error) { if err != nil { return nil, err } - buffer.WriteString(fmt.Sprintf(`"secretKeyRef":%s`, string(jsonVal))) + fmt.Fprintf(buffer, `"secretKeyRef":%s`, string(jsonVal)) } if !configMapKeyRefEmpty { @@ -323,7 +323,7 @@ func (configVarBool ConfigVarBool) MarshalJSON() ([]byte, error) { if err != nil { return nil, err } - buffer.WriteString(fmt.Sprintf(`%s"configMapKeyRef":%s`, leadingComma, jsonVal)) + fmt.Fprintf(buffer, `%s"configMapKeyRef":%s`, leadingComma, jsonVal) } if configVarBool.Value != nil { @@ -332,7 +332,7 @@ func (configVarBool ConfigVarBool) MarshalJSON() ([]byte, error) { return []byte{}, err } - buffer.WriteString(fmt.Sprintf(`,"value":%v`, string(jsonVal))) + fmt.Fprintf(buffer, `,"value":%v`, string(jsonVal)) } buffer.WriteString("}") @@ -344,7 +344,7 @@ func (configVarBool *ConfigVarBool) UnmarshalJSON(b []byte) error { if !bytes.HasPrefix(b, []byte("{")) { var val *bool if err := json.Unmarshal(b, &val); err != nil { - return fmt.Errorf("Error parsing value: '%w'", err) + return fmt.Errorf("error parsing value: '%w'", err) } configVarBool.Value = val diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 973fb6754..2ccc9b439 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -201,8 +201,8 @@ func testScenario(ctx context.Context, t *testing.T, testCase scenario, cloudPro } kubernetesCompliantName := fmt.Sprintf("%s-%s", testCase.name, cloudProvider) - kubernetesCompliantName = strings.Replace(kubernetesCompliantName, " ", "-", -1) - kubernetesCompliantName = strings.Replace(kubernetesCompliantName, ".", "-", -1) + kubernetesCompliantName = strings.ReplaceAll(kubernetesCompliantName, " ", "-") + kubernetesCompliantName = strings.ReplaceAll(kubernetesCompliantName, ".", "-") kubernetesCompliantName = strings.ToLower(kubernetesCompliantName) scenarioParams := append([]string(nil), testParams...) diff --git a/test/e2e/provisioning/verify.go b/test/e2e/provisioning/verify.go index 89abf61b8..92e8ceed9 100644 --- a/test/e2e/provisioning/verify.go +++ b/test/e2e/provisioning/verify.go @@ -67,7 +67,7 @@ func verifyCreateAndDelete(ctx context.Context, kubeConfig, manifestPath string, } if err := deleteAndAssure(ctx, machineDeployment, client, timeout); err != nil { - return fmt.Errorf("Failed to verify if a machine/node has been created/deleted, due to: \n%w", err) + return fmt.Errorf("failed to verify if a machine/node has been created/deleted, due to: \n%w", err) } klog.Infof("Successfully finished test for MachineDeployment %s", machineDeployment.Name) @@ -122,7 +122,7 @@ func prepare(kubeConfig, manifestPath string, parameters []string) (ctrlruntimec // init kube related stuff cfg, err := clientcmd.BuildConfigFromFlags("", kubeConfig) if err != nil { - return nil, "", fmt.Errorf("Error building kubeconfig: %w", err) + return nil, "", fmt.Errorf("error building kubeconfig: %w", err) } client, err := ctrlruntimeclient.New(cfg, ctrlruntimeclient.Options{}) if err != nil { @@ -339,7 +339,7 @@ func readAndModifyManifest(pathToManifest string, keyValuePairs []string) (strin if len(kv) != 2 { return "", fmt.Errorf("the given key value pair = %v is incorrect, the correct form is key=value", keyValuePair) } - content = strings.Replace(content, kv[0], kv[1], -1) + content = strings.ReplaceAll(content, kv[0], kv[1]) } return content, nil From ff89f649cc98642311f1cae96e183e2e543dd8b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Apr 2025 16:26:07 +0200 Subject: [PATCH 467/489] Bump github.com/golang-jwt/jwt/v4 from 4.5.1 to 4.5.2 (#1910) Bumps [github.com/golang-jwt/jwt/v4](https://github.com/golang-jwt/jwt) from 4.5.1 to 4.5.2. - [Release notes](https://github.com/golang-jwt/jwt/releases) - [Changelog](https://github.com/golang-jwt/jwt/blob/main/VERSION_HISTORY.md) - [Commits](https://github.com/golang-jwt/jwt/compare/v4.5.1...v4.5.2) --- updated-dependencies: - dependency-name: github.com/golang-jwt/jwt/v4 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8ba164f6a..236e6c1b6 100644 --- a/go.mod +++ b/go.mod @@ -113,7 +113,7 @@ require ( github.com/go-resty/resty/v2 v2.14.0 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.1 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect diff --git a/go.sum b/go.sum index eb3142053..930efcbaa 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptG github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= -github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= From a9dc4e656d96ab9a29f4c7e34fbc7ad3c79237d2 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Tue, 8 Apr 2025 10:43:12 +0200 Subject: [PATCH 468/489] Update to controller-runtime 0.20, bump golang.org/x libraries (#1917) * bump to controller-runtime 0.20, kube 1.32 * bump golang.org/x to fix various CVEs --- go.mod | 58 +++++++++++++++---------------- go.sum | 107 +++++++++++++++++++++++++++++---------------------------- 2 files changed, 83 insertions(+), 82 deletions(-) diff --git a/go.mod b/go.mod index 236e6c1b6..0ae8126e9 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module k8c.io/machine-controller -go 1.22.3 +go 1.23.0 -toolchain go1.23.1 +toolchain go1.24.0 replace k8c.io/machine-controller/sdk => ./sdk @@ -46,23 +46,23 @@ require ( github.com/vultr/govultr/v3 v3.9.1 go.anx.io/go-anxcloud v0.7.3 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.31.0 - golang.org/x/oauth2 v0.23.0 + golang.org/x/crypto v0.36.0 + golang.org/x/oauth2 v0.28.0 gomodules.xyz/jsonpatch/v2 v2.4.0 google.golang.org/api v0.197.0 gopkg.in/yaml.v3 v3.0.1 k8c.io/machine-controller/sdk v0.0.0-00010101000000-000000000000 - k8s.io/api v0.31.1 - k8s.io/apiextensions-apiserver v0.31.1 - k8s.io/apimachinery v0.31.1 - k8s.io/client-go v0.31.1 - k8s.io/cloud-provider v0.31.1 + k8s.io/api v0.32.1 + k8s.io/apiextensions-apiserver v0.32.1 + k8s.io/apimachinery v0.32.1 + k8s.io/client-go v0.32.1 + k8s.io/cloud-provider v0.32.1 k8s.io/klog v1.0.0 - k8s.io/kubectl v0.31.1 - k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 + k8s.io/kubectl v0.32.1 + k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 kubevirt.io/api v1.3.1 kubevirt.io/containerized-data-importer-api v1.60.3 - sigs.k8s.io/controller-runtime v0.19.0 + sigs.k8s.io/controller-runtime v0.20.4 ) require ( @@ -96,7 +96,7 @@ require ( github.com/dimchansky/utfbom v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/evanphx/json-patch v5.9.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.9.0 // indirect + github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect @@ -116,18 +116,18 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect + github.com/google/btree v1.1.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect + github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect github.com/google/s2a-go v0.1.8 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect github.com/googleapis/gax-go/v2 v2.13.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-version v1.7.0 // indirect - github.com/imdario/mergo v0.3.16 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -144,8 +144,8 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oklog/ulid v1.3.1 // indirect - github.com/onsi/ginkgo/v2 v2.20.1 // indirect - github.com/onsi/gomega v1.34.1 // indirect + github.com/onsi/ginkgo/v2 v2.22.0 // indirect + github.com/onsi/gomega v1.36.1 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect github.com/peterhellberg/link v1.2.0 // indirect @@ -164,28 +164,28 @@ require ( go.opentelemetry.io/otel/trace v1.30.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/term v0.27.0 // indirect - golang.org/x/text v0.21.0 // indirect - golang.org/x/time v0.6.0 // indirect - golang.org/x/tools v0.25.0 // indirect + golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 // indirect + golang.org/x/time v0.11.0 // indirect + golang.org/x/tools v0.31.0 // indirect google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/grpc v1.66.2 // indirect - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.35.1 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect + k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect - sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect + sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 930efcbaa..91a69b99f 100644 --- a/go.sum +++ b/go.sum @@ -135,8 +135,8 @@ github.com/equinix/equinix-sdk-go v0.46.0/go.mod h1:hEb3XLaedz7xhl/dpPIS6eOIiXNP github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= -github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= +github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -230,6 +230,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -251,8 +253,8 @@ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k= -github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= +github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -284,8 +286,6 @@ github.com/hetznercloud/hcloud-go/v2 v2.13.1 h1:jq0GP4QaYE5d8xR/Zw17s9qoaESRJMXf github.com/hetznercloud/hcloud-go/v2 v2.13.1/go.mod h1:dhix40Br3fDiBhwaSG/zgaYOFFddpfBm/6R1Zz0IiF0= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= -github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= github.com/jhump/protoreflect v1.14.0 h1:MBbQK392K3u8NTLbKOCIi3XdI+y+c6yt5oMq0X3xviw= @@ -368,15 +368,15 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.20.1 h1:YlVIbqct+ZmnEph770q9Q7NVAz4wwIiVNahee6JyUzo= -github.com/onsi/ginkgo/v2 v2.20.1/go.mod h1:lG9ey2Z29hR41WMVthyJBGUBcBhGOtoPF2VFMvBXFCI= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= -github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= +github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= @@ -493,16 +493,16 @@ golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= +golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= +golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -548,11 +548,11 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= +golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -564,8 +564,8 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -599,8 +599,8 @@ golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -611,8 +611,8 @@ golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -625,10 +625,11 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -649,8 +650,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= -golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= +golang.org/x/tools v0.31.0 h1:0EedkvKDbh+qistFTd0Bcwe/YLh4vHwWEkiI0toFIBU= +golang.org/x/tools v0.31.0/go.mod h1:naFTU+Cev749tSJRXJlna0T3WxKvb1kWEx15xA4SdmQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -695,8 +696,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= +google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -729,17 +730,17 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= -k8s.io/api v0.31.1 h1:Xe1hX/fPW3PXYYv8BlozYqw63ytA92snr96zMW9gWTU= -k8s.io/api v0.31.1/go.mod h1:sbN1g6eY6XVLeqNsZGLnI5FwVseTrZX7Fv3O26rhAaI= -k8s.io/apiextensions-apiserver v0.31.1 h1:L+hwULvXx+nvTYX/MKM3kKMZyei+UiSXQWciX/N6E40= -k8s.io/apiextensions-apiserver v0.31.1/go.mod h1:tWMPR3sgW+jsl2xm9v7lAyRF1rYEK71i9G5dRtkknoQ= +k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc= +k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k= +k8s.io/apiextensions-apiserver v0.32.1 h1:hjkALhRUeCariC8DiVmb5jj0VjIc1N0DREP32+6UXZw= +k8s.io/apiextensions-apiserver v0.32.1/go.mod h1:sxWIGuGiYov7Io1fAS2X06NjMIk5CbRHc2StSmbaQto= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= -k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= -k8s.io/client-go v0.31.1 h1:f0ugtWSbWpxHR7sjVpQwuvw9a3ZKLXX0u0itkFXufb0= -k8s.io/client-go v0.31.1/go.mod h1:sKI8871MJN2OyeqRlmA4W4KM9KBdBUpDLu/43eGemCg= -k8s.io/cloud-provider v0.31.1 h1:40b6AgDizwm5eWratZbqubTHMob25VWr6NX2Ei5TwZA= -k8s.io/cloud-provider v0.31.1/go.mod h1:xAdkE7fdZdu9rKLuOZUMBfagu7bM+bas3iPux/2nLGg= +k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs= +k8s.io/apimachinery v0.32.1/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/client-go v0.32.1 h1:otM0AxdhdBIaQh7l1Q0jQpmo7WOFIk5FFa4bg6YMdUU= +k8s.io/client-go v0.32.1/go.mod h1:aTTKZY7MdxUaJ/KiUs8D+GssR9zJZi77ZqtzcGXIiDg= +k8s.io/cloud-provider v0.32.1 h1:74rRhnfca3o4CsjjnIp/C3ARVuSmyNsxgWPtH0yc9Z0= +k8s.io/cloud-provider v0.32.1/go.mod h1:GECSanFT+EeZ/ToX3xlasjETzMUI+VFu92zHUDUsGHw= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= @@ -753,14 +754,14 @@ k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 h1:1dWzkmJrrprYvjGwh9kEUxmcUV/CtNU8QM7h1FLWQOo= -k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA= -k8s.io/kubectl v0.31.1 h1:ih4JQJHxsEggFqDJEHSOdJ69ZxZftgeZvYo7M/cpp24= -k8s.io/kubectl v0.31.1/go.mod h1:aNuQoR43W6MLAtXQ/Bu4GDmoHlbhHKuyD49lmTC8eJM= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= +k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= +k8s.io/kubectl v0.32.1 h1:/btLtXLQUU1rWx8AEvX9jrb9LaI6yeezt3sFALhB8M8= +k8s.io/kubectl v0.32.1/go.mod h1:sezNuyWi1STk4ZNPVRIFfgjqMI6XMf+oCVLjZen/pFQ= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3 h1:b2FmK8YH+QEwq/Sy2uAEhmqL5nPfGYbJOcaqjeYYZoA= -k8s.io/utils v0.0.0-20240902221715-702e33fdd3c3/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= +k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v1.3.1 h1:MoTNo/zvDlZ44c2ocXLPln8XTaQOeUodiYbEKrTCqv4= kubevirt.io/api v1.3.1/go.mod h1:tCn7VAZktEvymk490iPSMPCmKM9UjbbfH2OsFR/IOLU= kubevirt.io/containerized-data-importer-api v1.60.3 h1:kQEXi7scpzUa0RPf3/3MKk1Kmem0ZlqqiuK3kDF5L2I= @@ -768,15 +769,15 @@ kubevirt.io/containerized-data-importer-api v1.60.3/go.mod h1:8mwrkZIdy8j/LmCyKt kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 h1:fZYvD3/Vnitfkx6IJxjLAk8ugnZQ7CXVYcRfkSKmuZY= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q= -sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/controller-runtime v0.20.4 h1:X3c+Odnxz+iPTRobG4tp092+CvBU9UK0t/bRf+n0DGU= +sigs.k8s.io/controller-runtime v0.20.4/go.mod h1:xg2XB0K5ShQzAgsoujxuKN4LNXR2LfwwHsPj7Iaw+XY= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= -sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= +sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= +sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= From 2530a75d41fc8ccbbf703562f6c986ed8f0d2033 Mon Sep 17 00:00:00 2001 From: Archana Sawant Date: Tue, 13 May 2025 11:38:09 +0530 Subject: [PATCH 469/489] Update the new image tag for KKP owned images (#1922) Signed-off-by: archups --- examples/kubevirt-local-mounter.yaml | 2 +- hack/lib.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/kubevirt-local-mounter.yaml b/examples/kubevirt-local-mounter.yaml index 302d99e81..8c2236e02 100644 --- a/examples/kubevirt-local-mounter.yaml +++ b/examples/kubevirt-local-mounter.yaml @@ -15,7 +15,7 @@ spec: hostPID: true containers: - name: startup-script - image: quay.io/kubermatic/startup-script:v0.1.0 + image: quay.io/kubermatic/startup-script:v0.2.1 securityContext: privileged: true env: diff --git a/hack/lib.sh b/hack/lib.sh index 1d9930c18..aa98ead68 100644 --- a/hack/lib.sh +++ b/hack/lib.sh @@ -52,7 +52,7 @@ is_containerized() { containerize() { local cmd="$1" - local image="${CONTAINERIZE_IMAGE:-quay.io/kubermatic/util:2.2.0}" + local image="${CONTAINERIZE_IMAGE:-quay.io/kubermatic/util:2.6.0}" local gocache="${CONTAINERIZE_GOCACHE:-/tmp/.gocache}" local gomodcache="${CONTAINERIZE_GOMODCACHE:-/tmp/.gomodcache}" local skip="${NO_CONTAINERIZE:-}" From 7a93ac526de3b0e6a3cf2a31e46b9dc243a2cd7d Mon Sep 17 00:00:00 2001 From: Moath Qasim Date: Tue, 20 May 2025 23:28:57 +0200 Subject: [PATCH 470/489] Add ClusterNamespace Annotations to Worker Nodes (#1924) * add clusterNamespace annotations Signed-off-by: moadqassem Signed-off-by: moadqassem * initialize annotation Signed-off-by: moadqassem --------- Signed-off-by: moadqassem --- pkg/admission/machines.go | 7 +++++++ pkg/cloudprovider/provider/kubevirt/provider.go | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index b371529d2..50f81dd4c 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -131,6 +131,13 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec } } + // For KubeVirt we need to initialize the annotations for MachineDeployment, to enable setting of the needed annotations. + if providerConfig.CloudProvider == providerconfig.CloudProviderKubeVirt { + if spec.Annotations == nil { + spec.Annotations = make(map[string]string) + } + } + configResolver := configvar.NewResolver(ctx, ad.workerClient) prov, err := cloudprovider.ForProvider(providerConfig.CloudProvider, configResolver) if err != nil { diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index f59e369e9..988f7c23b 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -80,6 +80,8 @@ const ( // node affinity constraints on a PersistentVolume. topologyRegionKey = "topology.kubernetes.io/region" topologyZoneKey = "topology.kubernetes.io/zone" + // clusterNamespace represents the infra cluster namespace, where KubeVirt resources are created. + clusterNamespace = "cluster.x-k8s.io/cluster-namespace" ) type provider struct { @@ -674,6 +676,13 @@ func (p *provider) AddDefaults(_ *zap.SugaredLogger, spec clusterv1alpha1.Machin return spec, err } + annotations := spec.Annotations + if annotations == nil { + annotations = make(map[string]string) + } + + annotations[clusterNamespace] = c.Namespace + spec.Annotations = annotations if err := appendTopologiesLabels(context.TODO(), c, spec.Labels); err != nil { return spec, err } From d8c397d9081cbdd602e9b4bfaed2fc545be55939 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Fri, 30 May 2025 22:36:11 +0200 Subject: [PATCH 471/489] Handle SDK version tags (#1925) * only consider main version tags during build * fix log line --- Makefile | 2 +- hack/ci/setup-machine-controller-in-kind.sh | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f27111938..a5da22431 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ GOOS ?= $(shell go env GOOS) export CGO_ENABLED := 0 -export GIT_TAG ?= $(shell git tag --points-at HEAD) +export GIT_TAG ?= $(shell git tag --points-at HEAD 'v*') export GOFLAGS?=-mod=readonly -trimpath diff --git a/hack/ci/setup-machine-controller-in-kind.sh b/hack/ci/setup-machine-controller-in-kind.sh index 6aa09c2bf..32aa4b6d6 100755 --- a/hack/ci/setup-machine-controller-in-kind.sh +++ b/hack/ci/setup-machine-controller-in-kind.sh @@ -68,11 +68,10 @@ if [ ! -f machine-controller-deployed ]; then fi OSM_TMP_DIR=/tmp/osm -echodate "Clone OSM respository" ( # Clone OSM repo mkdir -p $OSM_TMP_DIR - echodate "Cloning cluster exposer" + echodate "Cloning OSM respository" git clone --depth 1 --branch "${OSM_REPO_TAG}" "${OSM_REPO_URL}" $OSM_TMP_DIR ) From bab47c4563be85f8fa4a369a0337141065442ff3 Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Mon, 9 Jun 2025 10:47:15 +0200 Subject: [PATCH 472/489] fix AMI query for Ubuntu images (#1927) --- pkg/cloudprovider/provider/aws/provider.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index cee0d668c..e8e813ab2 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -130,7 +130,7 @@ var ( providerconfig.OperatingSystemUbuntu: { awstypes.CPUArchitectureX86_64: { // Be as precise as possible - otherwise we might get a nightly dev build - description: "Canonical, Ubuntu, 24.04 LTS, amd64 noble image build on ????-??-??", + description: "Canonical, Ubuntu, 24.04, amd64 noble image", // The AWS marketplace ID from Canonical owner: "099720109477", }, From 94d2587a89e83990bf4409ff24911337fc5b3242 Mon Sep 17 00:00:00 2001 From: Kubermatic Bot <41968677+kubermatic-bot@users.noreply.github.com> Date: Fri, 13 Jun 2025 10:16:19 +0200 Subject: [PATCH 473/489] Synchronize OWNERS_ALIASES file with Github teams (#1930) --- OWNERS_ALIASES | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 35764dfbc..dccdae651 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -11,6 +11,7 @@ aliases: - moadqassem - moelsayed - mohamed-rafraf + - rajasahil - soer3n - xmudrii - xrstf From b0fd2c8fa84d7edb72485b1b349bca87a5c035b1 Mon Sep 17 00:00:00 2001 From: Archana Sawant Date: Wed, 2 Jul 2025 12:58:06 +0530 Subject: [PATCH 474/489] Bump Go version to 1.24.4 (#1934) Signed-off-by: archups --- .prow/e2e-features.yaml | 8 ++++---- .prow/postsubmits.yaml | 4 ++-- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 12 ++++++------ .prow/provider-azure.yaml | 6 +++--- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 ++-- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 10 +++++----- .prow/verify.yaml | 16 ++++++++-------- Dockerfile | 2 +- Makefile | 2 +- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- 22 files changed, 45 insertions(+), 45 deletions(-) diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index 0dcaf5103..beda7a53d 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -35,7 +35,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -65,7 +65,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -95,7 +95,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -123,7 +123,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index 8ff4a990b..e4000d373 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -27,7 +27,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - /bin/bash - -c @@ -56,7 +56,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index a1461b83c..d539d3d11 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index aff05124a..bd6b6e4c8 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -32,7 +32,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 68eaa6b41..dc0437276 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -96,7 +96,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -130,7 +130,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -162,7 +162,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -194,7 +194,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 551e1ba2f..112724446 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -96,7 +96,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 6da6e33b8..3131f3a7c 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index fff89c2e1..b70cdef95 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index 322d38ad4..f20b57669 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 62501fc53..a4b2a6d76 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index bec18fafc..6eb67c22e 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 2e615e29f..5198a8c6b 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 8e90f0a72..68b435c0e 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index e02483e16..562ca575e 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -64,7 +64,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index ec467e703..63f67bb81 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index ffaaed035..39134beb3 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -32,7 +32,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 096d986b1..2dd7a9453 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -95,7 +95,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -128,7 +128,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -161,7 +161,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 20ada4993..3f212142f 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -22,7 +22,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-6 command: - make args: @@ -44,7 +44,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-6 command: - make args: @@ -66,7 +66,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-6 command: - make args: @@ -87,7 +87,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-6 command: - make args: @@ -107,7 +107,7 @@ presubmits: path_alias: k8c.io/machine-controller spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-6 command: - "/usr/local/bin/shfmt" args: @@ -136,7 +136,7 @@ presubmits: path_alias: k8c.io/machine-controller spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-6 command: - "./hack/verify-boilerplate.sh" resources: @@ -156,7 +156,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-6 command: - ./hack/verify-licenses.sh resources: @@ -173,7 +173,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-3 + - image: quay.io/kubermatic/build:go-1.24-node-20-6 command: - make args: diff --git a/Dockerfile b/Dockerfile index 85ec085a7..633546fa4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.24.2 +ARG GO_VERSION=1.24.4 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/k8c.io/machine-controller COPY . . diff --git a/Makefile b/Makefile index a5da22431..d2558cc1c 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.24.2 +GO_VERSION ?= 1.24.4 GOOS ?= $(shell go env GOOS) diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index 667d517ec..f2d148e35 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.24-node-20-3 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.24-node-20-6 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 6ad5f712b..eb372848c 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.24-node-20-3 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.24-node-20-6 containerize ./hack/verify-licenses.sh go mod vendor From aee94dfb0b0e775c6ef79538cdaebd0ef6ac040f Mon Sep 17 00:00:00 2001 From: Kubermatic Bot <41968677+kubermatic-bot@users.noreply.github.com> Date: Tue, 8 Jul 2025 17:50:24 +0200 Subject: [PATCH 475/489] Synchronize OWNERS_ALIASES file with Github teams (#1936) --- OWNERS_ALIASES | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index dccdae651..348515559 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -3,6 +3,7 @@ aliases: sig-cluster-management: + - adoi - ahmedwaleedmalik - cnvergence - embik From aad8f344b21e8b5be77c2cf3b715ad674b33eba7 Mon Sep 17 00:00:00 2001 From: Kubermatic Bot <41968677+kubermatic-bot@users.noreply.github.com> Date: Mon, 28 Jul 2025 09:33:42 +0200 Subject: [PATCH 476/489] Synchronize OWNERS_ALIASES file with Github teams (#1946) --- OWNERS_ALIASES | 1 + 1 file changed, 1 insertion(+) diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 348515559..e952ce181 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -5,6 +5,7 @@ aliases: sig-cluster-management: - adoi - ahmedwaleedmalik + - buraksekili - cnvergence - embik - julioc-p From cd742c3b48ae7881c74ec09054b62ed2a6d2523c Mon Sep 17 00:00:00 2001 From: Christoph Mewes Date: Mon, 28 Jul 2025 11:31:41 +0200 Subject: [PATCH 477/489] fix various typos (#1935) --- docs/cloud-provider.md | 2 +- docs/vsphere.md | 4 ++-- hack/ci/setup-machine-controller-in-kind.sh | 2 +- pkg/cloudprovider/provider/kubevirt/provider.go | 4 ++-- pkg/cloudprovider/provider/openstack/provider_test.go | 2 +- pkg/controller/machine/controller.go | 4 ++-- pkg/controller/machinedeployment/controller.go | 2 +- pkg/controller/machineset/controller.go | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index 18478c133..cda08ba1c 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -54,7 +54,7 @@ accessKeyId: "<< YOUR_ACCESS_KEY_ID >>" secretAccessKey: "<< YOUR_SECRET_ACCESS_KEY_ID >>" # region for the instance region: "eu-central-1" -# avaiability zone for the instance +# availability zone for the instance availabilityZone: "eu-central-1a" # vpc id for the instance vpcId: "vpc-079f7648481a11e77" diff --git a/docs/vsphere.md b/docs/vsphere.md index 0a299cb2a..81cf0d98e 100644 --- a/docs/vsphere.md +++ b/docs/vsphere.md @@ -27,7 +27,7 @@ To see where to locate the OVAs go to the OS specific section. 3. Click through the dialog until "Select storage" 4. Select the same storage you want to use for your machines 5. Select the same network you want to use for your machines -6. Leave everyhting in the "Customize Template" and "Ready to complete" dialog as it is +6. Leave everything in the "Customize Template" and "Ready to complete" dialog as it is 7. Wait until the VM got fully imported and the "Snapshots" => "Create Snapshot" button is not grayed out anymore #### Command-line procedure @@ -168,7 +168,7 @@ Procedure: 3. Upload to vSphere using WebUI or GOVC: Make sure to replace the parameters on the command below with the correct - values specific to yout vSphere environment. + values specific to your vSphere environment. ``` govc import.vmdk -dc=dc-1 -pool=/dc-1/host/cl-1/Resources -ds=ds-1 "./${image_name}.vmdk" diff --git a/hack/ci/setup-machine-controller-in-kind.sh b/hack/ci/setup-machine-controller-in-kind.sh index 32aa4b6d6..535346ff9 100755 --- a/hack/ci/setup-machine-controller-in-kind.sh +++ b/hack/ci/setup-machine-controller-in-kind.sh @@ -71,7 +71,7 @@ OSM_TMP_DIR=/tmp/osm ( # Clone OSM repo mkdir -p $OSM_TMP_DIR - echodate "Cloning OSM respository" + echodate "Cloning OSM repository" git clone --depth 1 --branch "${OSM_REPO_TAG}" "${OSM_REPO_URL}" $OSM_TMP_DIR ) diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 988f7c23b..5fc906bdf 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -1144,14 +1144,14 @@ func appendTopologiesLabels(ctx context.Context, c *Config, labels map[string]st return nil } -func getStorageTopologies(ctx context.Context, storageClasName string, c *Config, labels map[string]string) error { +func getStorageTopologies(ctx context.Context, storageClassName string, c *Config, labels map[string]string) error { kubeClient, err := ctrlruntimeclient.New(c.RestConfig, ctrlruntimeclient.Options{}) if err != nil { return fmt.Errorf("failed to get kubevirt client: %w", err) } sc := &storagev1.StorageClass{} - if err := kubeClient.Get(ctx, types.NamespacedName{Name: storageClasName}, sc); err != nil { + if err := kubeClient.Get(ctx, types.NamespacedName{Name: storageClassName}, sc); err != nil { return err } diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index c85ec0136..37f3d709e 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -374,7 +374,7 @@ func ExpectServerCreated(t *testing.T, expectedServer string) { // expectedServer copied into the response (e.g. name). err := json.Unmarshal([]byte(expectedServer), &res) if err != nil { - t.Fatalf("Error occurred while unmarshaling the expected server manifest.") + t.Fatalf("Error occurred while unmarshalling the expected server manifest.") } res.Server.ID = "1bea47ed-f6a9-463b-b423-14b9cca9ad27" srvRes, err := json.Marshal(res) diff --git a/pkg/controller/machine/controller.go b/pkg/controller/machine/controller.go index c7749980c..fa5f8918f 100644 --- a/pkg/controller/machine/controller.go +++ b/pkg/controller/machine/controller.go @@ -81,7 +81,7 @@ const ( // AnnotationMachineUninitialized indicates that a machine is not yet // ready to be worked on by the machine-controller. The machine-controller - // will ignore all machines that have this anotation with any value + // will ignore all machines that have this annotation with any value // Its value should consist of one or more initializers, separated by a comma. AnnotationMachineUninitialized = "machine-controller.kubermatic.io/initializers" @@ -261,7 +261,7 @@ func enqueueRequestsForNodes(ctx context.Context, log *zap.SugaredLogger, mgr ma ownerUIDString, exists = nodeLabels[NodeOwnerLabelName] } if !exists { - // We get triggered by node{Add,Update}, so enqeue machines if they + // We get triggered by node{Add,Update}, so enqueue machines if they // have no nodeRef yet to make matching happen ASAP for _, machine := range machinesList.Items { if machine.Status.NodeRef == nil { diff --git a/pkg/controller/machinedeployment/controller.go b/pkg/controller/machinedeployment/controller.go index 615043f00..1ff2def02 100644 --- a/pkg/controller/machinedeployment/controller.go +++ b/pkg/controller/machinedeployment/controller.go @@ -278,7 +278,7 @@ func (r *ReconcileMachineDeployment) getMachineDeploymentsForMachineSet(ctx cont return deployments } -// MachineSetTodeployments is a handler.MapFunc to be used to enqeue requests for reconciliation +// MachineSetTodeployments is a handler.MapFunc to be used to enqueue requests for reconciliation // for MachineDeployments that might adopt an orphaned MachineSet. func (r *ReconcileMachineDeployment) MachineSetToDeployments() handler.MapFunc { return func(ctx context.Context, o ctrlruntimeclient.Object) []ctrlruntime.Request { diff --git a/pkg/controller/machineset/controller.go b/pkg/controller/machineset/controller.go index b7e20e197..9db8acbb2 100644 --- a/pkg/controller/machineset/controller.go +++ b/pkg/controller/machineset/controller.go @@ -404,7 +404,7 @@ func (r *ReconcileMachineSet) waitForMachineDeletion(ctx context.Context, machin return nil } -// MachineToMachineSets is a handler.ToRequestsFunc to be used to enqeue requests for reconciliation +// MachineToMachineSets is a handler.ToRequestsFunc to be used to enqueue requests for reconciliation // for MachineSets that might adopt an orphaned Machine. func (r *ReconcileMachineSet) MachineToMachineSets() handler.MapFunc { return func(ctx context.Context, o ctrlruntimeclient.Object) []ctrlruntime.Request { From a941ab2340394d0f54a8472e7edccb1b0de13e0c Mon Sep 17 00:00:00 2001 From: Frank <639906+syphernl@users.noreply.github.com> Date: Tue, 29 Jul 2025 11:17:43 +0200 Subject: [PATCH 478/489] feat(openstack): add support for defining multiple networks (#1950) * feat(openstack): add support for defining multiple networks Signed-off-by: Frank Klaassen <639906+syphernl@users.noreply.github.com> * fix tests Signed-off-by: Frank Klaassen <639906+syphernl@users.noreply.github.com> * fixes Signed-off-by: Frank Klaassen <639906+syphernl@users.noreply.github.com> * fix TestMigratingMachine Signed-off-by: Frank Klaassen <639906+syphernl@users.noreply.github.com> * handle both network and networks Signed-off-by: Frank Klaassen <639906+syphernl@users.noreply.github.com> --------- Signed-off-by: Frank Klaassen <639906+syphernl@users.noreply.github.com> Signed-off-by: Frank <639906+syphernl@users.noreply.github.com> --- .../provider/openstack/provider.go | 118 +++++-- .../provider/openstack/provider_test.go | 291 ++++++++++++++---- .../machinesv1alpha1machine/openstack.yaml | 4 + sdk/cloudprovider/openstack/types.go | 1 + test/e2e/provisioning/all_e2e_test.go | 1 + ...eployment-openstack-multiple-networks.yaml | 53 ++++ 6 files changed, 387 insertions(+), 81 deletions(-) create mode 100644 test/e2e/provisioning/testdata/machinedeployment-openstack-multiple-networks.yaml diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 7fcb69ded..a2ec0806f 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -98,6 +98,7 @@ type Config struct { Flavor string SecurityGroups []string Network string + Networks []string Subnet string FloatingIPPool string AvailabilityZone string @@ -177,6 +178,31 @@ func (p *provider) getConfigAuth(c *Config, rawConfig *openstacktypes.RawConfig) return nil } +func (p *provider) resolveNetworks(cfg *Config) ([]string, error) { + if len(cfg.Networks) > 0 { + networks := make([]string, 0, len(cfg.Networks)+1) + seen := make(map[string]struct{}) + if cfg.Network != "" { + networks = append(networks, cfg.Network) + seen[cfg.Network] = struct{}{} + } + for _, n := range cfg.Networks { + if _, exists := seen[n]; !exists { + networks = append(networks, n) + seen[n] = struct{}{} + } + } + if len(networks) == 0 { + return nil, fmt.Errorf("no networks specified") + } + return networks, nil + } + if cfg.Network != "" { + return []string{cfg.Network}, nil + } + return nil, fmt.Errorf("no networks specified") +} + func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfig.Config, *openstacktypes.RawConfig, error) { pconfig, err := providerconfig.GetConfig(provSpec) if err != nil { @@ -247,6 +273,16 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, nil, err } + for _, network := range rawConfig.Networks { + networkValue, err := p.configVarResolver.GetStringValue(network) + if err != nil { + return nil, nil, nil, err + } + if networkValue != "" { + cfg.Networks = append(cfg.Networks, networkValue) + } + } + cfg.Subnet, err = p.configVarResolver.GetStringValue(rawConfig.Subnet) if err != nil { return nil, nil, nil, err @@ -398,7 +434,7 @@ func (p *provider) AddDefaults(log *zap.SugaredLogger, spec clusterv1alpha1.Mach return spec, err } - if c.Network == "" { + if c.Network == "" && len(c.Networks) == 0 { log.Debug("Trying to default network for machine...") net, err := getDefaultNetwork(netClient) if err != nil { @@ -406,7 +442,7 @@ func (p *provider) AddDefaults(log *zap.SugaredLogger, spec clusterv1alpha1.Mach } if net != nil { log.Debugw("Defaulted network for machine ", "network", net.Name) - // Use the id as the name may not be unique + // Use the single network field for backward compatibility when defaulting rawConfig.Network.Value = net.ID } } @@ -414,22 +450,28 @@ func (p *provider) AddDefaults(log *zap.SugaredLogger, spec clusterv1alpha1.Mach if c.Subnet == "" { log.Debug("Trying to default subnet for machine...") - networkID := c.Network - if rawConfig.Network.Value != "" { - networkID = rawConfig.Network.Value + var primaryNetworkID string + if len(c.Networks) > 0 { + primaryNetworkID = c.Networks[0] + } else if c.Network != "" { + primaryNetworkID = c.Network + } else if rawConfig.Network.Value != "" { + primaryNetworkID = rawConfig.Network.Value } - net, err := getNetwork(netClient, networkID) - if err != nil { - return spec, osErrorToTerminalError(log, err, fmt.Sprintf("failed to get network for subnet defaulting '%s", networkID)) - } - subnet, err := getDefaultSubnet(netClient, net) - if err != nil { - return spec, osErrorToTerminalError(log, err, "error defaulting subnet") - } - if subnet != nil { - log.Debugw("Defaulted subnet for machine", "subnet", *subnet) - rawConfig.Subnet.Value = *subnet + if primaryNetworkID != "" { + net, err := getNetwork(netClient, primaryNetworkID) + if err != nil { + return spec, osErrorToTerminalError(log, err, fmt.Sprintf("failed to get network for subnet defaulting '%s", primaryNetworkID)) + } + subnet, err := getDefaultSubnet(netClient, net) + if err != nil { + return spec, osErrorToTerminalError(log, err, "error defaulting subnet") + } + if subnet != nil { + log.Debugw("Defaulted subnet for machine", "subnet", *subnet) + rawConfig.Subnet.Value = *subnet + } } } @@ -519,8 +561,16 @@ func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec cluste return err } - if _, err := getNetwork(netClient, c.Network); err != nil { - return fmt.Errorf("failed to get network %q: %w", c.Network, err) + networks, err := p.resolveNetworks(c) + if err != nil { + return err + } + + // Validate each network exists + for _, networkName := range networks { + if _, err := getNetwork(netClient, networkName); err != nil { + return fmt.Errorf("failed to get network %q: %w", networkName, err) + } } if _, err := getSubnet(netClient, c.Subnet); err != nil { @@ -536,6 +586,7 @@ func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec cluste if _, err := getAvailabilityZone(computeClient, c); err != nil { return fmt.Errorf("failed to get availability zone %q: %w", c.AvailabilityZone, err) } + // Optional fields. if len(c.SecurityGroups) != 0 { for _, s := range c.SecurityGroups { @@ -593,9 +644,30 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * return nil, err } - network, err := getNetwork(netClient, cfg.Network) + networkNames, err := p.resolveNetworks(cfg) if err != nil { - return nil, osErrorToTerminalError(log, err, fmt.Sprintf("failed to get network %s", cfg.Network)) + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Failed to resolve networks: %v", err), + } + } + + // Get network objects for all specified networks + var networks []osservers.Network + var primaryNetwork *osnetworks.Network // Keep track of first network for floating IP assignment + + for i, networkName := range networkNames { + network, err := getNetwork(netClient, networkName) + if err != nil { + return nil, osErrorToTerminalError(log, err, fmt.Sprintf("failed to get network %s", networkName)) + } + + networks = append(networks, osservers.Network{UUID: network.ID}) + + // Use first network as primary for floating IP assignment (backwards compatibility) + if i == 0 { + primaryNetwork = network + } } securityGroups := cfg.SecurityGroups @@ -619,7 +691,7 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * ConfigDrive: &cfg.ConfigDrive, SecurityGroups: securityGroups, AvailabilityZone: cfg.AvailabilityZone, - Networks: []osservers.Network{{UUID: network.ID}}, + Networks: networks, Metadata: allTags, } @@ -670,12 +742,12 @@ func (p *provider) Create(ctx context.Context, log *zap.SugaredLogger, machine * if cfg.FloatingIPPool != "" { instanceLog := log.With("instance", server.ID) - if err := p.portReadinessWaiter(ctx, instanceLog, netClient, server.ID, network.ID, cfg.InstanceReadyCheckPeriod, cfg.InstanceReadyCheckTimeout); err != nil { + if err := p.portReadinessWaiter(ctx, instanceLog, netClient, server.ID, primaryNetwork.ID, cfg.InstanceReadyCheckPeriod, cfg.InstanceReadyCheckTimeout); err != nil { instanceLog.Infow("Port for instance did not became active", zap.Error(err)) } // Find a free FloatingIP or allocate a new one. - if err := assignFloatingIPToInstance(instanceLog, data.Update, machine, netClient, server.ID, cfg.FloatingIPPool, cfg.Region, network); err != nil { + if err := assignFloatingIPToInstance(instanceLog, data.Update, machine, netClient, server.ID, cfg.FloatingIPPool, cfg.Region, primaryNetwork); err != nil { defer deleteInstanceDueToFatalLogged(instanceLog, computeClient, server.ID) return nil, fmt.Errorf("failed to assign a floating ip to instance %s: %w", server.ID, err) } diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index 37f3d709e..ec130a6e4 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -67,8 +67,7 @@ const expectedServerRequest = `{ ], "user_data": "ZmFrZS11c2VyZGF0YQ==" } -} -` +}` const expectedBlockDeviceBootRequest = `{ "server": { @@ -145,6 +144,36 @@ const expectedBlockDeviceBootVolumeTypeRequest = `{ } }` +const expectedMultipleNetworksRequest = `{ + "server": { + "availability_zone": "eu-de-01", + "config_drive": false, + "flavorRef": "1", + "imageRef": "1bea47ed-f6a9-463b-b423-14b9cca9ad27", + "metadata": { + "kubernetes-cluster": "xyz", + "machine-uid": "", + "system-cluster": "zyx", + "system-project": "xxx" + }, + "name": "test", + "networks": [ + { + "uuid": "d32019d3-bc6e-4319-9c1d-6722fc136a22" + }, + { + "uuid": "1df1458e-bd0c-423d-b201-2e5f56c94714" + } + ], + "security_groups": [ + { + "name": "kubernetes-xyz" + } + ], + "user_data": "ZmFrZS11c2VyZGF0YQ==" + } +}` + type openstackProviderSpecConf struct { IdentityEndpointURL string RootDiskSizeGB *int32 @@ -157,69 +186,95 @@ type openstackProviderSpecConf struct { TenantName string ConfigDrive bool ComputeAPIVersion string + Network string + Networks []string + Subnet string } func (o openstackProviderSpecConf) rawProviderSpec(t *testing.T) []byte { var out bytes.Buffer - tmpl, err := template.New("test").Parse(`{ - "cloudProvider": "openstack", - "cloudProviderSpec": { - "availabilityZone": "eu-de-01", - "domainName": "openstack_domain_name", - "flavor": "m1.tiny", - "identityEndpoint": "{{ .IdentityEndpointURL }}", - "image": "Standard_Ubuntu_18.04_latest", - "network": "public", - "nodeVolumeAttachLimit": null, - "region": "eu-de", - "instanceReadyCheckPeriod": "2m", - "instanceReadyCheckTimeout": "2m", - {{- if .ComputeAPIVersion }} - "computeAPIVersion": {{ .ComputeAPIVersion }}, - {{- end }} - {{- if .RootDiskSizeGB }} - "rootDiskSizeGB": {{ .RootDiskSizeGB }}, - {{- end }} - {{- if .RootDiskVolumeType }} - "rootDiskVolumeType": "{{ .RootDiskVolumeType }}", - {{- end }} - "securityGroups": [ - "kubernetes-xyz" - ], - "subnet": "subnetid", - "tags": { - "kubernetes-cluster": "xyz", - "system-cluster": "zyx", - "system-project": "xxx" + tmplStr := `{ + "cloudProvider": "openstack", + "cloudProviderSpec": { + "availabilityZone": "eu-de-01", + "domainName": "openstack_domain_name", + "flavor": "m1.tiny", + "identityEndpoint": "{{ .IdentityEndpointURL }}", + "image": "Standard_Ubuntu_18.04_latest", + {{- if .Network }} + "network": "{{ .Network }}", + {{- end }} + {{- if .Subnet }} + "subnet": "{{ .Subnet }}", + {{- end }} + {{- if .Networks }} + "networks": [ + {{- range $i, $e := .Networks }} + {{- if $i }},{{- end }} + "{{ $e }}" + {{- end }} + ], + {{- end }} + "nodeVolumeAttachLimit": null, + "region": "eu-de", + "instanceReadyCheckPeriod": "2m", + "instanceReadyCheckTimeout": "2m", + {{- if .ComputeAPIVersion }} + "computeAPIVersion": "{{ .ComputeAPIVersion }}", + {{- end }} + {{- if .RootDiskSizeGB }} + "rootDiskSizeGB": {{ .RootDiskSizeGB }}, + {{- end }} + {{- if .RootDiskVolumeType }} + "rootDiskVolumeType": "{{ .RootDiskVolumeType }}", + {{- end }} + "securityGroups": [ + "kubernetes-xyz" + ], + "tags": { + "kubernetes-cluster": "xyz", + "system-cluster": "zyx", + "system-project": "xxx" + }, + {{- if .ApplicationCredentialID }} + "applicationCredentialID": "{{ .ApplicationCredentialID }}", + "applicationCredentialSecret": "{{ .ApplicationCredentialSecret }}", + {{- else }} + {{- if .ProjectID }} + "projectID": "{{ .ProjectID }}", + "projectName": "{{ .ProjectName }}", + {{- end }} + {{- if .TenantID }} + "tenantID": "{{ .TenantID }}", + "tenantName": "{{ .TenantName }}", + {{- end }} + "username": "dummy", + "password": "this_is_a_password", + {{- end }} + "tokenId": "", + "trustDevicePath": false }, - {{- if .ApplicationCredentialID }} - "applicationCredentialID": "{{ .ApplicationCredentialID }}", - "applicationCredentialSecret": "{{ .ApplicationCredentialSecret }}", - {{- else }} - {{ if .ProjectID }} - "projectID": "{{ .ProjectID }}", - "projectName": "{{ .ProjectName }}", - {{- end }} - {{- if .TenantID }} - "tenantID": "{{ .TenantID }}", - "tenantName": "{{ .TenantName }}", - {{- end }} - "username": "dummy", - "password": "this_is_a_password", - {{- end }} - "tokenId": "", - "trustDevicePath": false - }, - "operatingSystem": "flatcar", - "operatingSystemSpec": { - "disableAutoUpdate": false, - "disableLocksmithD": true, - "disableUpdateEngine": false - } -}`) + "operatingSystem": "flatcar", + "operatingSystemSpec": { + "disableAutoUpdate": false, + "disableLocksmithD": true, + "disableUpdateEngine": false + } + }` + + tmpl, err := template.New("test").Parse(tmplStr) if err != nil { t.Fatalf("Error occurred while parsing openstack provider spec template: %v", err) } + + if o.Networks == nil && o.Network == "" { + o.Network = "public" + } + + if o.Subnet == "" { + o.Subnet = "subnetid" + } + err = tmpl.Execute(&out, o) if err != nil { t.Fatalf("Error occurred while executing openstack provider spec template: %v", err) @@ -267,6 +322,36 @@ func TestCreateServer(t *testing.T) { userdata: "fake-userdata", wantServerReq: expectedServerRequest, }, + { + name: "Backward compatibility with single network", + specConf: openstackProviderSpecConf{Network: "public", Subnet: "subnetid"}, + userdata: "fake-userdata", + wantServerReq: expectedServerRequest, + }, + { + name: "Networks key used with single network", + specConf: openstackProviderSpecConf{Networks: []string{"public"}}, + userdata: "fake-userdata", + wantServerReq: expectedServerRequest, + }, + { + name: "Duplicate networks provided", + specConf: openstackProviderSpecConf{Networks: []string{"public", "public"}}, + userdata: "fake-userdata", + wantServerReq: expectedServerRequest, + }, + { + name: "Multiple networks provided", + specConf: openstackProviderSpecConf{Networks: []string{"public", "private"}}, + userdata: "fake-userdata", + wantServerReq: expectedMultipleNetworksRequest, + }, + { + name: "Both network and networks specified (network becomes primary)", + specConf: openstackProviderSpecConf{Network: "public", Networks: []string{"private"}}, + userdata: "fake-userdata", + wantServerReq: expectedMultipleNetworksRequest, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -359,6 +444,78 @@ func TestProjectAuthVarsAreCorrectlyLoaded(t *testing.T) { } } +func TestResolveNetworks(t *testing.T) { + tests := []struct { + name string + cfg *Config + expected []string + wantErr bool + }{ + { + name: "Only networks specified", + cfg: &Config{Networks: []string{"public", "private"}}, + expected: []string{"public", "private"}, + wantErr: false, + }, + { + name: "Only network specified (backward compatibility)", + cfg: &Config{Network: "public"}, + expected: []string{"public"}, + wantErr: false, + }, + { + name: "Both network and networks specified (network becomes primary)", + cfg: &Config{Network: "public", Networks: []string{"private"}}, + expected: []string{"public", "private"}, + wantErr: false, + }, + { + name: "Handle duplicated networks", + cfg: &Config{Network: "public", Networks: []string{"public"}}, + expected: []string{"public"}, + wantErr: false, + }, + { + name: "Neither specified", + cfg: &Config{}, + expected: nil, + wantErr: true, + }, + { + name: "Empty networks array", + cfg: &Config{Networks: []string{}}, + expected: nil, + wantErr: true, + }, + } + + p := &provider{} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := p.resolveNetworks(tt.cfg) + + if (err != nil) != tt.wantErr { + t.Errorf("resolveNetworks() error = %v, wantErr %v", err, tt.wantErr) + return + } + + if !tt.wantErr { + if len(result) != len(tt.expected) { + t.Errorf("result = %v, expected %v", result, tt.expected) + t.Errorf("resolveNetworks() result length = %v, expected length %v", len(result), len(tt.expected)) + return + } + + for i, network := range result { + if network != tt.expected[i] { + t.Errorf("resolveNetworks() result[%d] = %v, expected %v", i, network, tt.expected[i]) + } + } + } + }) + } +} + type ServerResponse struct { Server servers.Server `json:"server"` } @@ -374,7 +531,7 @@ func ExpectServerCreated(t *testing.T, expectedServer string) { // expectedServer copied into the response (e.g. name). err := json.Unmarshal([]byte(expectedServer), &res) if err != nil { - t.Fatalf("Error occurred while unmarshalling the expected server manifest.") + t.Fatalf("Error occurred while unmarshaling the expected server manifest: %v", err) } res.Server.ID = "1bea47ed-f6a9-463b-b423-14b9cca9ad27" srvRes, err := json.Marshal(res) @@ -540,6 +697,24 @@ func ExpectServerCreated(t *testing.T, expectedServer string) { "port_security_enabled": true, "dns_domain": "local.", "mtu": 1500 + }, + { + "status": "ACTIVE", + "subnets": [ + "55b45ada-e384-4130-a70b-17df1c3e1d3d" + ], + "name": "private", + "admin_state_up": true, + "tenant_id": "4fd44f30292945e481c7b8a0c8908869", + "shared": false, + "id": "1df1458e-bd0c-423d-b201-2e5f56c94714", + "provider:segmentation_id": 9876543211, + "provider:physical_network": null, + "provider:network_type": "local", + "router:external": false, + "port_security_enabled": true, + "dns_domain": "local.", + "mtu": 1500 } ] }`) diff --git a/sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml b/sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml index 9c0b49dda..bd5e73696 100644 --- a/sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml +++ b/sdk/apis/cluster/v1alpha1/conversions/testdata/machinesv1alpha1machine/openstack.yaml @@ -56,6 +56,10 @@ spec: region: "" # Only required if there is more than one network available network: "" + # If you want to use multiple networks, you can specify them here. The first network in the list will be used as the primary network (e.g. for floating IP's). + # networks: + # - "private-network" + # - "public-network" # Only required if the network has more than one subnet subnet: "" # Can be increased for slower OpenStack setups. No values below 1m (60s) possible as this makes no sense. diff --git a/sdk/cloudprovider/openstack/types.go b/sdk/cloudprovider/openstack/types.go index 27028857b..e544a9d65 100644 --- a/sdk/cloudprovider/openstack/types.go +++ b/sdk/cloudprovider/openstack/types.go @@ -44,6 +44,7 @@ type RawConfig struct { Flavor providerconfig.ConfigVarString `json:"flavor"` SecurityGroups []providerconfig.ConfigVarString `json:"securityGroups,omitempty"` Network providerconfig.ConfigVarString `json:"network,omitempty"` + Networks []providerconfig.ConfigVarString `json:"networks,omitempty"` Subnet providerconfig.ConfigVarString `json:"subnet,omitempty"` FloatingIPPool providerconfig.ConfigVarString `json:"floatingIpPool,omitempty"` AvailabilityZone providerconfig.ConfigVarString `json:"availabilityZone,omitempty"` diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index eae8ced26..632457625 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -74,6 +74,7 @@ const ( OSManifest = "./testdata/machinedeployment-openstack.yaml" OSManifestProjectAuth = "./testdata/machinedeployment-openstack-project-auth.yaml" OSUpgradeManifest = "./testdata/machinedeployment-openstack-upgrade.yml" + OSMultipleNetwork = "./testdata/machinedeployment-openstack-multiple-networks.yaml" invalidMachineManifest = "./testdata/machine-invalid.yaml" kubevirtManifest = "./testdata/machinedeployment-kubevirt.yaml" alibabaManifest = "./testdata/machinedeployment-alibaba.yaml" diff --git a/test/e2e/provisioning/testdata/machinedeployment-openstack-multiple-networks.yaml b/test/e2e/provisioning/testdata/machinedeployment-openstack-multiple-networks.yaml new file mode 100644 index 000000000..32f2a9e3b --- /dev/null +++ b/test/e2e/provisioning/testdata/machinedeployment-openstack-multiple-networks.yaml @@ -0,0 +1,53 @@ +apiVersion: "cluster.k8s.io/v1alpha1" +kind: MachineDeployment +metadata: + name: << MACHINE_NAME >> + namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + name: << MACHINE_NAME >> + template: + metadata: + labels: + name: << MACHINE_NAME >> + spec: + providerSpec: + value: + sshPublicKeys: + - "<< YOUR_PUBLIC_KEY >>" + cloudProvider: "openstack" + cloudProviderSpec: + identityEndpoint: "<< IDENTITY_ENDPOINT >>" + username: "<< USERNAME >>" + password: "<< PASSWORD >>" + tenantName: "<< TENANT_NAME >>" + image: "<< OS_IMAGE >>" + flavor: "m1.tiny" + floatingIpPool: "" + domainName: "<< DOMAIN_NAME >>" + region: "<< REGION >>" + networks: + - "<< NETWORK_NAME >>" + - "test-network-2" + instanceReadyCheckPeriod: 5s + instanceReadyCheckTimeout: 2m + operatingSystem: "<< OS_NAME >>" + operatingSystemSpec: + distUpgradeOnBoot: false + disableAutoUpdate: true + # 'rhelSubscriptionManagerUser' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_USER` + rhelSubscriptionManagerUser: "<< RHEL_SUBSCRIPTION_MANAGER_USER >>" + # 'rhelSubscriptionManagerPassword' is only used for rhel os and can be set via env var `RHEL_SUBSCRIPTION_MANAGER_PASSWORD` + rhelSubscriptionManagerPassword: "<< RHEL_SUBSCRIPTION_MANAGER_PASSWORD >>" + rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" + versions: + kubelet: "<< KUBERNETES_VERSION >>" From 0d86633c46693860a50ef27f4ab4101c9071e2de Mon Sep 17 00:00:00 2001 From: Adrian Berger <43774417+adberger@users.noreply.github.com> Date: Mon, 4 Aug 2025 16:09:07 +0200 Subject: [PATCH 479/489] feat(nutanix): use CachedClient, chore(nutanix): more error log information (#1952) * chore: Add more logging information for nutanix provider Signed-off-by: Adrian Berger * feat: Use clientCache for nutanix provider and activate sessionAuth Signed-off-by: Adrian Berger --------- Signed-off-by: Adrian Berger --- pkg/cloudprovider/provider/nutanix/client.go | 86 +++++++++++++++++--- 1 file changed, 75 insertions(+), 11 deletions(-) diff --git a/pkg/cloudprovider/provider/nutanix/client.go b/pkg/cloudprovider/provider/nutanix/client.go index d557f2acd..dff3a545c 100644 --- a/pkg/cloudprovider/provider/nutanix/client.go +++ b/pkg/cloudprovider/provider/nutanix/client.go @@ -22,10 +22,14 @@ import ( "encoding/json" "errors" "fmt" + "net" + "net/http" + "net/url" + "strconv" "strings" "time" - nutanixclient "github.com/nutanix-cloud-native/prism-go-client" + "github.com/nutanix-cloud-native/prism-go-client/environment/types" nutanixv3 "github.com/nutanix-cloud-native/prism-go-client/v3" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" @@ -38,6 +42,9 @@ import ( "k8s.io/utils/ptr" ) +// Shared client cache to persist between calls. +var clientCache = nutanixv3.NewClientCache(nutanixv3.WithSessionAuth(true)) + const ( invalidCredentials = "invalid Nutanix Credentials" ) @@ -46,6 +53,22 @@ type ClientSet struct { Prism *nutanixv3.Client } +// cachedClientParams implements the nutanixv3.CachedClientParams interface. +type cachedClientParams struct { + managementEndpoint types.ManagementEndpoint + clusterName string +} + +// ManagementEndpoint returns the management endpoint. +func (c *cachedClientParams) ManagementEndpoint() types.ManagementEndpoint { + return c.managementEndpoint +} + +// Key returns a unique key for the client. +func (c *cachedClientParams) Key() string { + return c.clusterName +} + func GetClientSet(config *Config) (*ClientSet, error) { if config == nil { return nil, errors.New("no configuration passed") @@ -63,26 +86,67 @@ func GetClientSet(config *Config) (*ClientSet, error) { return nil, errors.New("no endpoint specified") } + if config.ClusterName == "" { + return nil, errors.New("no clusterName specified") + } + // set up 9440 as default port if none is passed via config port := 9440 if config.Port != nil { port = *config.Port } - credentials := nutanixclient.Credentials{ - URL: fmt.Sprintf("%s:%d", config.Endpoint, port), - Endpoint: config.Endpoint, - Port: fmt.Sprint(port), - Username: config.Username, - Password: config.Password, + // Create the management endpoint URL + endpointURL, err := url.Parse(fmt.Sprintf("https://%s", net.JoinHostPort(config.Endpoint, strconv.Itoa(port)))) + if err != nil { + return nil, fmt.Errorf("failed to parse endpoint URL: %w", err) + } + + // Create the management endpoint + managementEndpoint := types.ManagementEndpoint{ + ApiCredentials: types.ApiCredentials{ + Username: config.Username, + Password: config.Password, + }, + Address: endpointURL, Insecure: config.AllowInsecure, } + // Create cached client parameters + cachedParams := &cachedClientParams{ + managementEndpoint: managementEndpoint, + clusterName: config.ClusterName, + } + + // Prepare client options + var clientOptions []nutanixv3.ClientOption + + // Add proxy configuration if provided if config.ProxyURL != "" { - credentials.ProxyURL = config.ProxyURL + proxyURL, err := url.Parse(config.ProxyURL) + if err != nil { + return nil, fmt.Errorf("failed to parse proxy URL: %w", err) + } + + // Create a custom transport with proxy + transport := &http.Transport{ + Proxy: http.ProxyURL(proxyURL), + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + + clientOptions = append(clientOptions, nutanixv3.WithRoundTripper(transport)) } - clientV3, err := nutanixv3.NewV3Client(credentials) + // Get or create the cached client + clientV3, err := clientCache.GetOrCreate(cachedParams, clientOptions...) if err != nil { return nil, err } @@ -402,13 +466,13 @@ func waitForCompletion(ctx context.Context, client *ClientSet, taskID string, in switch *task.Status { case "INVALID_UUID", "FAILED": - return false, fmt.Errorf("bad status: %s", *task.Status) + return false, fmt.Errorf("bad status: %s, error detail: %s, progress message: %s", *task.Status, *task.ErrorDetail, *task.ProgressMessage) case "QUEUED", "RUNNING": return false, nil case "SUCCEEDED": return true, nil default: - return false, fmt.Errorf("unknown status: %s", *task.Status) + return false, fmt.Errorf("unknown status: %s, error detail: %s, progress message: %s", *task.Status, *task.ErrorDetail, *task.ProgressMessage) } }) } From f6a7dc023cf69047d41078c6c4a7a44c62f868c1 Mon Sep 17 00:00:00 2001 From: Sahil Raja Date: Fri, 22 Aug 2025 19:36:01 +0530 Subject: [PATCH 480/489] add support for rocky linux 9 i.e default to 9.6 (#1951) * add support for rocky linux 9 i.e default to 9.6 Signed-off-by: rajaSahil * fix tests Signed-off-by: rajaSahil --------- Signed-off-by: rajaSahil --- examples/azure-machinedeployment.yaml | 2 +- pkg/cloudprovider/provider/anexia/provider_test.go | 2 +- pkg/cloudprovider/provider/aws/provider.go | 4 ++-- pkg/cloudprovider/provider/azure/provider.go | 14 +++++++------- .../provider/digitalocean/provider.go | 2 +- pkg/cloudprovider/provider/hetzner/provider.go | 2 +- .../testdata/machinedeployment-azure.yaml | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/examples/azure-machinedeployment.yaml b/examples/azure-machinedeployment.yaml index 11d40775a..94b36cb14 100644 --- a/examples/azure-machinedeployment.yaml +++ b/examples/azure-machinedeployment.yaml @@ -65,7 +65,7 @@ spec: location: "westeurope" resourceGroup: "<< YOUR_RESOURCE_GROUP >>" vnetResourceGroup: "<< YOUR_VNET_RESOURCE_GROUP >>" - vmSize: "Standard_F2" + vmSize: "Standard_F2s_v2" # optional disk size values in GB. If not set, the defaults for the vmSize will be used. osDiskSize: 30 dataDiskSize: 30 diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index 30fd475c4..24b51c416 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -39,7 +39,6 @@ import ( "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/progress" "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" "go.uber.org/zap" - "sigs.k8s.io/controller-runtime/pkg/client/fake" cloudprovidererrors "k8c.io/machine-controller/pkg/cloudprovider/errors" cloudprovidertypes "k8c.io/machine-controller/pkg/cloudprovider/types" @@ -51,6 +50,7 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" ) const ( diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index e8e813ab2..d6e51ad7e 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -105,12 +105,12 @@ var ( amiFilters = map[providerconfig.OperatingSystem]map[awstypes.CPUArchitecture]amiFilter{ providerconfig.OperatingSystemRockyLinux: { awstypes.CPUArchitectureX86_64: { - description: "*Rocky-8-EC2-*.x86_64", + description: "*Rocky-9-EC2-*.x86_64", // The AWS marketplace ID from Rocky Linux Community Platform Engineering (CPE) owner: "792107900819", }, awstypes.CPUArchitectureARM64: { - description: "*Rocky-8-EC2-*.aarch64", + description: "*Rocky-9-EC2-*.aarch64", // The AWS marketplace ID from Rocky Linux Community Platform Engineering (CPE) owner: "792107900819", }, diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index d4971ee16..c46fb5475 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -159,10 +159,10 @@ var imageReferences = map[providerconfig.OperatingSystem]compute.ImageReference{ Version: to.StringPtr("3374.2.0"), }, providerconfig.OperatingSystemRockyLinux: { - Publisher: to.StringPtr("procomputers"), - Offer: to.StringPtr("rocky-linux-8-5"), - Sku: to.StringPtr("rocky-linux-8-5"), - Version: to.StringPtr("8.5.20211118"), + Publisher: to.StringPtr("resf"), + Offer: to.StringPtr("rockylinux-x86_64"), + Sku: to.StringPtr("9-base"), + Version: to.StringPtr("9.6.20250531"), }, } @@ -178,9 +178,9 @@ var osPlans = map[providerconfig.OperatingSystem]*compute.Plan{ Product: ptr.To("rhel-byos"), }, providerconfig.OperatingSystemRockyLinux: { - Name: ptr.To("rocky-linux-8-5"), - Publisher: ptr.To("procomputers"), - Product: ptr.To("rocky-linux-8-5"), + Name: ptr.To("9-base"), + Publisher: ptr.To("resf"), + Product: ptr.To("rockylinux-x86_64"), }, } diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index d2e99dfa2..da65bc846 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -86,7 +86,7 @@ func getSlugForOS(os providerconfig.OperatingSystem) (string, error) { case providerconfig.OperatingSystemUbuntu: return "ubuntu-24-04-x64", nil case providerconfig.OperatingSystemRockyLinux: - return "rockylinux-8-x64", nil + return "rockylinux-9-x64", nil } return "", providerconfig.ErrOSNotSupported } diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 797a8ab90..81b943716 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -73,7 +73,7 @@ func getNameForOS(os providerconfig.OperatingSystem) (string, error) { case providerconfig.OperatingSystemUbuntu: return "ubuntu-24.04", nil case providerconfig.OperatingSystemRockyLinux: - return "rocky-8", nil + return "rocky-9", nil } return "", providerconfig.ErrOSNotSupported } diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure.yaml index 3b6ed09d4..a25a725a3 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure.yaml @@ -33,7 +33,7 @@ spec: location: "westeurope" resourceGroup: "machine-controller-e2e" vnetResourceGroup: "" - vmSize: "Standard_F2" + vmSize: "Standard_F2s_v2" # optional disk size values in GB. If not set, the defaults for the vmSize will be used. osDiskSize: << OS_DISK_SIZE >> osDiskSKU: << AZURE_OS_DISK_SKU >> From bfda6d694538ed85c97d8d2bc99887ca11b542a3 Mon Sep 17 00:00:00 2001 From: Waleed Malik Date: Wed, 3 Sep 2025 12:44:23 +0500 Subject: [PATCH 481/489] Use flatcar-container-linux-corevm-amd64 for flatcar on Azure (#1956) Signed-off-by: Waleed Malik --- pkg/cloudprovider/provider/azure/provider.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index c46fb5475..274b46a1b 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -154,9 +154,10 @@ var imageReferences = map[providerconfig.OperatingSystem]compute.ImageReference{ }, providerconfig.OperatingSystemFlatcar: { Publisher: to.StringPtr("kinvolk"), - Offer: to.StringPtr("flatcar-container-linux"), - Sku: to.StringPtr("stable"), - Version: to.StringPtr("3374.2.0"), + // flatcar-container-linux-corevm-amd64 doesn't require a plan. For more info: https://www.flatcar.org/docs/latest/installing/cloud/azure/#corevm + Offer: to.StringPtr("flatcar-container-linux-corevm-amd64"), + Sku: to.StringPtr("stable"), + Version: to.StringPtr("4230.2.2"), }, providerconfig.OperatingSystemRockyLinux: { Publisher: to.StringPtr("resf"), @@ -167,11 +168,6 @@ var imageReferences = map[providerconfig.OperatingSystem]compute.ImageReference{ } var osPlans = map[providerconfig.OperatingSystem]*compute.Plan{ - providerconfig.OperatingSystemFlatcar: { - Name: ptr.To("stable"), - Publisher: ptr.To("kinvolk"), - Product: ptr.To("flatcar-container-linux"), - }, providerconfig.OperatingSystemRHEL: { Name: ptr.To("rhel-lvm85"), Publisher: ptr.To("redhat"), From 5d1aa41dc6041cf37c9bb5ad9e075d29227ca5af Mon Sep 17 00:00:00 2001 From: Sahil Raja Date: Thu, 11 Sep 2025 16:10:26 +0530 Subject: [PATCH 482/489] add support for rhel 9 (#1954) Signed-off-by: rajaSahil --- pkg/cloudprovider/provider/aws/provider.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index d6e51ad7e..8c48823d6 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -1176,13 +1176,13 @@ func getInstanceCountForMachine(machine clusterv1alpha1.Machine, reservations [] func filterSupportedRHELImages(images []ec2types.Image) ([]ec2types.Image, error) { var filteredImages []ec2types.Image for _, image := range images { - if strings.HasPrefix(*image.Name, "RHEL-8") { + if strings.HasPrefix(*image.Name, "RHEL-9") { filteredImages = append(filteredImages, image) } } if filteredImages == nil { - return nil, errors.New("rhel 8 images are not found") + return nil, errors.New("rhel 9 images are not found") } return filteredImages, nil From 3e25421983504cf443ca7d67b9eea3c19fd07610 Mon Sep 17 00:00:00 2001 From: Sahil Raja Date: Tue, 16 Sep 2025 14:21:22 +0530 Subject: [PATCH 483/489] add support for rhel 9 (#1963) Signed-off-by: rajaSahil --- pkg/cloudprovider/provider/azure/provider.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index 274b46a1b..9b828adcb 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -149,8 +149,8 @@ var imageReferences = map[providerconfig.OperatingSystem]compute.ImageReference{ providerconfig.OperatingSystemRHEL: { Publisher: to.StringPtr("RedHat"), Offer: to.StringPtr("rhel-byos"), - Sku: to.StringPtr("rhel-lvm85"), - Version: to.StringPtr("8.5.20220316"), + Sku: to.StringPtr("rhel-lvm95"), + Version: to.StringPtr("9.5.2024112215"), }, providerconfig.OperatingSystemFlatcar: { Publisher: to.StringPtr("kinvolk"), @@ -169,7 +169,7 @@ var imageReferences = map[providerconfig.OperatingSystem]compute.ImageReference{ var osPlans = map[providerconfig.OperatingSystem]*compute.Plan{ providerconfig.OperatingSystemRHEL: { - Name: ptr.To("rhel-lvm85"), + Name: ptr.To("rhel-lvm95"), Publisher: ptr.To("redhat"), Product: ptr.To("rhel-byos"), }, From ad427b248b39a2091d8888e4645f50fa07e192dd Mon Sep 17 00:00:00 2001 From: Sahil Raja Date: Wed, 17 Sep 2025 16:48:24 +0530 Subject: [PATCH 484/489] update rockylinux and rhel image for e2e tests (#1964) * update image for rockylinux 9 and rhel 9 Signed-off-by: rajaSahil * update storage class name Signed-off-by: rajaSahil * update vm template name for vSphere Signed-off-by: rajaSahil --------- Signed-off-by: rajaSahil --- test/e2e/provisioning/helper.go | 8 ++++---- .../provisioning/testdata/machinedeployment-kubevirt.yaml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 2ccc9b439..9fca707ea 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -49,9 +49,9 @@ var ( openStackImages = map[string]string{ string(providerconfigtypes.OperatingSystemUbuntu): "kubermatic-ubuntu", - string(providerconfigtypes.OperatingSystemRHEL): "machine-controller-e2e-rhel-8-5", + string(providerconfigtypes.OperatingSystemRHEL): "machine-controller-e2e-rhel-9-6", string(providerconfigtypes.OperatingSystemFlatcar): "kubermatic-e2e-flatcar", - string(providerconfigtypes.OperatingSystemRockyLinux): "machine-controller-e2e-rockylinux", + string(providerconfigtypes.OperatingSystemRockyLinux): "machine-controller-e2e-rockylinux-9-6", } openNebulaImages = map[string]string{ @@ -61,8 +61,8 @@ var ( vSphereOSImageTemplates = map[string]string{ string(providerconfigtypes.OperatingSystemFlatcar): "kkp-flatcar-3139.2.0", - string(providerconfigtypes.OperatingSystemRHEL): "kkp-rhel-8.6", - string(providerconfigtypes.OperatingSystemRockyLinux): "kkp-rockylinux-8", + string(providerconfigtypes.OperatingSystemRHEL): "kkp-rhel-9.6", + string(providerconfigtypes.OperatingSystemRockyLinux): "kkp-rockylinux-9.6", string(providerconfigtypes.OperatingSystemUbuntu): "kkp-ubuntu-24.04", } diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index d97169adf..e903f266f 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -38,7 +38,7 @@ spec: primaryDisk: osImage: http://image-repo.kube-system.svc/images/<< KUBEVIRT_OS_IMAGE >>.img size: "25Gi" - storageClassName: rook-ceph-block + storageClassName: local-path dnsPolicy: "None" dnsConfig: nameservers: From e7f85ba4d31d422a5cc41f1119f897a99609fae5 Mon Sep 17 00:00:00 2001 From: Archana Sawant Date: Thu, 18 Sep 2025 13:06:25 +0530 Subject: [PATCH 485/489] Add support for k8s 1.34 (#1955) * Add support for k8s 1.34 Signed-off-by: archups * Bump controller-runtime dependency to v0.22.0 Signed-off-by: archups * Update golangci for-lint staticcheck exclusion Signed-off-by: archups Fix lint error Signed-off-by: archups * Bump Go build image to 1.25 in Prow jobs Signed-off-by: archups * Update golangci-lint to exclude few existing noctx func implementation Signed-off-by: archups * Update Go build image tag in prow job and fix test failure issue Signed-off-by: archups * Update the kubelet version to latest-1 Signed-off-by: archups * Add supported k8s entries to e2e tests Signed-off-by: archups * Fix noctx lint issues Signed-off-by: archups * Bump Go version to secure one 1.25.1/1.24.7 Signed-off-by: archups * Update e2e test with k8s versions Signed-off-by: archups * Bump kubermatic/util image to 2.7.0 and kubermatic/startup-script to 0.3.0 Signed-off-by: archups * Bump k8s patch releases Signed-off-by: Archana Sawant * Remove negative selectors Signed-off-by: Archana Sawant --------- Signed-off-by: archups Signed-off-by: Archana Sawant --- .golangci.yml | 5 + .prow/e2e-features.yaml | 8 +- .prow/postsubmits.yaml | 4 +- .prow/provider-alibaba.yaml | 2 +- .prow/provider-anexia.yaml | 2 +- .prow/provider-aws.yaml | 12 +- .prow/provider-azure.yaml | 6 +- .prow/provider-digitalocean.yaml | 2 +- .prow/provider-equinix-metal.yaml | 2 +- .prow/provider-gcp.yaml | 2 +- .prow/provider-hetzner.yaml | 2 +- .prow/provider-kubevirt.yaml | 2 +- .prow/provider-linode.yaml | 2 +- .prow/provider-nutanix.yaml | 2 +- .prow/provider-openstack.yaml | 4 +- .prow/provider-scaleway.yaml | 2 +- .prow/provider-vmware-cloud-director.yaml | 2 +- .prow/provider-vsphere.yaml | 10 +- .prow/verify.yaml | 16 +- Dockerfile | 2 +- Makefile | 2 +- README.md | 5 +- examples/alibaba-machinedeployment.yaml | 2 +- examples/anexia-machinedeployment.yaml | 2 +- examples/aws-machinedeployment.yaml | 2 +- examples/azure-machinedeployment.yaml | 2 +- examples/digitalocean-machinedeployment.yaml | 2 +- examples/equinixmetal-machinedeployment.yaml | 2 +- examples/gce-machinedeployment.yaml | 2 +- examples/hetzner-machinedeployment.yaml | 2 +- examples/kubevirt-local-mounter.yaml | 2 +- examples/kubevirt-machinedeployment.yaml | 2 +- examples/linode-machinedeployment.yaml | 2 +- examples/nutanix-machinedeployment.yaml | 2 +- examples/opennebula-machinedeployment.yaml | 2 +- examples/openstack-machinedeployment.yaml | 2 +- examples/scaleway-machinedeployment.yaml | 2 +- ...ware-cloud-director-machinedeployment.yaml | 2 +- ...e-datastore-cluster-machinedeployment.yaml | 2 +- examples/vsphere-machinedeployment.yaml | 2 +- examples/vultr-machinedeployment.yaml | 2 +- go.mod | 75 ++++----- go.sum | 150 ++++++++++-------- hack/lib.sh | 2 +- hack/update-fixtures.sh | 2 +- hack/verify-licenses.sh | 2 +- .../provider/kubevirt/provider_test.go | 7 +- pkg/cloudprovider/provider/vsphere/helper.go | 4 +- .../provider/vsphere/provider.go | 2 +- pkg/clusterinfo/configmap_test.go | 2 - pkg/rhsm/satellite_subscription_manager.go | 2 +- pkg/rhsm/subscription_manager.go | 4 +- test/e2e/provisioning/all_e2e_test.go | 20 +-- test/e2e/provisioning/helper.go | 7 +- 54 files changed, 218 insertions(+), 195 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 3f32b1828..ab56b3715 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -96,6 +96,11 @@ linters: text: cyclomatic complexity [0-9]+ of func `\(\*provider\)\.getConfig` is high - path: (.+)\.go$ text: 'SA1019: s.server.IPv6 is deprecated' + # TODO: Should be fixed via https://github.com/kubermatic/machine-controller/issues/1960 + - path: (.+)\.go$ + text: 'SA1019: corev1.Endpoints is deprecated: This API is deprecated in v1.33+.' + - path: (.+)\.go$ + text: 'SA1019: corev1.EndpointSubset is deprecated: This API is deprecated in v1.33+.' paths: - apis/machines - third_party$ diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index beda7a53d..65c4949e5 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -35,7 +35,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -65,7 +65,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -95,7 +95,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -123,7 +123,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index e4000d373..306aebf5c 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -27,7 +27,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - /bin/bash - -c @@ -56,7 +56,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index d539d3d11..ca537ca50 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index bd6b6e4c8..980319c01 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -32,7 +32,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index dc0437276..3ebf52720 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -96,7 +96,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -130,7 +130,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -162,7 +162,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -194,7 +194,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 112724446..5221c1681 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -96,7 +96,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 3131f3a7c..e8427f12a 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index b70cdef95..5b52d8529 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index f20b57669..239bf7ed0 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index a4b2a6d76..a22938b56 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -27,7 +27,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 6eb67c22e..a1ec2074d 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -31,7 +31,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 5198a8c6b..d60a09e04 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index 68b435c0e..b79fafc60 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index 562ca575e..02dd62fe2 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -30,7 +30,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -64,7 +64,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 63f67bb81..9e9b21d1b 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -28,7 +28,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 39134beb3..c24986a56 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -32,7 +32,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 2dd7a9453..7f4287805 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -29,7 +29,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -62,7 +62,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -95,7 +95,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -128,7 +128,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -161,7 +161,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-kind-0.27-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-kind-0.30-4 command: - "./hack/ci/run-e2e-tests.sh" args: diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 3f212142f..dbc05a702 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -22,7 +22,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-4 command: - make args: @@ -44,7 +44,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-4 command: - make args: @@ -66,7 +66,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-4 command: - make args: @@ -87,7 +87,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-4 command: - make args: @@ -107,7 +107,7 @@ presubmits: path_alias: k8c.io/machine-controller spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-4 command: - "/usr/local/bin/shfmt" args: @@ -136,7 +136,7 @@ presubmits: path_alias: k8c.io/machine-controller spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-4 command: - "./hack/verify-boilerplate.sh" resources: @@ -156,7 +156,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-4 command: - ./hack/verify-licenses.sh resources: @@ -173,7 +173,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.24-node-20-6 + - image: quay.io/kubermatic/build:go-1.25-node-22-4 command: - make args: diff --git a/Dockerfile b/Dockerfile index 633546fa4..cf38a1b4b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.24.4 +ARG GO_VERSION=1.25.1 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/k8c.io/machine-controller COPY . . diff --git a/Makefile b/Makefile index d2558cc1c..1d6e4c89e 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.24.4 +GO_VERSION ?= 1.25.1 GOOS ?= $(shell go env GOOS) diff --git a/README.md b/README.md index 858bc3aac..fbe372ebf 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,10 @@ machine-controller tries to follow the Kubernetes version Currently supported K8S versions are: +- 1.34 +- 1.33 +- 1.32 - 1.31 -- 1.30 -- 1.29 ### Community Providers diff --git a/examples/alibaba-machinedeployment.yaml b/examples/alibaba-machinedeployment.yaml index 6fb621c98..506b61a03 100644 --- a/examples/alibaba-machinedeployment.yaml +++ b/examples/alibaba-machinedeployment.yaml @@ -62,4 +62,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index da31dd1d7..64722b6ef 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -83,4 +83,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/aws-machinedeployment.yaml b/examples/aws-machinedeployment.yaml index d111744ed..127012e35 100644 --- a/examples/aws-machinedeployment.yaml +++ b/examples/aws-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/azure-machinedeployment.yaml b/examples/azure-machinedeployment.yaml index 94b36cb14..abfe62ba8 100644 --- a/examples/azure-machinedeployment.yaml +++ b/examples/azure-machinedeployment.yaml @@ -91,4 +91,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/digitalocean-machinedeployment.yaml b/examples/digitalocean-machinedeployment.yaml index 50793d6c5..e57123176 100644 --- a/examples/digitalocean-machinedeployment.yaml +++ b/examples/digitalocean-machinedeployment.yaml @@ -56,4 +56,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/equinixmetal-machinedeployment.yaml b/examples/equinixmetal-machinedeployment.yaml index 66f519943..9540d65e5 100644 --- a/examples/equinixmetal-machinedeployment.yaml +++ b/examples/equinixmetal-machinedeployment.yaml @@ -51,4 +51,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index fa8a06404..e7b2bf239 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -90,4 +90,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/hetzner-machinedeployment.yaml b/examples/hetzner-machinedeployment.yaml index 8f3c80bf7..922d364a2 100644 --- a/examples/hetzner-machinedeployment.yaml +++ b/examples/hetzner-machinedeployment.yaml @@ -67,4 +67,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/kubevirt-local-mounter.yaml b/examples/kubevirt-local-mounter.yaml index 8c2236e02..55aff3497 100644 --- a/examples/kubevirt-local-mounter.yaml +++ b/examples/kubevirt-local-mounter.yaml @@ -15,7 +15,7 @@ spec: hostPID: true containers: - name: startup-script - image: quay.io/kubermatic/startup-script:v0.2.1 + image: quay.io/kubermatic/startup-script:v0.3.0 securityContext: privileged: true env: diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index 567c29f2c..0313d4b8d 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -69,4 +69,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/linode-machinedeployment.yaml b/examples/linode-machinedeployment.yaml index 4df4da709..43ab305ee 100644 --- a/examples/linode-machinedeployment.yaml +++ b/examples/linode-machinedeployment.yaml @@ -53,4 +53,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/nutanix-machinedeployment.yaml b/examples/nutanix-machinedeployment.yaml index 06c6235db..cc8675c84 100644 --- a/examples/nutanix-machinedeployment.yaml +++ b/examples/nutanix-machinedeployment.yaml @@ -82,4 +82,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/opennebula-machinedeployment.yaml b/examples/opennebula-machinedeployment.yaml index e0a48b9bd..87b4f87cf 100644 --- a/examples/opennebula-machinedeployment.yaml +++ b/examples/opennebula-machinedeployment.yaml @@ -67,4 +67,4 @@ spec: # use cloud-init for flatcar as ignition doesn't know anything about OpenNebula yet provisioningUtility: "cloud-init" versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index 5b9dec66e..c99f0f6cf 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -165,4 +165,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/scaleway-machinedeployment.yaml b/examples/scaleway-machinedeployment.yaml index 1f5b8d163..ad56eb180 100644 --- a/examples/scaleway-machinedeployment.yaml +++ b/examples/scaleway-machinedeployment.yaml @@ -60,4 +60,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/vmware-cloud-director-machinedeployment.yaml b/examples/vmware-cloud-director-machinedeployment.yaml index 00a882308..1e011aacd 100644 --- a/examples/vmware-cloud-director-machinedeployment.yaml +++ b/examples/vmware-cloud-director-machinedeployment.yaml @@ -89,4 +89,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml index 4e42ac34c..aa5e92386 100644 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ b/examples/vsphere-datastore-cluster-machinedeployment.yaml @@ -79,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index 85930ca1f..16c662751 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -81,4 +81,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/examples/vultr-machinedeployment.yaml b/examples/vultr-machinedeployment.yaml index b1e2554b9..68416da54 100644 --- a/examples/vultr-machinedeployment.yaml +++ b/examples/vultr-machinedeployment.yaml @@ -70,4 +70,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.30.5 + kubelet: 1.33.4 diff --git a/go.mod b/go.mod index 0ae8126e9..4731d101b 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module k8c.io/machine-controller -go 1.23.0 +go 1.24.0 -toolchain go1.24.0 +toolchain go1.24.7 replace k8c.io/machine-controller/sdk => ./sdk @@ -27,6 +27,7 @@ require ( github.com/go-logr/logr v1.4.2 github.com/go-logr/zapr v1.3.0 github.com/go-test/deep v1.1.0 + github.com/google/go-cmp v0.7.0 github.com/google/uuid v1.6.0 github.com/gophercloud/gophercloud v1.14.0 github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb @@ -37,9 +38,9 @@ require ( github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 - github.com/prometheus/client_golang v1.20.3 + github.com/prometheus/client_golang v1.22.0 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 - github.com/spf13/pflag v1.0.5 + github.com/spf13/pflag v1.0.6 github.com/tinkerbell/tink v0.10.1 github.com/vmware/go-vcloud-director/v2 v2.25.0 github.com/vmware/govmomi v0.43.0 @@ -52,24 +53,24 @@ require ( google.golang.org/api v0.197.0 gopkg.in/yaml.v3 v3.0.1 k8c.io/machine-controller/sdk v0.0.0-00010101000000-000000000000 - k8s.io/api v0.32.1 - k8s.io/apiextensions-apiserver v0.32.1 - k8s.io/apimachinery v0.32.1 - k8s.io/client-go v0.32.1 - k8s.io/cloud-provider v0.32.1 + k8s.io/api v0.34.0 + k8s.io/apiextensions-apiserver v0.34.0 + k8s.io/apimachinery v0.34.0 + k8s.io/client-go v0.34.0 + k8s.io/cloud-provider v0.33.4 k8s.io/klog v1.0.0 - k8s.io/kubectl v0.32.1 - k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 + k8s.io/kubectl v0.33.4 + k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 kubevirt.io/api v1.3.1 kubevirt.io/containerized-data-importer-api v1.60.3 - sigs.k8s.io/controller-runtime v0.20.4 + sigs.k8s.io/controller-runtime v0.22.0 ) require ( cloud.google.com/go v0.115.1 // indirect cloud.google.com/go/auth v0.9.4 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect - cloud.google.com/go/compute/metadata v0.5.1 // indirect + cloud.google.com/go/compute/metadata v0.6.0 // indirect cloud.google.com/go/longrunning v0.6.1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.29 // indirect @@ -94,12 +95,12 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.23.0 // indirect github.com/go-openapi/errors v0.22.0 // indirect @@ -115,12 +116,9 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect - github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect github.com/google/s2a-go v0.1.8 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect @@ -131,7 +129,6 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.9 // indirect github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect @@ -141,7 +138,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/onsi/ginkgo/v2 v2.22.0 // indirect @@ -150,20 +147,23 @@ require ( github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect github.com/peterhellberg/link v1.2.0 // indirect github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.59.1 // indirect + github.com/prometheus/common v0.62.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.mongodb.org/mongo-driver v1.16.1 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 // indirect - go.opentelemetry.io/otel v1.30.0 // indirect - go.opentelemetry.io/otel/metric v1.30.0 // indirect - go.opentelemetry.io/otel/trace v1.30.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/sync v0.12.0 // indirect @@ -173,19 +173,20 @@ require ( golang.org/x/time v0.11.0 // indirect golang.org/x/tools v0.31.0 // indirect google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect - google.golang.org/grpc v1.66.2 // indirect - google.golang.org/protobuf v1.35.1 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/grpc v1.72.1 // indirect + google.golang.org/protobuf v1.36.5 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect k8s.io/klog/v2 v2.130.1 // indirect - k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect + k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect - sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect + sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index 91a69b99f..720b5557a 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ cloud.google.com/go/auth v0.9.4 h1:DxF7imbEbiFu9+zdKC6cKBko1e8XeJnipNqIbWZ+kDI= cloud.google.com/go/auth v0.9.4/go.mod h1:SHia8n6//Ya940F1rLimhJCjjx7KE17t0ctFEci3HkA= cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= -cloud.google.com/go/compute/metadata v0.5.1 h1:NM6oZeZNlYjiwYje+sYFjEpP0Q0zCan1bmQW/KmIrGs= -cloud.google.com/go/compute/metadata v0.5.1/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= +cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= cloud.google.com/go/iam v1.2.0 h1:kZKMKVNk/IsSSc/udOb83K0hL/Yh/Gcqpz+oAkoIFN8= cloud.google.com/go/iam v1.2.0/go.mod h1:zITGuWgsLZxd8OwAlX+eMFgZDXzBm7icj1PVTYG766Q= cloud.google.com/go/logging v1.11.0 h1:v3ktVzXMV7CwHq1MBF65wcqLMA7i+z3YxbUsoK7mOKs= @@ -124,8 +124,8 @@ github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkg github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= -github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= +github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -144,12 +144,12 @@ github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fullstorydev/grpcurl v1.8.7 h1:xJWosq3BQovQ4QrdPO72OrPiWuGgEsxY8ldYsJbPrqI= github.com/fullstorydev/grpcurl v1.8.7/go.mod h1:pVtM4qe3CMoLaIzYS8uvTuDj2jVYmXqMUkZeijnXp/E= -github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= -github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= @@ -232,8 +232,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= -github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= -github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -243,9 +243,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -306,8 +306,8 @@ github.com/keploy/go-sdk v0.9.0 h1:kpSNcCTDdELsa1gWyhoD9oV57SgSMbG/wq6Cjp4y7cY= github.com/keploy/go-sdk v0.9.0/go.mod h1:vNKXoFd2MaK+Gly/K6XeP1Hs9dP834C74szH+vtBPwg= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kolo/xmlrpc v0.0.0-20190717152603-07c4ee3fd181/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b h1:udzkj9S/zlT5X367kqJis0QP7YMxobob6zhzq6Yre00= github.com/kolo/xmlrpc v0.0.0-20220921171641-a4b6fa1dd06b/go.mod h1:pcaDhQK0/NJZEvtCO0qQPPropqV0sJOJ6YW7X+9kRwM= @@ -349,8 +349,9 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -393,19 +394,19 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.20.3 h1:oPksm4K8B+Vt35tUhw6GbSNSgVlVSBH0qELP/7u83l4= -github.com/prometheus/client_golang v1.20.3/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0= -github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.30 h1:yoKAVkEVwAqbGbR8n87rHQ1dulL25rKloGadb3vm770= @@ -415,12 +416,15 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -429,8 +433,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tinkerbell/tink v0.10.1 h1:mxdPQf7n4nB/AVdjbqCm5c98vsITU35g7Yw5cdOWmCw= github.com/tinkerbell/tink v0.10.1/go.mod h1:yULdVrzAfPnA8KdOkjvo8qDn6pw0JD6kBzF94gtXMjA= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= @@ -459,18 +463,22 @@ go.mongodb.org/mongo-driver v1.16.1 h1:rIVLL3q0IHM39dvE+z2ulZLp9ENZKThVfuvN/IiN4 go.mongodb.org/mongo-driver v1.16.1/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0 h1:hCq2hNMwsegUvPzI7sPOvtO9cqyy5GbWt/Ybp2xrx8Q= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.55.0/go.mod h1:LqaApwGx/oUmzsbqxkzuBvyoPpkxk3JQWnqfVrJ3wCA= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0 h1:ZIg3ZT/aQ7AfKqdwp7ECpOK6vHqquXXuyTjIO8ZdmPs= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.55.0/go.mod h1:DQAwmETtZV00skUwgD6+0U89g80NKsJE3DCKeLLPQMI= -go.opentelemetry.io/otel v1.30.0 h1:F2t8sK4qf1fAmY9ua4ohFS/K+FUuOPemHUIXHtktrts= -go.opentelemetry.io/otel v1.30.0/go.mod h1:tFw4Br9b7fOS+uEao81PJjVMjW/5fvNCbpsDIXqP0pc= -go.opentelemetry.io/otel/metric v1.30.0 h1:4xNulvn9gjzo4hjg+wzIKG7iNFEaBMX00Qd4QIZs7+w= -go.opentelemetry.io/otel/metric v1.30.0/go.mod h1:aXTfST94tswhWEb+5QjlSqG+cZlmyXy/u8jFpor3WqQ= -go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBqWyE= -go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= -go.opentelemetry.io/otel/trace v1.30.0 h1:7UBkkYzeg3C7kQX8VAidWh2biiQbtAKjyIML8dQ9wmc= -go.opentelemetry.io/otel/trace v1.30.0/go.mod h1:5EyKqTzzmyqB9bwtCCq6pDLktPK6fmGf/Dph+8VI02o= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 h1:yd02MEjBdJkG3uabWP9apV+OuWRIXGDuJEUJbOHmCFU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0/go.mod h1:umTcuxiv1n/s/S6/c2AT/g2CQ7u5C59sHDNmfSwgz7Q= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= @@ -480,6 +488,10 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= +go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -672,17 +684,17 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1 h1:BulPr26Jqjnd4eYDVe+YvyR7Yc2vJGkO5/0UxD0/jZU= google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:hL97c3SYopEHblzpxRL4lSs523++l8DYxGM1FQiYmb4= -google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc= -google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= +google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= -google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= +google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -696,8 +708,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA= -google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -730,17 +742,17 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= -k8s.io/api v0.32.1 h1:f562zw9cy+GvXzXf0CKlVQ7yHJVYzLfL6JAS4kOAaOc= -k8s.io/api v0.32.1/go.mod h1:/Yi/BqkuueW1BgpoePYBRdDYfjPF5sgTr5+YqDZra5k= -k8s.io/apiextensions-apiserver v0.32.1 h1:hjkALhRUeCariC8DiVmb5jj0VjIc1N0DREP32+6UXZw= -k8s.io/apiextensions-apiserver v0.32.1/go.mod h1:sxWIGuGiYov7Io1fAS2X06NjMIk5CbRHc2StSmbaQto= +k8s.io/api v0.34.0 h1:L+JtP2wDbEYPUeNGbeSa/5GwFtIA662EmT2YSLOkAVE= +k8s.io/api v0.34.0/go.mod h1:YzgkIzOOlhl9uwWCZNqpw6RJy9L2FK4dlJeayUoydug= +k8s.io/apiextensions-apiserver v0.34.0 h1:B3hiB32jV7BcyKcMU5fDaDxk882YrJ1KU+ZSkA9Qxoc= +k8s.io/apiextensions-apiserver v0.34.0/go.mod h1:hLI4GxE1BDBy9adJKxUxCEHBGZtGfIg98Q+JmTD7+g0= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.32.1 h1:683ENpaCBjma4CYqsmZyhEzrGz6cjn1MY/X2jB2hkZs= -k8s.io/apimachinery v0.32.1/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= -k8s.io/client-go v0.32.1 h1:otM0AxdhdBIaQh7l1Q0jQpmo7WOFIk5FFa4bg6YMdUU= -k8s.io/client-go v0.32.1/go.mod h1:aTTKZY7MdxUaJ/KiUs8D+GssR9zJZi77ZqtzcGXIiDg= -k8s.io/cloud-provider v0.32.1 h1:74rRhnfca3o4CsjjnIp/C3ARVuSmyNsxgWPtH0yc9Z0= -k8s.io/cloud-provider v0.32.1/go.mod h1:GECSanFT+EeZ/ToX3xlasjETzMUI+VFu92zHUDUsGHw= +k8s.io/apimachinery v0.34.0 h1:eR1WO5fo0HyoQZt1wdISpFDffnWOvFLOOeJ7MgIv4z0= +k8s.io/apimachinery v0.34.0/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/client-go v0.34.0 h1:YoWv5r7bsBfb0Hs2jh8SOvFbKzzxyNo0nSb0zC19KZo= +k8s.io/client-go v0.34.0/go.mod h1:ozgMnEKXkRjeMvBZdV1AijMHLTh3pbACPvK7zFR+QQY= +k8s.io/cloud-provider v0.33.4 h1:et4DyeV0W8W+m2ByS34VVFMg8Aj0sz+UDVwanNkspTo= +k8s.io/cloud-provider v0.33.4/go.mod h1:cAC2s7mGpqVWwUars8TFgnvgXy+trDOF3+WSeKNsy/M= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= @@ -754,14 +766,14 @@ k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= -k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= -k8s.io/kubectl v0.32.1 h1:/btLtXLQUU1rWx8AEvX9jrb9LaI6yeezt3sFALhB8M8= -k8s.io/kubectl v0.32.1/go.mod h1:sezNuyWi1STk4ZNPVRIFfgjqMI6XMf+oCVLjZen/pFQ= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= +k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/kubectl v0.33.4 h1:nXEI6Vi+oB9hXxoAHyHisXolm/l1qutK3oZQMak4N98= +k8s.io/kubectl v0.33.4/go.mod h1:Xe7P9X4DfILvKmlBsVqUtzktkI56lEj22SJW7cFy6nE= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= -k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= +k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kubevirt.io/api v1.3.1 h1:MoTNo/zvDlZ44c2ocXLPln8XTaQOeUodiYbEKrTCqv4= kubevirt.io/api v1.3.1/go.mod h1:tCn7VAZktEvymk490iPSMPCmKM9UjbbfH2OsFR/IOLU= kubevirt.io/containerized-data-importer-api v1.60.3 h1:kQEXi7scpzUa0RPf3/3MKk1Kmem0ZlqqiuK3kDF5L2I= @@ -769,16 +781,18 @@ kubevirt.io/containerized-data-importer-api v1.60.3/go.mod h1:8mwrkZIdy8j/LmCyKt kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 h1:fZYvD3/Vnitfkx6IJxjLAk8ugnZQ7CXVYcRfkSKmuZY= kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= -sigs.k8s.io/controller-runtime v0.20.4 h1:X3c+Odnxz+iPTRobG4tp092+CvBU9UK0t/bRf+n0DGU= -sigs.k8s.io/controller-runtime v0.20.4/go.mod h1:xg2XB0K5ShQzAgsoujxuKN4LNXR2LfwwHsPj7Iaw+XY= +sigs.k8s.io/controller-runtime v0.22.0 h1:mTOfibb8Hxwpx3xEkR56i7xSjB+nH4hZG37SrlCY5e0= +sigs.k8s.io/controller-runtime v0.22.0/go.mod h1:FwiwRjkRPbiN+zp2QRp7wlTCzbUXxZ/D4OzuQUDwBHY= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= -sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3/go.mod h1:18nIHnGi6636UCz6m8i4DhaJ65T6EruyzmoQqI2BVDo= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= +sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2 h1:MdmvkGuXi/8io6ixD5wud3vOLwc1rj0aNqRlpuvjmwA= -sigs.k8s.io/structured-merge-diff/v4 v4.4.2/go.mod h1:N8f93tFZh9U6vpxwRArLiikrE5/2tiu1w1AGfACIGE4= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= +sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= -sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= -sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/hack/lib.sh b/hack/lib.sh index aa98ead68..cfeb7bb52 100644 --- a/hack/lib.sh +++ b/hack/lib.sh @@ -52,7 +52,7 @@ is_containerized() { containerize() { local cmd="$1" - local image="${CONTAINERIZE_IMAGE:-quay.io/kubermatic/util:2.6.0}" + local image="${CONTAINERIZE_IMAGE:-quay.io/kubermatic/util:2.7.0}" local gocache="${CONTAINERIZE_GOCACHE:-/tmp/.gocache}" local gomodcache="${CONTAINERIZE_GOMODCACHE:-/tmp/.gomodcache}" local skip="${NO_CONTAINERIZE:-}" diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index f2d148e35..290a7cf70 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.24-node-20-6 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.25-node-22-4 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index eb372848c..1182ef49f 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.24-node-20-6 containerize ./hack/verify-licenses.sh +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.25-node-22-4 containerize ./hack/verify-licenses.sh go mod vendor diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go index f2d5518e2..659f4470a 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider_test.go +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -25,6 +25,8 @@ import ( "reflect" "testing" + "github.com/google/go-cmp/cmp" + kubevirtcorev1 "kubevirt.io/api/core/v1" cdicorev1beta1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" @@ -37,7 +39,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apimachinery/pkg/util/diff" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -321,7 +322,9 @@ func TestNewVirtualMachine(t *testing.T) { vm.APIVersion, vm.Kind = kubevirtcorev1.VirtualMachineGroupVersionKind.ToAPIVersionAndKind() if !equality.Semantic.DeepEqual(vm, expectedVms[tt.name]) { - t.Errorf("Diff %v", diff.ObjectGoPrintDiff(expectedVms[tt.name], vm)) + if diff := cmp.Diff(expectedVms[tt.name], vm); diff != "" { + t.Errorf("Diff:\n%s", diff) + } } }) } diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go index e1abcf4d1..899c3d631 100644 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ b/pkg/cloudprovider/provider/vsphere/helper.go @@ -313,7 +313,7 @@ func uploadAndAttachISO(ctx context.Context, log *zap.SugaredLogger, session *Se return vmRef.EditDevice(ctx, devices.InsertIso(cdrom, iso)) } -func generateLocalUserdataISO(userdata, name string) (string, error) { +func generateLocalUserdataISO(ctx context.Context, userdata, name string) (string, error) { // We must create a directory, because the iso-generation commands // take a directory as input userdataDir, err := os.MkdirTemp(localTempDir, name) @@ -367,7 +367,7 @@ func generateLocalUserdataISO(userdata, name string) (string, error) { return "", errors.New("system is missing genisoimage or mkisofs, can't generate userdata iso without it") } - cmd := exec.Command(command, args...) + cmd := exec.CommandContext(ctx, command, args...) if output, err := cmd.CombinedOutput(); err != nil { return "", fmt.Errorf("error executing command `%s %s`: output: `%s`, error: `%w`", command, args, string(output), err) } diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index 31707cc63..61587db9e 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -397,7 +397,7 @@ func (p *provider) create(ctx context.Context, log *zap.SugaredLogger, machine * } if pc.OperatingSystem != providerconfig.OperatingSystemFlatcar { - localUserdataIsoFilePath, err := generateLocalUserdataISO(userdata, machine.Spec.Name) + localUserdataIsoFilePath, err := generateLocalUserdataISO(ctx, userdata, machine.Spec.Name) if err != nil { return nil, fmt.Errorf("failed to generate local userdadata iso: %w", err) } diff --git a/pkg/clusterinfo/configmap_test.go b/pkg/clusterinfo/configmap_test.go index c3a902423..1770914dc 100644 --- a/pkg/clusterinfo/configmap_test.go +++ b/pkg/clusterinfo/configmap_test.go @@ -42,7 +42,6 @@ clusters: contexts: null current-context: "" kind: Config -preferences: {} users: null ` clusterInfoKubeconfig2 = `apiVersion: v1 @@ -54,7 +53,6 @@ clusters: contexts: null current-context: "" kind: Config -preferences: {} users: null ` ) diff --git a/pkg/rhsm/satellite_subscription_manager.go b/pkg/rhsm/satellite_subscription_manager.go index 9dab43350..18dfb6ad2 100644 --- a/pkg/rhsm/satellite_subscription_manager.go +++ b/pkg/rhsm/satellite_subscription_manager.go @@ -96,7 +96,7 @@ func (s *DefaultSatelliteSubscriptionManager) executeDeleteRequest(ctx context.C requestURL.Host = serverURL requestURL.Path = path.Join("api", "v2", "hosts", machineName) - deleteHostRequest, err := http.NewRequest(http.MethodDelete, requestURL.String(), nil) + deleteHostRequest, err := http.NewRequestWithContext(ctx, http.MethodDelete, requestURL.String(), nil) deleteHostRequest = deleteHostRequest.WithContext(ctx) if err != nil { return fmt.Errorf("failed to create a delete host request: %w", err) diff --git a/pkg/rhsm/subscription_manager.go b/pkg/rhsm/subscription_manager.go index 40873dd3f..64f2da14d 100644 --- a/pkg/rhsm/subscription_manager.go +++ b/pkg/rhsm/subscription_manager.go @@ -150,7 +150,7 @@ func (d *defaultRedHatSubscriptionManager) findSystemsProfile(ctx context.Contex func (d *defaultRedHatSubscriptionManager) deleteSubscription(ctx context.Context, uuid, offlineToken string) error { client := newOAuthClientWithRefreshToken(ctx, offlineToken, d.authURL) - req, err := http.NewRequest("DELETE", fmt.Sprintf("%s/%s", d.apiURL, uuid), nil) + req, err := http.NewRequestWithContext(ctx, "DELETE", fmt.Sprintf("%s/%s", d.apiURL, uuid), nil) if err != nil { return fmt.Errorf("failed to create delete system request: %w", err) } @@ -179,7 +179,7 @@ func (d *defaultRedHatSubscriptionManager) deleteSubscription(ctx context.Contex func (d *defaultRedHatSubscriptionManager) executeFindSystemsRequest(ctx context.Context, offlineToken string, offset int) (*systemsResponse, error) { client := newOAuthClientWithRefreshToken(ctx, offlineToken, d.authURL) - req, err := http.NewRequest("GET", fmt.Sprintf(d.apiURL+"?limit=%v&offset=%v", d.requestsLimiter, offset), nil) + req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf(d.apiURL+"?limit=%v&offset=%v", d.requestsLimiter, offset), nil) if err != nil { return nil, fmt.Errorf("failed to create fetch systems request: %w", err) } diff --git a/test/e2e/provisioning/all_e2e_test.go b/test/e2e/provisioning/all_e2e_test.go index 632457625..ad94fa78b 100644 --- a/test/e2e/provisioning/all_e2e_test.go +++ b/test/e2e/provisioning/all_e2e_test.go @@ -85,7 +85,7 @@ const ( ) const ( - defaultKubernetesVersion = "1.29.9" + defaultKubernetesVersion = "v1.33.5" awsDefaultKubernetesVersion = "1.26.12" defaultContainerRuntime = "containerd" ) @@ -348,7 +348,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.26. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.29.9", "1.30.5", "1.31.1"))) + selector := Not(OsSelector("amzn2")) runScenarios(context.Background(), t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -424,7 +424,7 @@ func TestAWSProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := Not(VersionSelector("1.29.9", "1.30.5", "1.31.1")) + selector := OsSelector("amzn2", "ubuntu", "rhel", "rockylinux", "flatcar") // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -478,7 +478,7 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { } // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.29.9", "1.30.5", "1.31.1"))) + selector := OsSelector("ubuntu") // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -500,7 +500,7 @@ func TestAWSARMProvisioningE2E(t *testing.T) { t.Fatal("Unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } // In-tree cloud provider is not supported from Kubernetes v1.27. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.29.9", "1.30.5", "1.31.1"))) + selector := OsSelector("ubuntu") // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), @@ -573,7 +573,7 @@ func TestAzureProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(Not(OsSelector("amzn2")), Not(VersionSelector("1.30.5", "1.31.1"))) + selector := Not(OsSelector("amzn2")) // act params := []string{ @@ -602,7 +602,7 @@ func TestAzureCustomImageReferenceProvisioningE2E(t *testing.T) { } // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.5", "1.31.1"))) + selector := OsSelector("ubuntu") // act params := []string{ fmt.Sprintf("<< AZURE_TENANT_ID >>=%s", azureTenantID), @@ -816,7 +816,7 @@ func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.5", "1.31.1"))) + selector := OsSelector("ubuntu") params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) @@ -828,7 +828,7 @@ func TestVsphereMultipleNICProvisioningE2E(t *testing.T) { t.Parallel() // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(OsSelector("ubuntu"), Not(VersionSelector("1.30.5", "1.31.1"))) + selector := OsSelector("ubuntu") params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereMultipleNICManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) @@ -857,7 +857,7 @@ func TestVsphereDatastoreClusterProvisioningE2E(t *testing.T) { t.Parallel() // In-tree cloud provider is not supported from Kubernetes v1.30. - selector := And(OsSelector("ubuntu", "rhel", "flatcar"), Not(VersionSelector("1.30.5", "1.31.1"))) + selector := OsSelector("ubuntu", "rhel", "flatcar") params := getVSphereTestParams(t) runScenarios(context.Background(), t, selector, params, VSPhereDSCManifest, fmt.Sprintf("vs-dsc-%s", *testRunIdentifier)) diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index 9fca707ea..d22e89473 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -34,9 +34,10 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.29.9"), - semver.MustParse("v1.30.5"), - semver.MustParse("v1.31.1"), + semver.MustParse("v1.31.13"), + semver.MustParse("v1.32.9"), + semver.MustParse("v1.33.5"), + semver.MustParse("v1.34.1"), } operatingSystems = []providerconfigtypes.OperatingSystem{ From ff105859cf2272d96a5036d1179a25ece83369e9 Mon Sep 17 00:00:00 2001 From: Kai Fink Date: Mon, 22 Sep 2025 10:50:38 +0200 Subject: [PATCH 486/489] Add support to provisioning vcloud workers with multiple nics/networks (#1941) * Add support to provision vcloud workers with multiple nics/networks Signed-off-by: Kai Fink * Migrate VMware Cloud Director network settings to support multi-network configurations Signed-off-by: Kai Fink * Prepend instead of append, to ensure the same network stays the primary Signed-off-by: Kai Fink * Deduplicate networks to avoid duplicate entries in configurations Signed-off-by: Kai Fink * Remove unnecessary newline in error handling block Signed-off-by: Kai Fink --------- Signed-off-by: Kai Fink --- .../machinedeployments_validation.go | 8 ++++ pkg/admission/util.go | 39 +++++++++++++++++++ .../provider/vmwareclouddirector/helper.go | 24 +++++++----- .../provider/vmwareclouddirector/provider.go | 31 ++++++++++++--- .../vmwareclouddirector/types.go | 6 ++- ...chinedeployment-vmware-cloud-director.yaml | 3 +- 6 files changed, 93 insertions(+), 18 deletions(-) diff --git a/pkg/admission/machinedeployments_validation.go b/pkg/admission/machinedeployments_validation.go index 7079d9ba6..2e58d7da5 100644 --- a/pkg/admission/machinedeployments_validation.go +++ b/pkg/admission/machinedeployments_validation.go @@ -128,6 +128,14 @@ func mutationsForMachineDeployment(md *clusterv1alpha1.MachineDeployment) error } } + // Migrate + if providerConfig.CloudProvider == providerconfigtypes.CloudProviderVMwareCloudDirector { + err := migrateVMwareCloudDirector(providerConfig) + if err != nil { + return fmt.Errorf("failed to migrate VMware Cloud Director Network Settings: %w", err) + } + } + // Update value in original object md.Spec.Template.Spec.ProviderSpec.Value.Raw, err = json.Marshal(providerConfig) if err != nil { diff --git a/pkg/admission/util.go b/pkg/admission/util.go index d435b356f..1e49dbf26 100644 --- a/pkg/admission/util.go +++ b/pkg/admission/util.go @@ -20,6 +20,7 @@ import ( "encoding/json" "fmt" + vcdtypes "k8c.io/machine-controller/sdk/cloudprovider/vmwareclouddirector" providerconfigtypes "k8c.io/machine-controller/sdk/providerconfig" ) @@ -49,3 +50,41 @@ func migrateToEquinixMetal(providerConfig *providerconfigtypes.Config) (err erro } return nil } + +func migrateVMwareCloudDirector(providerConfig *providerconfigtypes.Config) (err error) { + config, err := vcdtypes.GetConfig(*providerConfig) + if err != nil { + return fmt.Errorf("failed to get vcd config: %w", err) + } + + if config.Network.Value != "" { + config.Networks = append([]providerconfigtypes.ConfigVarString{config.Network}, config.Networks...) + config.Network.Value = "" + p := &providerconfigtypes.ConfigVarString{Value: ""} + config.Network = *p + } + + config.Networks = Deduplicate(config.Networks) + + cloudProviderSpecRaw, err := json.Marshal(config) + if err != nil { + return fmt.Errorf("failed to marshal cloudProviderConfig: %w", err) + } + + providerConfig.CloudProviderSpec.Raw = cloudProviderSpecRaw + return nil +} + +func Deduplicate[T comparable](slice []T) []T { + seen := make(map[T]struct{}) + result := []T{} + + for _, val := range slice { + if _, exists := seen[val]; !exists { + seen[val] = struct{}{} + result = append(result, val) + } + } + + return result +} diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go index 9883e1ca4..6da1d1777 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/helper.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/helper.go @@ -123,6 +123,20 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * } } + var networkConnections []*vcdapitypes.NetworkConnection + for i, network := range c.Networks { + networkConnections = append(networkConnections, &vcdapitypes.NetworkConnection{ + Network: network, + NeedsCustomization: false, + IsConnected: true, + IPAddressAllocationMode: string(c.IPAllocationMode), + NetworkAdapterType: "VMXNET3", + NetworkConnectionIndex: i, + }) + } + + fmt.Printf("network connections: %+v\n", networkConnections) + // 4. At this point we are ready to create our initial VMs. // // Multiple API calls to re-compose the vApp are handled in a synchronous manner, where each request has to wait @@ -145,15 +159,7 @@ func createVM(client *Client, machine *clusterv1alpha1.Machine, c *Config, org * }, InstantiationParams: &vcdapitypes.InstantiationParams{ NetworkConnectionSection: &vcdapitypes.NetworkConnectionSection{ - NetworkConnection: []*vcdapitypes.NetworkConnection{ - { - Network: c.Network, - NeedsCustomization: false, - IsConnected: true, - IPAddressAllocationMode: string(c.IPAllocationMode), - NetworkAdapterType: "VMXNET3", - }, - }, + NetworkConnection: networkConnections, }, }, StorageProfile: storageProfile, diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go index 9d649d65e..bc92e48e2 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -79,7 +79,7 @@ type Config struct { SizingPolicy *string // Network configuration. - Network string + Networks []string IPAllocationMode vcdtypes.IPAllocationMode // Compute configuration. @@ -375,11 +375,23 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, nil, err } - c.Network, err = p.configVarResolver.GetStringValue(rawConfig.Network) + singleNetwork, err := p.configVarResolver.GetStringValue(rawConfig.Network) if err != nil { return nil, nil, nil, err } + if singleNetwork != "" { + c.Networks = append([]string{singleNetwork}, c.Networks...) + } + + for _, network := range rawConfig.Networks { + networkValue, err := p.configVarResolver.GetStringValue(network) + if err != nil { + return nil, nil, nil, err + } + c.Networks = append(c.Networks, networkValue) + } + c.IPAllocationMode = rawConfig.IPAllocationMode if rawConfig.DiskSizeGB != nil && *rawConfig.DiskSizeGB < 0 { @@ -509,11 +521,18 @@ func (p *provider) Validate(_ context.Context, _ *zap.SugaredLogger, spec cluste return fmt.Errorf("diskSizeGB '%v' cannot be less than the template size '%v': %w", *c.DiskSizeGB, catalogItem.CatalogItem.Size, err) } - // Ensure that the network exists + // Ensure that the networks exists // It can either be a vApp network or a vApp Org network. - _, err = GetVappNetworkType(c.Network, *vapp) - if err != nil { - return fmt.Errorf("failed to get network '%s' for vapp '%s': %w", c.Network, c.VApp, err) + + if len(c.Networks) == 0 { + return fmt.Errorf("at least one network must be specified") + } + + for _, network := range c.Networks { + _, err = GetVappNetworkType(network, *vapp) + if err != nil { + return fmt.Errorf("failed to get network '%s' for vapp '%s': %w", network, c.VApp, err) + } } if c.SizingPolicy != nil || c.PlacementPolicy != nil { diff --git a/sdk/cloudprovider/vmwareclouddirector/types.go b/sdk/cloudprovider/vmwareclouddirector/types.go index f09afbec9..e53ea1cd5 100644 --- a/sdk/cloudprovider/vmwareclouddirector/types.go +++ b/sdk/cloudprovider/vmwareclouddirector/types.go @@ -46,8 +46,10 @@ type RawConfig struct { PlacementPolicy *string `json:"placementPolicy,omitempty"` // Network configuration. - Network providerconfig.ConfigVarString `json:"network"` - IPAllocationMode IPAllocationMode `json:"ipAllocationMode,omitempty"` + // Deprecated: Use networks instead. + Network providerconfig.ConfigVarString `json:"network,omitempty"` + Networks []providerconfig.ConfigVarString `json:"networks"` + IPAllocationMode IPAllocationMode `json:"ipAllocationMode,omitempty"` // Compute configuration. CPUs int64 `json:"cpus"` diff --git a/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml b/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml index 8ce2a4fc0..2cce44033 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml @@ -36,7 +36,8 @@ spec: vapp: "kubermatic-e2e" catalog: "kubermatic" template: "machine-controller-<< OS_NAME >>" - network: "kubermatic-e2e-routed-network" + networks: + - "kubermatic-e2e-routed-network" ipAllocationMode: "DHCP" cpus: 2 cpuCores: 1 From 1b7f31166cf4827955e52acb8581a7aa48eebba4 Mon Sep 17 00:00:00 2001 From: soer3n <43064202+soer3n@users.noreply.github.com> Date: Thu, 25 Sep 2025 16:07:42 +0200 Subject: [PATCH 487/489] add support for nvme devices for tinkerbell provider (#1921) * change rendering of block device to respect that nvme device paths have a 'p' as a prefix Signed-off-by: soer3n --- .../plugins/tinkerbell/client/template.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go index 4bc1cffaf..7e77b317c 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/client/template.go @@ -197,17 +197,17 @@ func createGrowPartitionAction(destDisk string) Action { Image: "quay.io/tinkerbell/actions/cexec:c5bde803d9f6c90f1a9d5e06930d856d1481854c", Timeout: 90, Environment: map[string]string{ - "BLOCK_DEVICE": fmt.Sprintf("{{ index .Hardware.Disks 0 }}%s", PartitionNumber), + "BLOCK_DEVICE": "{{ formatPartition ( index .Hardware.Disks 0 ) (.partition_number | int) }}", "FS_TYPE": fsType, "CHROOT": "y", "DEFAULT_INTERPRETER": defaultInterpreter, - "CMD_LINE": fmt.Sprintf("growpart %s %s && resize2fs %s%s", destDisk, PartitionNumber, destDisk, PartitionNumber), + "CMD_LINE": fmt.Sprintf("growpart %s %s && resize2fs '{{ formatPartition ( index .Hardware.Disks 0 ) (.partition_number | int) }}'", destDisk, PartitionNumber), }, } } func createNetworkConfigAction() Action { - netplaneConfig := ` + netplanConfig := ` network: version: 2 renderer: networkd @@ -227,10 +227,10 @@ network: Image: "quay.io/tinkerbell-actions/writefile:v1.0.0", Timeout: 90, Environment: map[string]string{ - "DEST_DISK": fmt.Sprintf("{{ index .Hardware.Disks 0 }}%s", PartitionNumber), + "DEST_DISK": "{{ formatPartition ( index .Hardware.Disks 0 ) (.partition_number | int) }}", "FS_TYPE": fsType, "DEST_PATH": "/etc/netplan/config.yaml", - "CONTENTS": netplaneConfig, + "CONTENTS": netplanConfig, "UID": "0", "GID": "0", "MODE": "0644", @@ -252,7 +252,7 @@ echo 'local-hostname: {{.hardware_name}}' >> /var/lib/cloud/seed/nocloud/meta-da Image: "quay.io/tinkerbell-actions/cexec:v1.0.0", Timeout: 90, Environment: map[string]string{ - "BLOCK_DEVICE": fmt.Sprintf("{{ index .Hardware.Disks 0 }}%s", PartitionNumber), + "BLOCK_DEVICE": "{{ formatPartition ( index .Hardware.Disks 0 ) (.partition_number | int) }}", "FS_TYPE": fsType, "CHROOT": "y", "DEFAULT_INTERPRETER": defaultInterpreter, @@ -267,7 +267,7 @@ func decodeCloudInitFile(hardwareName string) Action { Image: "quay.io/tinkerbell/actions/cexec:latest", Timeout: 90, Environment: map[string]string{ - "BLOCK_DEVICE": fmt.Sprintf("{{ index .Hardware.Disks 0 }}%s", PartitionNumber), + "BLOCK_DEVICE": "{{ formatPartition ( index .Hardware.Disks 0 ) (.partition_number | int) }}", "FS_TYPE": fsType, "CHROOT": "y", "DEFAULT_INTERPRETER": "/bin/sh -c", From c32d551cfde12f0423ec478e4ecd7e28e6798925 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Mon, 13 Oct 2025 11:34:26 +0300 Subject: [PATCH 488/489] Add new kubelet configs (#1972) Signed-off-by: Artiom Diomin --- sdk/apis/cluster/common/consts.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sdk/apis/cluster/common/consts.go b/sdk/apis/cluster/common/consts.go index 34eeacacf..7e4bbde5d 100644 --- a/sdk/apis/cluster/common/consts.go +++ b/sdk/apis/cluster/common/consts.go @@ -139,6 +139,10 @@ const ( ContainerLogMaxSizeKubeletConfig = "ContainerLogMaxSize" ContainerLogMaxFilesKubeletConfig = "ContainerLogMaxFiles" MaxPodsKubeletConfig = "MaxPods" + ImageGCHighThresholdPercent = "ImageGCHighThresholdPercent" + ImageGCLowThresholdPercent = "ImageGCLowThresholdPercent" + ImageMinimumGCAge = "ImageMinimumGCAge" + ImageMaximumGCAge = "ImageMaximumGCAge" ) const ( From b362a3a0fa305092e0142f638aa3c817c1c31c75 Mon Sep 17 00:00:00 2001 From: Artiom Diomin Date: Mon, 13 Oct 2025 12:58:25 +0300 Subject: [PATCH 489/489] Move some pkg constants to sdk. (#1973) Alias them back for backward compatibility. Signed-off-by: Artiom Diomin --- pkg/controller/util/machine_deployment.go | 46 +++++++---------------- sdk/apis/cluster/common/consts.go | 31 +++++++++++++++ 2 files changed, 45 insertions(+), 32 deletions(-) diff --git a/pkg/controller/util/machine_deployment.go b/pkg/controller/util/machine_deployment.go index f6b0ef24b..ee6df5c10 100644 --- a/pkg/controller/util/machine_deployment.go +++ b/pkg/controller/util/machine_deployment.go @@ -27,7 +27,7 @@ import ( "github.com/davecgh/go-spew/spew" "go.uber.org/zap" - "k8c.io/machine-controller/sdk/apis/cluster/common" + sdkclustercommon "k8c.io/machine-controller/sdk/apis/cluster/common" clusterv1alpha1 "k8c.io/machine-controller/sdk/apis/cluster/v1alpha1" corev1 "k8s.io/api/core/v1" @@ -41,34 +41,16 @@ import ( ) const ( - DefaultMachineDeploymentUniqueLabelKey = "machine-template-hash" - - // RevisionAnnotation is the revision annotation of a machine deployment's machine sets which records its rollout sequence. - RevisionAnnotation = "machinedeployment.clusters.k8s.io/revision" - // RevisionHistoryAnnotation maintains the history of all old revisions that a machine set has served for a machine deployment. - RevisionHistoryAnnotation = "machinedeployment.clusters.k8s.io/revision-history" - // DesiredReplicasAnnotation is the desired replicas for a machine deployment recorded as an annotation - // in its machine sets. Helps in separating scaling events from the rollout process and for - // determining if the new machine set for a deployment is really saturated. - DesiredReplicasAnnotation = "machinedeployment.clusters.k8s.io/desired-replicas" - // MaxReplicasAnnotation is the maximum replicas a deployment can have at a given point, which - // is machinedeployment.spec.replicas + maxSurge. Used by the underlying machine sets to estimate their - // proportions in case the deployment has surge replicas. - MaxReplicasAnnotation = "machinedeployment.clusters.k8s.io/max-replicas" - - // FailedMSCreateReason is added in a machine deployment when it cannot create a new machine set. - FailedMSCreateReason = "MachineSetCreateError" - // FoundNewMSReason is added in a machine deployment when it adopts an existing machine set. - FoundNewMSReason = "FoundNewMachineSet" - // PausedDeployReason is added in a deployment when it is paused. Lack of progress shouldn't be - // estimated once a deployment is paused. - PausedDeployReason = "DeploymentPaused" - - // MinimumReplicasAvailable is added in a deployment when it has its minimum replicas required available. - MinimumReplicasAvailable = "MinimumReplicasAvailable" - // MinimumReplicasUnavailable is added in a deployment when it doesn't have the minimum required replicas - // available. - MinimumReplicasUnavailable = "MinimumReplicasUnavailable" + DefaultMachineDeploymentUniqueLabelKey = sdkclustercommon.DefaultMachineDeploymentUniqueLabelKey + RevisionAnnotation = sdkclustercommon.RevisionAnnotation + RevisionHistoryAnnotation = sdkclustercommon.RevisionHistoryAnnotation + DesiredReplicasAnnotation = sdkclustercommon.DesiredReplicasAnnotation + MaxReplicasAnnotation = sdkclustercommon.MaxReplicasAnnotation + FailedMSCreateReason = sdkclustercommon.FailedMSCreateReason + FoundNewMSReason = sdkclustercommon.FoundNewMSReason + PausedDeployReason = sdkclustercommon.PausedDeployReason + MinimumReplicasAvailable = sdkclustercommon.MinimumReplicasAvailable + MinimumReplicasUnavailable = sdkclustercommon.MinimumReplicasUnavailable ) // MachineSetsByCreationTimestamp sorts a list of MachineSet by creation timestamp, using their names as a tie breaker. @@ -241,7 +223,7 @@ func SetNewMachineSetAnnotations(mdLog *zap.SugaredLogger, deployment *clusterv1 msLog.Infow("MachineSet revision annotation is not a valid integer", "value", oldRevision, zap.Error(err)) return false } - //If the MS annotation is empty then initialise it to 0 + // If the MS annotation is empty then initialise it to 0 oldRevisionInt = 0 } @@ -505,7 +487,7 @@ func GetAvailableReplicaCountForMachineSets(machineSets []*clusterv1alpha1.Machi // IsRollingUpdate returns true if the strategy type is a rolling update. func IsRollingUpdate(deployment *clusterv1alpha1.MachineDeployment) bool { - return deployment.Spec.Strategy.Type == common.RollingUpdateMachineDeploymentStrategyType + return deployment.Spec.Strategy.Type == sdkclustercommon.RollingUpdateMachineDeploymentStrategyType } // DeploymentComplete considers a deployment to be complete once all of its desired replicas @@ -523,7 +505,7 @@ func DeploymentComplete(deployment *clusterv1alpha1.MachineDeployment, newStatus // 2) Max number of machines allowed is reached: deployment's replicas + maxSurge == all MSs' replicas. func NewMSNewReplicas(deployment *clusterv1alpha1.MachineDeployment, allMSs []*clusterv1alpha1.MachineSet, newMS *clusterv1alpha1.MachineSet) (int32, error) { switch deployment.Spec.Strategy.Type { - case common.RollingUpdateMachineDeploymentStrategyType: + case sdkclustercommon.RollingUpdateMachineDeploymentStrategyType: // Check if we can scale up. maxSurge, err := intstrutil.GetValueFromIntOrPercent(deployment.Spec.Strategy.RollingUpdate.MaxSurge, int(*(deployment.Spec.Replicas)), true) if err != nil { diff --git a/sdk/apis/cluster/common/consts.go b/sdk/apis/cluster/common/consts.go index 7e4bbde5d..add594654 100644 --- a/sdk/apis/cluster/common/consts.go +++ b/sdk/apis/cluster/common/consts.go @@ -132,6 +132,37 @@ const ( ExternalCloudProviderKubeletFlag KubeletFlags = "ExternalCloudProvider" ) +const ( + DefaultMachineDeploymentUniqueLabelKey = "machine-template-hash" + + // RevisionAnnotation is the revision annotation of a machine deployment's machine sets which records its rollout sequence. + RevisionAnnotation = "machinedeployment.clusters.k8s.io/revision" + // RevisionHistoryAnnotation maintains the history of all old revisions that a machine set has served for a machine deployment. + RevisionHistoryAnnotation = "machinedeployment.clusters.k8s.io/revision-history" + // DesiredReplicasAnnotation is the desired replicas for a machine deployment recorded as an annotation + // in its machine sets. Helps in separating scaling events from the rollout process and for + // determining if the new machine set for a deployment is really saturated. + DesiredReplicasAnnotation = "machinedeployment.clusters.k8s.io/desired-replicas" + // MaxReplicasAnnotation is the maximum replicas a deployment can have at a given point, which + // is machinedeployment.spec.replicas + maxSurge. Used by the underlying machine sets to estimate their + // proportions in case the deployment has surge replicas. + MaxReplicasAnnotation = "machinedeployment.clusters.k8s.io/max-replicas" + + // FailedMSCreateReason is added in a machine deployment when it cannot create a new machine set. + FailedMSCreateReason = "MachineSetCreateError" + // FoundNewMSReason is added in a machine deployment when it adopts an existing machine set. + FoundNewMSReason = "FoundNewMachineSet" + // PausedDeployReason is added in a deployment when it is paused. Lack of progress shouldn't be + // estimated once a deployment is paused. + PausedDeployReason = "DeploymentPaused" + + // MinimumReplicasAvailable is added in a deployment when it has its minimum replicas required available. + MinimumReplicasAvailable = "MinimumReplicasAvailable" + // MinimumReplicasUnavailable is added in a deployment when it doesn't have the minimum required replicas + // available. + MinimumReplicasUnavailable = "MinimumReplicasUnavailable" +) + const ( SystemReservedKubeletConfig = "SystemReserved" KubeReservedKubeletConfig = "KubeReserved"